From bd998017d5ffd877c1dde65aa6855e7774c760be Mon Sep 17 00:00:00 2001 From: John Kung Date: Fri, 23 Nov 2018 14:05:44 -0500 Subject: [PATCH] SysInv Decoupling: Create Inventory Service Create host inventory services (api, conductor and agent) and python-inventoryclient. The inventory service collects the host resources and provides a REST API and client to expose the host resources. Create plugin for integration with system configuration (sysinv) service. This is the initial inventory service infratructure commit. Puppet configuration, SM integration and host integration with sysinv(systemconfig) changes are pending and planned to be delivered in future commits. Tests Performed: Verify the changes are inert on config_controller installation and provisioning. Puppet and spec changes are required in order to create keystone, database and activate inventory services. Unit tests performed (when puppet configuration for keystone, database is applied): Trigger host configure_check, configure signals into systemconfig(sysinv). Verify python-inventoryclient and api service: Disks and related storage resources are pending. inventory host-cpu-list/show inventory host-device-list/show/modify inventory host-ethernetport-list/show inventory host-lldp-neighbor-list inventory host-lldp-agent-list/show inventory host-memory-list/show inventory host-node-list/show inventory host-port-list/show Tox Unit tests: inventory: pep8 python-inventoryclient: py27, pep8, cover, pylint Change-Id: I744ac0de098608c55b9356abf180cc36601cfb8d Story: 2002950 Task: 22952 Signed-off-by: John Kung --- centos_iso_image.inc | 4 + centos_pkg_dirs | 2 + inventory/PKG-INFO | 13 + inventory/centos/build_srpm.data | 2 + inventory/centos/inventory.spec | 195 + inventory/inventory/.coveragerc | 6 + inventory/inventory/.gitignore | 59 + inventory/inventory/.mailmap | 3 + inventory/inventory/.stestr.conf | 3 + inventory/inventory/CONTRIBUTING.rst | 19 + inventory/inventory/HACKING.rst | 4 + inventory/inventory/LICENSE | 176 + inventory/inventory/README.rst | 3 + inventory/inventory/babel.cfg | 2 + inventory/inventory/doc/requirements.txt | 4 + .../inventory/doc/source/admin/index.rst | 5 + inventory/inventory/doc/source/cli/index.rst | 5 + inventory/inventory/doc/source/conf.py | 82 + .../doc/source/configuration/index.rst | 5 + .../doc/source/contributor/contributing.rst | 4 + .../doc/source/contributor/index.rst | 9 + inventory/inventory/doc/source/index.rst | 30 + .../doc/source/install/common_configure.rst | 10 + .../source/install/common_prerequisites.rst | 75 + .../doc/source/install/get_started.rst | 9 + .../inventory/doc/source/install/index.rst | 17 + .../doc/source/install/install-obs.rst | 34 + .../doc/source/install/install-rdo.rst | 33 + .../doc/source/install/install-ubuntu.rst | 31 + .../inventory/doc/source/install/install.rst | 20 + .../doc/source/install/next-steps.rst | 9 + .../inventory/doc/source/install/verify.rst | 24 + .../inventory/doc/source/library/index.rst | 7 + inventory/inventory/doc/source/readme.rst | 1 + .../inventory/doc/source/reference/index.rst | 5 + inventory/inventory/doc/source/user/index.rst | 5 + .../inventory/etc/inventory/delete_load.sh | 20 + .../etc/inventory/inventory-agent-pmond.conf | 9 + .../inventory/inventory_goenabled_check.sh | 36 + inventory/inventory/etc/inventory/motd-system | 10 + inventory/inventory/etc/inventory/policy.json | 5 + inventory/inventory/inventory/__init__.py | 11 + .../inventory/inventory/agent/__init__.py | 0 .../inventory/inventory/agent/base_manager.py | 114 + inventory/inventory/inventory/agent/disk.py | 369 ++ .../inventory/agent/lldp/__init__.py | 0 .../inventory/inventory/agent/lldp/config.py | 23 + .../inventory/agent/lldp/drivers/__init__.py | 0 .../inventory/agent/lldp/drivers/base.py | 47 + .../agent/lldp/drivers/lldpd/__init__.py | 0 .../agent/lldp/drivers/lldpd/driver.py | 321 ++ .../agent/lldp/drivers/ovs/__init__.py | 0 .../agent/lldp/drivers/ovs/driver.py | 167 + .../inventory/inventory/agent/lldp/manager.py | 176 + .../inventory/inventory/agent/lldp/plugin.py | 246 ++ .../inventory/inventory/agent/manager.py | 973 +++++ inventory/inventory/inventory/agent/node.py | 608 +++ inventory/inventory/inventory/agent/pci.py | 621 +++ inventory/inventory/inventory/agent/rpcapi.py | 161 + inventory/inventory/inventory/api/__init__.py | 0 inventory/inventory/inventory/api/app.py | 90 + inventory/inventory/inventory/api/config.py | 73 + .../inventory/api/controllers/__init__.py | 0 .../inventory/api/controllers/root.py | 115 + .../inventory/api/controllers/v1/__init__.py | 198 + .../inventory/api/controllers/v1/base.py | 130 + .../api/controllers/v1/collection.py | 58 + .../inventory/api/controllers/v1/cpu.py | 303 ++ .../inventory/api/controllers/v1/cpu_utils.py | 330 ++ .../api/controllers/v1/ethernet_port.py | 310 ++ .../inventory/api/controllers/v1/host.py | 3585 +++++++++++++++++ .../inventory/api/controllers/v1/link.py | 58 + .../api/controllers/v1/lldp_agent.py | 366 ++ .../api/controllers/v1/lldp_neighbour.py | 390 ++ .../inventory/api/controllers/v1/lldp_tlv.py | 297 ++ .../inventory/api/controllers/v1/memory.py | 729 ++++ .../inventory/api/controllers/v1/node.py | 261 ++ .../api/controllers/v1/pci_device.py | 313 ++ .../inventory/api/controllers/v1/port.py | 334 ++ .../inventory/api/controllers/v1/query.py | 168 + .../inventory/api/controllers/v1/sensor.py | 586 +++ .../api/controllers/v1/sensorgroup.py | 751 ++++ .../inventory/api/controllers/v1/state.py | 39 + .../inventory/api/controllers/v1/sysinv.py | 49 + .../inventory/api/controllers/v1/system.py | 266 ++ .../inventory/api/controllers/v1/types.py | 215 + .../inventory/api/controllers/v1/utils.py | 567 +++ .../inventory/api/controllers/v1/versions.py | 66 + inventory/inventory/inventory/api/hooks.py | 110 + .../inventory/api/middleware/__init__.py | 19 + .../inventory/api/middleware/auth_token.py | 75 + .../api/middleware/parsable_error.py | 99 + inventory/inventory/inventory/cmd/__init__.py | 31 + inventory/inventory/inventory/cmd/agent.py | 58 + inventory/inventory/inventory/cmd/api.py | 86 + .../inventory/inventory/cmd/conductor.py | 55 + inventory/inventory/inventory/cmd/dbsync.py | 19 + .../inventory/cmd/dnsmasq_lease_update.py | 133 + .../inventory/inventory/common/__init__.py | 0 inventory/inventory/inventory/common/base.py | 43 + inventory/inventory/inventory/common/ceph.py | 211 + .../inventory/inventory/common/config.py | 127 + .../inventory/inventory/common/constants.py | 612 +++ .../inventory/inventory/common/context.py | 153 + .../inventory/inventory/common/exception.py | 738 ++++ inventory/inventory/inventory/common/fm.py | 104 + .../inventory/inventory/common/health.py | 289 ++ .../inventory/inventory/common/hwmon_api.py | 184 + inventory/inventory/inventory/common/i18n.py | 12 + .../inventory/inventory/common/k_host.py | 110 + .../inventory/inventory/common/k_host_agg.py | 15 + .../inventory/inventory/common/k_lldp.py | 45 + inventory/inventory/inventory/common/k_pci.py | 25 + .../inventory/inventory/common/keystone.py | 121 + .../inventory/inventory/common/mtce_api.py | 102 + .../inventory/inventory/common/patch_api.py | 60 + .../inventory/inventory/common/policy.py | 96 + .../inventory/inventory/common/rest_api.py | 73 + inventory/inventory/inventory/common/rpc.py | 153 + .../inventory/inventory/common/rpc_service.py | 95 + .../inventory/inventory/common/service.py | 47 + .../inventory/inventory/common/sm_api.py | 184 + .../inventory/common/storage_backend_conf.py | 450 +++ inventory/inventory/inventory/common/utils.py | 1263 ++++++ .../inventory/inventory/common/vim_api.py | 156 + .../inventory/inventory/conductor/__init__.py | 0 .../inventory/conductor/base_manager.py | 118 + .../inventory/inventory/conductor/manager.py | 1946 +++++++++ .../inventory/conductor/openstack.py | 878 ++++ .../inventory/inventory/conductor/rpcapi.py | 571 +++ .../inventory/inventory/conf/__init__.py | 25 + .../inventory/inventory/conf/database.py | 31 + inventory/inventory/inventory/conf/default.py | 121 + inventory/inventory/inventory/conf/opts.py | 31 + .../inventory/inventory/config-generator.conf | 18 + inventory/inventory/inventory/db/__init__.py | 0 inventory/inventory/inventory/db/api.py | 45 + inventory/inventory/inventory/db/migration.py | 56 + .../inventory/db/sqlalchemy/__init__.py | 0 .../inventory/inventory/db/sqlalchemy/api.py | 2570 ++++++++++++ .../db/sqlalchemy/migrate_repo/README | 4 + .../db/sqlalchemy/migrate_repo/__init__.py | 0 .../db/sqlalchemy/migrate_repo/manage.py | 11 + .../db/sqlalchemy/migrate_repo/migrate.cfg | 21 + .../migrate_repo/versions/001_init.py | 605 +++ .../migrate_repo/versions/__init__.py | 0 .../inventory/db/sqlalchemy/migration.py | 69 + .../inventory/db/sqlalchemy/models.py | 589 +++ inventory/inventory/inventory/db/utils.py | 48 + .../inventory/inventory/objects/__init__.py | 43 + inventory/inventory/inventory/objects/base.py | 345 ++ inventory/inventory/inventory/objects/cpu.py | 119 + .../inventory/inventory/objects/fields.py | 160 + inventory/inventory/inventory/objects/host.py | 118 + .../inventory/inventory/objects/lldp_agent.py | 122 + .../inventory/objects/lldp_neighbour.py | 124 + .../inventory/inventory/objects/lldp_tlv.py | 114 + .../inventory/inventory/objects/memory.py | 141 + inventory/inventory/inventory/objects/node.py | 73 + .../inventory/inventory/objects/pci_device.py | 99 + inventory/inventory/inventory/objects/port.py | 117 + .../inventory/objects/port_ethernet.py | 93 + .../inventory/inventory/objects/sensor.py | 82 + .../inventory/objects/sensor_analog.py | 72 + .../inventory/objects/sensor_discrete.py | 60 + .../inventory/objects/sensorgroup.py | 86 + .../inventory/objects/sensorgroup_analog.py | 68 + .../inventory/objects/sensorgroup_discrete.py | 58 + .../inventory/inventory/objects/system.py | 87 + .../inventory/systemconfig/__init__.py | 0 .../inventory/systemconfig/config.py | 19 + .../systemconfig/drivers/__init__.py | 0 .../inventory/systemconfig/drivers/base.py | 42 + .../systemconfig/drivers/sysinv/__init__.py | 0 .../systemconfig/drivers/sysinv/driver.py | 235 ++ .../inventory/systemconfig/manager.py | 179 + .../inventory/systemconfig/plugin.py | 176 + .../inventory/inventory/tests/__init__.py | 0 inventory/inventory/inventory/tests/base.py | 23 + .../inventory/tests/test_inventory.py | 28 + inventory/inventory/inventory/version.py | 18 + .../inventory/releasenotes/notes/.placeholder | 0 .../releasenotes/source/_static/.placeholder | 0 .../source/_templates/.placeholder | 0 .../inventory/releasenotes/source/conf.py | 281 ++ .../inventory/releasenotes/source/index.rst | 8 + .../releasenotes/source/unreleased.rst | 5 + inventory/inventory/requirements.txt | 47 + .../inventory/scripts/inventory-agent-initd | 204 + .../inventory/scripts/inventory-agent.service | 15 + inventory/inventory/scripts/inventory-api | 409 ++ .../inventory/scripts/inventory-api.service | 15 + .../inventory/scripts/inventory-conductor | 357 ++ .../scripts/inventory-conductor.service | 15 + inventory/inventory/setup.cfg | 57 + inventory/inventory/setup.py | 29 + inventory/inventory/test-requirements.txt | 37 + inventory/inventory/tox.ini | 101 + python-inventoryclient/PKG-INFO | 13 + python-inventoryclient/centos/build_srpm.data | 2 + .../centos/python-inventoryclient.spec | 82 + .../inventoryclient/.gitignore | 35 + .../inventoryclient/.testr.conf | 10 + .../inventoryclient/LICENSE | 176 + .../inventoryclient/__init__.py | 22 + .../inventoryclient/inventoryclient/client.py | 92 + .../inventoryclient/common/__init__.py | 0 .../inventoryclient/common/base.py | 164 + .../inventoryclient/common/cli_no_wrap.py | 42 + .../inventoryclient/common/exceptions.py | 226 ++ .../inventoryclient/common/http.py | 402 ++ .../inventoryclient/common/i18n.py | 13 + .../inventoryclient/common/options.py | 136 + .../inventoryclient/common/utils.py | 777 ++++ .../common/wrapping_formatters.py | 871 ++++ .../inventoryclient/inventoryclient/exc.py | 101 + .../inventoryclient/inventoryclient/shell.py | 326 ++ .../inventoryclient/tests/__init__.py | 0 .../inventoryclient/tests/test_shell.py | 92 + .../inventoryclient/tests/test_utils.py | 92 + .../inventoryclient/tests/utils.py | 69 + .../inventoryclient/v1/__init__.py | 0 .../inventoryclient/v1/client.py | 58 + .../inventoryclient/inventoryclient/v1/cpu.py | 206 + .../inventoryclient/v1/cpu_shell.py | 76 + .../inventoryclient/v1/ethernetport.py | 65 + .../inventoryclient/v1/ethernetport_shell.py | 89 + .../inventoryclient/v1/host.py | 116 + .../inventoryclient/v1/host_shell.py | 393 ++ .../inventoryclient/v1/lldp_agent.py | 35 + .../inventoryclient/v1/lldp_agent_shell.py | 97 + .../inventoryclient/v1/lldp_neighbour.py | 35 + .../v1/lldp_neighbour_shell.py | 102 + .../inventoryclient/v1/memory.py | 63 + .../inventoryclient/v1/memory_shell.py | 197 + .../inventoryclient/v1/node.py | 54 + .../inventoryclient/v1/node_shell.py | 63 + .../inventoryclient/v1/pci_device.py | 47 + .../inventoryclient/v1/pci_device_shell.py | 126 + .../inventoryclient/v1/port.py | 39 + .../inventoryclient/v1/port_shell.py | 104 + .../inventoryclient/v1/shell.py | 41 + .../inventoryclient/pylint.rc | 218 + .../inventoryclient/setup.cfg | 33 + .../inventoryclient/setup.py | 30 + .../inventoryclient/test-requirements.txt | 23 + .../tools/inventory.bash_completion | 33 + .../inventoryclient/tox.ini | 64 + tox.ini | 9 +- 249 files changed, 40028 insertions(+), 1 deletion(-) create mode 100644 inventory/PKG-INFO create mode 100644 inventory/centos/build_srpm.data create mode 100644 inventory/centos/inventory.spec create mode 100644 inventory/inventory/.coveragerc create mode 100644 inventory/inventory/.gitignore create mode 100644 inventory/inventory/.mailmap create mode 100644 inventory/inventory/.stestr.conf create mode 100644 inventory/inventory/CONTRIBUTING.rst create mode 100644 inventory/inventory/HACKING.rst create mode 100644 inventory/inventory/LICENSE create mode 100644 inventory/inventory/README.rst create mode 100644 inventory/inventory/babel.cfg create mode 100644 inventory/inventory/doc/requirements.txt create mode 100644 inventory/inventory/doc/source/admin/index.rst create mode 100644 inventory/inventory/doc/source/cli/index.rst create mode 100755 inventory/inventory/doc/source/conf.py create mode 100644 inventory/inventory/doc/source/configuration/index.rst create mode 100644 inventory/inventory/doc/source/contributor/contributing.rst create mode 100644 inventory/inventory/doc/source/contributor/index.rst create mode 100644 inventory/inventory/doc/source/index.rst create mode 100644 inventory/inventory/doc/source/install/common_configure.rst create mode 100644 inventory/inventory/doc/source/install/common_prerequisites.rst create mode 100644 inventory/inventory/doc/source/install/get_started.rst create mode 100644 inventory/inventory/doc/source/install/index.rst create mode 100644 inventory/inventory/doc/source/install/install-obs.rst create mode 100644 inventory/inventory/doc/source/install/install-rdo.rst create mode 100644 inventory/inventory/doc/source/install/install-ubuntu.rst create mode 100644 inventory/inventory/doc/source/install/install.rst create mode 100644 inventory/inventory/doc/source/install/next-steps.rst create mode 100644 inventory/inventory/doc/source/install/verify.rst create mode 100644 inventory/inventory/doc/source/library/index.rst create mode 100644 inventory/inventory/doc/source/readme.rst create mode 100644 inventory/inventory/doc/source/reference/index.rst create mode 100644 inventory/inventory/doc/source/user/index.rst create mode 100644 inventory/inventory/etc/inventory/delete_load.sh create mode 100644 inventory/inventory/etc/inventory/inventory-agent-pmond.conf create mode 100644 inventory/inventory/etc/inventory/inventory_goenabled_check.sh create mode 100644 inventory/inventory/etc/inventory/motd-system create mode 100644 inventory/inventory/etc/inventory/policy.json create mode 100644 inventory/inventory/inventory/__init__.py create mode 100644 inventory/inventory/inventory/agent/__init__.py create mode 100644 inventory/inventory/inventory/agent/base_manager.py create mode 100644 inventory/inventory/inventory/agent/disk.py create mode 100644 inventory/inventory/inventory/agent/lldp/__init__.py create mode 100644 inventory/inventory/inventory/agent/lldp/config.py create mode 100644 inventory/inventory/inventory/agent/lldp/drivers/__init__.py create mode 100644 inventory/inventory/inventory/agent/lldp/drivers/base.py create mode 100644 inventory/inventory/inventory/agent/lldp/drivers/lldpd/__init__.py create mode 100644 inventory/inventory/inventory/agent/lldp/drivers/lldpd/driver.py create mode 100644 inventory/inventory/inventory/agent/lldp/drivers/ovs/__init__.py create mode 100644 inventory/inventory/inventory/agent/lldp/drivers/ovs/driver.py create mode 100644 inventory/inventory/inventory/agent/lldp/manager.py create mode 100644 inventory/inventory/inventory/agent/lldp/plugin.py create mode 100644 inventory/inventory/inventory/agent/manager.py create mode 100644 inventory/inventory/inventory/agent/node.py create mode 100644 inventory/inventory/inventory/agent/pci.py create mode 100644 inventory/inventory/inventory/agent/rpcapi.py create mode 100644 inventory/inventory/inventory/api/__init__.py create mode 100644 inventory/inventory/inventory/api/app.py create mode 100644 inventory/inventory/inventory/api/config.py create mode 100644 inventory/inventory/inventory/api/controllers/__init__.py create mode 100644 inventory/inventory/inventory/api/controllers/root.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/__init__.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/base.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/collection.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/cpu.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/cpu_utils.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/ethernet_port.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/host.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/link.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/lldp_agent.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/lldp_neighbour.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/lldp_tlv.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/memory.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/node.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/pci_device.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/port.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/query.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/sensor.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/sensorgroup.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/state.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/sysinv.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/system.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/types.py create mode 100755 inventory/inventory/inventory/api/controllers/v1/utils.py create mode 100644 inventory/inventory/inventory/api/controllers/v1/versions.py create mode 100644 inventory/inventory/inventory/api/hooks.py create mode 100644 inventory/inventory/inventory/api/middleware/__init__.py create mode 100644 inventory/inventory/inventory/api/middleware/auth_token.py create mode 100644 inventory/inventory/inventory/api/middleware/parsable_error.py create mode 100644 inventory/inventory/inventory/cmd/__init__.py create mode 100644 inventory/inventory/inventory/cmd/agent.py create mode 100644 inventory/inventory/inventory/cmd/api.py create mode 100644 inventory/inventory/inventory/cmd/conductor.py create mode 100644 inventory/inventory/inventory/cmd/dbsync.py create mode 100755 inventory/inventory/inventory/cmd/dnsmasq_lease_update.py create mode 100644 inventory/inventory/inventory/common/__init__.py create mode 100644 inventory/inventory/inventory/common/base.py create mode 100644 inventory/inventory/inventory/common/ceph.py create mode 100644 inventory/inventory/inventory/common/config.py create mode 100644 inventory/inventory/inventory/common/constants.py create mode 100644 inventory/inventory/inventory/common/context.py create mode 100644 inventory/inventory/inventory/common/exception.py create mode 100644 inventory/inventory/inventory/common/fm.py create mode 100755 inventory/inventory/inventory/common/health.py create mode 100755 inventory/inventory/inventory/common/hwmon_api.py create mode 100644 inventory/inventory/inventory/common/i18n.py create mode 100644 inventory/inventory/inventory/common/k_host.py create mode 100644 inventory/inventory/inventory/common/k_host_agg.py create mode 100644 inventory/inventory/inventory/common/k_lldp.py create mode 100644 inventory/inventory/inventory/common/k_pci.py create mode 100644 inventory/inventory/inventory/common/keystone.py create mode 100644 inventory/inventory/inventory/common/mtce_api.py create mode 100644 inventory/inventory/inventory/common/patch_api.py create mode 100644 inventory/inventory/inventory/common/policy.py create mode 100644 inventory/inventory/inventory/common/rest_api.py create mode 100644 inventory/inventory/inventory/common/rpc.py create mode 100644 inventory/inventory/inventory/common/rpc_service.py create mode 100644 inventory/inventory/inventory/common/service.py create mode 100644 inventory/inventory/inventory/common/sm_api.py create mode 100644 inventory/inventory/inventory/common/storage_backend_conf.py create mode 100644 inventory/inventory/inventory/common/utils.py create mode 100644 inventory/inventory/inventory/common/vim_api.py create mode 100644 inventory/inventory/inventory/conductor/__init__.py create mode 100644 inventory/inventory/inventory/conductor/base_manager.py create mode 100644 inventory/inventory/inventory/conductor/manager.py create mode 100644 inventory/inventory/inventory/conductor/openstack.py create mode 100644 inventory/inventory/inventory/conductor/rpcapi.py create mode 100644 inventory/inventory/inventory/conf/__init__.py create mode 100644 inventory/inventory/inventory/conf/database.py create mode 100644 inventory/inventory/inventory/conf/default.py create mode 100644 inventory/inventory/inventory/conf/opts.py create mode 100644 inventory/inventory/inventory/config-generator.conf create mode 100644 inventory/inventory/inventory/db/__init__.py create mode 100644 inventory/inventory/inventory/db/api.py create mode 100644 inventory/inventory/inventory/db/migration.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/__init__.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/api.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migrate_repo/README create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migrate_repo/__init__.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migrate_repo/manage.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migrate_repo/migrate.cfg create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/001_init.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/__init__.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/migration.py create mode 100644 inventory/inventory/inventory/db/sqlalchemy/models.py create mode 100644 inventory/inventory/inventory/db/utils.py create mode 100644 inventory/inventory/inventory/objects/__init__.py create mode 100644 inventory/inventory/inventory/objects/base.py create mode 100644 inventory/inventory/inventory/objects/cpu.py create mode 100644 inventory/inventory/inventory/objects/fields.py create mode 100644 inventory/inventory/inventory/objects/host.py create mode 100644 inventory/inventory/inventory/objects/lldp_agent.py create mode 100644 inventory/inventory/inventory/objects/lldp_neighbour.py create mode 100644 inventory/inventory/inventory/objects/lldp_tlv.py create mode 100644 inventory/inventory/inventory/objects/memory.py create mode 100644 inventory/inventory/inventory/objects/node.py create mode 100644 inventory/inventory/inventory/objects/pci_device.py create mode 100644 inventory/inventory/inventory/objects/port.py create mode 100644 inventory/inventory/inventory/objects/port_ethernet.py create mode 100644 inventory/inventory/inventory/objects/sensor.py create mode 100644 inventory/inventory/inventory/objects/sensor_analog.py create mode 100644 inventory/inventory/inventory/objects/sensor_discrete.py create mode 100644 inventory/inventory/inventory/objects/sensorgroup.py create mode 100644 inventory/inventory/inventory/objects/sensorgroup_analog.py create mode 100644 inventory/inventory/inventory/objects/sensorgroup_discrete.py create mode 100644 inventory/inventory/inventory/objects/system.py create mode 100644 inventory/inventory/inventory/systemconfig/__init__.py create mode 100644 inventory/inventory/inventory/systemconfig/config.py create mode 100644 inventory/inventory/inventory/systemconfig/drivers/__init__.py create mode 100644 inventory/inventory/inventory/systemconfig/drivers/base.py create mode 100644 inventory/inventory/inventory/systemconfig/drivers/sysinv/__init__.py create mode 100644 inventory/inventory/inventory/systemconfig/drivers/sysinv/driver.py create mode 100644 inventory/inventory/inventory/systemconfig/manager.py create mode 100644 inventory/inventory/inventory/systemconfig/plugin.py create mode 100644 inventory/inventory/inventory/tests/__init__.py create mode 100644 inventory/inventory/inventory/tests/base.py create mode 100644 inventory/inventory/inventory/tests/test_inventory.py create mode 100644 inventory/inventory/inventory/version.py create mode 100644 inventory/inventory/releasenotes/notes/.placeholder create mode 100644 inventory/inventory/releasenotes/source/_static/.placeholder create mode 100644 inventory/inventory/releasenotes/source/_templates/.placeholder create mode 100644 inventory/inventory/releasenotes/source/conf.py create mode 100644 inventory/inventory/releasenotes/source/index.rst create mode 100644 inventory/inventory/releasenotes/source/unreleased.rst create mode 100644 inventory/inventory/requirements.txt create mode 100755 inventory/inventory/scripts/inventory-agent-initd create mode 100644 inventory/inventory/scripts/inventory-agent.service create mode 100755 inventory/inventory/scripts/inventory-api create mode 100644 inventory/inventory/scripts/inventory-api.service create mode 100755 inventory/inventory/scripts/inventory-conductor create mode 100644 inventory/inventory/scripts/inventory-conductor.service create mode 100644 inventory/inventory/setup.cfg create mode 100644 inventory/inventory/setup.py create mode 100644 inventory/inventory/test-requirements.txt create mode 100644 inventory/inventory/tox.ini create mode 100644 python-inventoryclient/PKG-INFO create mode 100644 python-inventoryclient/centos/build_srpm.data create mode 100644 python-inventoryclient/centos/python-inventoryclient.spec create mode 100644 python-inventoryclient/inventoryclient/.gitignore create mode 100644 python-inventoryclient/inventoryclient/.testr.conf create mode 100644 python-inventoryclient/inventoryclient/LICENSE create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/__init__.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/client.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/__init__.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/base.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/cli_no_wrap.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/exceptions.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/http.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/i18n.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/options.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/utils.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/common/wrapping_formatters.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/exc.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/tests/__init__.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/tests/test_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/tests/test_utils.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/tests/utils.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/__init__.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/client.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/cpu.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/cpu_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/host.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/host_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/memory.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/memory_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/node.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/node_shell.py create mode 100755 python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/port.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/port_shell.py create mode 100644 python-inventoryclient/inventoryclient/inventoryclient/v1/shell.py create mode 100755 python-inventoryclient/inventoryclient/pylint.rc create mode 100644 python-inventoryclient/inventoryclient/setup.cfg create mode 100644 python-inventoryclient/inventoryclient/setup.py create mode 100644 python-inventoryclient/inventoryclient/test-requirements.txt create mode 100644 python-inventoryclient/inventoryclient/tools/inventory.bash_completion create mode 100644 python-inventoryclient/inventoryclient/tox.ini diff --git a/centos_iso_image.inc b/centos_iso_image.inc index d6fcc9c6..d607d55b 100644 --- a/centos_iso_image.inc +++ b/centos_iso_image.inc @@ -23,3 +23,7 @@ pxe-network-installer # platform-kickstarts platform-kickstarts + +# inventory +inventory +python-inventoryclient diff --git a/centos_pkg_dirs b/centos_pkg_dirs index 46ade9ae..a513b813 100644 --- a/centos_pkg_dirs +++ b/centos_pkg_dirs @@ -5,3 +5,5 @@ mtce-control mtce-storage installer/pxe-network-installer kickstart +inventory +python-inventoryclient diff --git a/inventory/PKG-INFO b/inventory/PKG-INFO new file mode 100644 index 00000000..242f667a --- /dev/null +++ b/inventory/PKG-INFO @@ -0,0 +1,13 @@ +Metadata-Version: 1.1 +Name: inventory +Version: 1.0 +Summary: Inventory +Home-page: https://wiki.openstack.org/wiki/StarlingX +Author: StarlingX +Author-email: starlingx-discuss@lists.starlingx.io +License: Apache-2.0 + +Description: Inventory Service + + +Platform: UNKNOWN diff --git a/inventory/centos/build_srpm.data b/inventory/centos/build_srpm.data new file mode 100644 index 00000000..611b4e17 --- /dev/null +++ b/inventory/centos/build_srpm.data @@ -0,0 +1,2 @@ +SRC_DIR="inventory" +TIS_PATCH_VER=1 diff --git a/inventory/centos/inventory.spec b/inventory/centos/inventory.spec new file mode 100644 index 00000000..5ae6b632 --- /dev/null +++ b/inventory/centos/inventory.spec @@ -0,0 +1,195 @@ +Summary: Inventory +Name: inventory +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River +URL: unknown +Source0: %{name}-%{version}.tar.gz + +BuildRequires: cgts-client +BuildRequires: python-setuptools +BuildRequires: python-jsonpatch +BuildRequires: python-keystoneauth1 +BuildRequires: python-keystonemiddleware +BuildRequires: python-mock +BuildRequires: python-neutronclient +BuildRequires: python-oslo-concurrency +BuildRequires: python-oslo-config +BuildRequires: python-oslo-context +BuildRequires: python-oslo-db +BuildRequires: python-oslo-db-tests +BuildRequires: python-oslo-i18n +BuildRequires: python-oslo-log +BuildRequires: python-oslo-messaging +BuildRequires: python-oslo-middleware +BuildRequires: python-oslo-policy +BuildRequires: python-oslo-rootwrap +BuildRequires: python-oslo-serialization +BuildRequires: python-oslo-service +BuildRequires: python-oslo-utils +BuildRequires: python-oslo-versionedobjects +BuildRequires: python-oslotest +BuildRequires: python-osprofiler +BuildRequires: python-os-testr +BuildRequires: python-pbr +BuildRequires: python-pecan +BuildRequires: python-psutil +BuildRequires: python-requests +BuildRequires: python-retrying +BuildRequires: python-six +BuildRequires: python-sqlalchemy +BuildRequires: python-stevedore +BuildRequires: python-webob +BuildRequires: python-wsme +BuildRequires: systemd +BuildRequires: systemd-devel + + +Requires: python-pyudev +Requires: pyparted +Requires: python-ipaddr +Requires: python-paste +Requires: python-eventlet +Requires: python-futurist >= 0.11.0 +Requires: python-jsonpatch +Requires: python-keystoneauth1 >= 3.1.0 +Requires: python-keystonemiddleware >= 4.12.0 +Requires: python-neutronclient >= 6.3.0 +Requires: python-oslo-concurrency >= 3.8.0 +Requires: python-oslo-config >= 2:4.0.0 +Requires: python-oslo-context >= 2.14.0 +Requires: python-oslo-db >= 4.24.0 +Requires: python-oslo-i18n >= 2.1.0 +Requires: python-oslo-log >= 3.22.0 +Requires: python-oslo-messaging >= 5.24.2 +Requires: python-oslo-middleware >= 3.27.0 +Requires: python-oslo-policy >= 1.23.0 +Requires: python-oslo-rootwrap >= 5.0.0 +Requires: python-oslo-serialization >= 1.10.0 +Requires: python-oslo-service >= 1.10.0 +Requires: python-oslo-utils >= 3.20.0 +Requires: python-oslo-versionedobjects >= 1.17.0 +Requires: python-osprofiler >= 1.4.0 +Requires: python-pbr +Requires: python-pecan +Requires: python-psutil +Requires: python-requests +Requires: python-retrying +Requires: python-six +Requires: python-sqlalchemy +Requires: python-stevedore >= 1.20.0 +Requires: python-webob >= 1.7.1 +Requires: python-wsme + +%description +Inventory Service + +%define local_bindir /usr/bin/ +%define local_etc_goenabledd /etc/goenabled.d/ +%define local_etc_inventory /etc/inventory/ +%define local_etc_motdd /etc/motd.d/ +%define pythonroot /usr/lib64/python2.7/site-packages +%define ocf_resourced /usr/lib/ocf/resource.d + +%define local_etc_initd /etc/init.d/ +%define local_etc_pmond /etc/pmon.d/ + +%define debug_package %{nil} + +%prep +%setup + +# Remove bundled egg-info +rm -rf *.egg-info + +%build +echo "Start inventory build" +export PBR_VERSION=%{version} +%{__python} setup.py build +PYTHONPATH=. oslo-config-generator --config-file=inventory/config-generator.conf + +%install +echo "Start inventory install" +export PBR_VERSION=%{version} +%{__python} setup.py install --root=%{buildroot} \ + --install-lib=%{pythonroot} \ + --prefix=/usr \ + --install-data=/usr/share \ + --single-version-externally-managed + +install -d -m 755 %{buildroot}%{local_etc_goenabledd} +install -p -D -m 755 etc/inventory/inventory_goenabled_check.sh %{buildroot}%{local_etc_goenabledd}/inventory_goenabled_check.sh + +install -d -m 755 %{buildroot}%{local_etc_inventory} +install -p -D -m 755 etc/inventory/policy.json %{buildroot}%{local_etc_inventory}/policy.json + +install -d -m 755 %{buildroot}%{local_etc_motdd} +install -p -D -m 755 etc/inventory/motd-system %{buildroot}%{local_etc_motdd}/10-system-config + +install -m 755 -p -D scripts/inventory-api %{buildroot}/usr/lib/ocf/resource.d/platform/inventory-api +install -m 755 -p -D scripts/inventory-conductor %{buildroot}/usr/lib/ocf/resource.d/platform/inventory-conductor + +install -m 644 -p -D scripts/inventory-api.service %{buildroot}%{_unitdir}/inventory-api.service +install -m 644 -p -D scripts/inventory-conductor.service %{buildroot}%{_unitdir}/inventory-conductor.service + +# TODO(jkung) activate inventory-agent with puppet integration) +# install -d -m 755 %{buildroot}%{local_etc_initd} +# install -p -D -m 755 scripts/inventory-agent-initd %{buildroot}%{local_etc_initd}/inventory-agent + +# install -d -m 755 %{buildroot}%{local_etc_pmond} +# install -p -D -m 644 etc/inventory/inventory-agent-pmond.conf %{buildroot}%{local_etc_pmond}/inventory-agent-pmond.conf +# install -p -D -m 644 scripts/inventory-agent.service %{buildroot}%{_unitdir}/inventory-agent.service + +# Install sql migration +install -m 644 inventory/db/sqlalchemy/migrate_repo/migrate.cfg %{buildroot}%{pythonroot}/inventory/db/sqlalchemy/migrate_repo/migrate.cfg + +# install default config files +cd %{_builddir}/%{name}-%{version} && oslo-config-generator --config-file inventory/config-generator.conf --output-file %{_builddir}/%{name}-%{version}/inventory.conf.sample +# install -p -D -m 644 %{_builddir}/%{name}-%{version}/inventory.conf.sample %{buildroot}%{_sysconfdir}/inventory/inventory.conf + + +# TODO(jkung) activate inventory-agent +# %post +# /usr/bin/systemctl enable inventory-agent.service >/dev/null 2>&1 + + +%clean +echo "CLEAN CALLED" +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%doc LICENSE + +%{local_bindir}/* + +%{pythonroot}/%{name} + +%{pythonroot}/%{name}-%{version}*.egg-info + +%{local_etc_goenabledd}/* + +%{local_etc_inventory}/* + +%{local_etc_motdd}/* + +# SM OCF Start/Stop/Monitor Scripts +%{ocf_resourced}/platform/inventory-api +%{ocf_resourced}/platform/inventory-conductor + +# systemctl service files +%{_unitdir}/inventory-api.service +%{_unitdir}/inventory-conductor.service + +# %{_bindir}/inventory-agent +%{_bindir}/inventory-api +%{_bindir}/inventory-conductor +%{_bindir}/inventory-dbsync +%{_bindir}/inventory-dnsmasq-lease-update + +# inventory-agent files +# %{local_etc_initd}/inventory-agent +# %{local_etc_pmond}/inventory-agent-pmond.conf +# %{_unitdir}/inventory-agent.service diff --git a/inventory/inventory/.coveragerc b/inventory/inventory/.coveragerc new file mode 100644 index 00000000..07c3d91c --- /dev/null +++ b/inventory/inventory/.coveragerc @@ -0,0 +1,6 @@ +[run] +branch = True +source = inventory + +[report] +ignore_errors = True diff --git a/inventory/inventory/.gitignore b/inventory/inventory/.gitignore new file mode 100644 index 00000000..59b35f50 --- /dev/null +++ b/inventory/inventory/.gitignore @@ -0,0 +1,59 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg* +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +cover/ +.coverage* +!.coveragerc +.tox +nosetests.xml +.testrepository +.stestr +.venv + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.*sw? + +# Files created by releasenotes build +releasenotes/build diff --git a/inventory/inventory/.mailmap b/inventory/inventory/.mailmap new file mode 100644 index 00000000..516ae6fe --- /dev/null +++ b/inventory/inventory/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# +# diff --git a/inventory/inventory/.stestr.conf b/inventory/inventory/.stestr.conf new file mode 100644 index 00000000..cf348572 --- /dev/null +++ b/inventory/inventory/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=./inventory/tests +top_dir=./ diff --git a/inventory/inventory/CONTRIBUTING.rst b/inventory/inventory/CONTRIBUTING.rst new file mode 100644 index 00000000..14e6c9a9 --- /dev/null +++ b/inventory/inventory/CONTRIBUTING.rst @@ -0,0 +1,19 @@ +If you would like to contribute to the development of StarlingX, you must +follow the steps in this page: + + https://wiki.openstack.org/wiki/StarlingX/Contribution_Guidelines + +If you already have a good understanding of how the system works and your +StarlingX accounts are set up, you can skip to the development workflow +section of this documentation to learn how changes to StarlingX should be +submitted for review via the Gerrit tool: + + http://docs.openstack.org/infra/manual/developers.html#development-workflow + +Pull requests submitted through GitHub will be ignored. + +Bugs should be filed on Launchpad: + https://bugs.launchpad.net/starlingx + +Storyboard: + https://storyboard.openstack.org/#!/story/2002950 diff --git a/inventory/inventory/HACKING.rst b/inventory/inventory/HACKING.rst new file mode 100644 index 00000000..6f58ce44 --- /dev/null +++ b/inventory/inventory/HACKING.rst @@ -0,0 +1,4 @@ +inventory Style Commandments +============================ + +Read the OpenStack Style Commandments https://docs.openstack.org/hacking/latest/ diff --git a/inventory/inventory/LICENSE b/inventory/inventory/LICENSE new file mode 100644 index 00000000..68c771a0 --- /dev/null +++ b/inventory/inventory/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/inventory/inventory/README.rst b/inventory/inventory/README.rst new file mode 100644 index 00000000..9d282cc5 --- /dev/null +++ b/inventory/inventory/README.rst @@ -0,0 +1,3 @@ +Placeholder to allow setup.py to work. +Removing this requires modifying the +setup.py manifest. diff --git a/inventory/inventory/babel.cfg b/inventory/inventory/babel.cfg new file mode 100644 index 00000000..15cd6cb7 --- /dev/null +++ b/inventory/inventory/babel.cfg @@ -0,0 +1,2 @@ +[python: **.py] + diff --git a/inventory/inventory/doc/requirements.txt b/inventory/inventory/doc/requirements.txt new file mode 100644 index 00000000..afd3597a --- /dev/null +++ b/inventory/inventory/doc/requirements.txt @@ -0,0 +1,4 @@ +sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +openstackdocstheme>=1.18.1 # Apache-2.0 +# releasenotes +reno>=2.5.0 # Apache-2.0 diff --git a/inventory/inventory/doc/source/admin/index.rst b/inventory/inventory/doc/source/admin/index.rst new file mode 100644 index 00000000..1c8c7ae4 --- /dev/null +++ b/inventory/inventory/doc/source/admin/index.rst @@ -0,0 +1,5 @@ +==================== +Administrators guide +==================== + +Administrators guide of inventory. diff --git a/inventory/inventory/doc/source/cli/index.rst b/inventory/inventory/doc/source/cli/index.rst new file mode 100644 index 00000000..df79ad2e --- /dev/null +++ b/inventory/inventory/doc/source/cli/index.rst @@ -0,0 +1,5 @@ +================================ +Command line interface reference +================================ + +CLI reference of inventory. diff --git a/inventory/inventory/doc/source/conf.py b/inventory/inventory/doc/source/conf.py new file mode 100755 index 00000000..f0f029ff --- /dev/null +++ b/inventory/inventory/doc/source/conf.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +sys.path.insert(0, os.path.abspath('../..')) +# -- General configuration ---------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + 'sphinx.ext.autodoc', + 'openstackdocstheme', + #'sphinx.ext.intersphinx', +] + +# autodoc generation is a bit aggressive and a nuisance when doing heavy +# text edit cycles. +# execute "export SPHINX_DEBUG=1" in your terminal to disable + +# The suffix of source filenames. +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'inventory' +copyright = u'2018, StarlingX' + +# openstackdocstheme options +repository_name = 'stx-metal' +bug_project = '22952' +bug_tag = '' +html_last_updated_fmt = '%Y-%m-%d %H:%M' + +# If true, '()' will be appended to :func: etc. cross-reference text. +add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = True + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# -- Options for HTML output -------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +# html_theme_path = ["."] +# html_theme = '_theme' +# html_static_path = ['static'] +html_theme = 'starlingxdocs' + +# Output file base name for HTML help builder. +htmlhelp_basename = '%sdoc' % project + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto/manual]). +latex_documents = [ + ('index', + '%s.tex' % project, + u'%s Documentation' % project, + u'OpenStack Developers', 'manual'), +] + +# Example configuration for intersphinx: refer to the Python standard library. +#intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/inventory/inventory/doc/source/configuration/index.rst b/inventory/inventory/doc/source/configuration/index.rst new file mode 100644 index 00000000..a16a8a47 --- /dev/null +++ b/inventory/inventory/doc/source/configuration/index.rst @@ -0,0 +1,5 @@ +============= +Configuration +============= + +Configuration of inventory. diff --git a/inventory/inventory/doc/source/contributor/contributing.rst b/inventory/inventory/doc/source/contributor/contributing.rst new file mode 100644 index 00000000..2aa07077 --- /dev/null +++ b/inventory/inventory/doc/source/contributor/contributing.rst @@ -0,0 +1,4 @@ +============ +Contributing +============ +.. include:: ../../../CONTRIBUTING.rst diff --git a/inventory/inventory/doc/source/contributor/index.rst b/inventory/inventory/doc/source/contributor/index.rst new file mode 100644 index 00000000..38cef815 --- /dev/null +++ b/inventory/inventory/doc/source/contributor/index.rst @@ -0,0 +1,9 @@ +========================= +Contributor Documentation +========================= + +.. toctree:: + :maxdepth: 2 + + contributing + diff --git a/inventory/inventory/doc/source/index.rst b/inventory/inventory/doc/source/index.rst new file mode 100644 index 00000000..2b5e6fee --- /dev/null +++ b/inventory/inventory/doc/source/index.rst @@ -0,0 +1,30 @@ +.. inventory documentation master file, created by + sphinx-quickstart on Tue Jul 9 22:26:36 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +========================================= +Welcome to the documentation of inventory +========================================= + +Contents: + +.. toctree:: + :maxdepth: 2 + + readme + install/index + library/index + contributor/index + configuration/index + cli/index + user/index + admin/index + reference/index + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/inventory/inventory/doc/source/install/common_configure.rst b/inventory/inventory/doc/source/install/common_configure.rst new file mode 100644 index 00000000..9e4b639e --- /dev/null +++ b/inventory/inventory/doc/source/install/common_configure.rst @@ -0,0 +1,10 @@ +2. Edit the ``/etc/inventory/inventory.conf`` file and complete the following + actions: + + * In the ``[database]`` section, configure database access: + + .. code-block:: ini + + [database] + ... + connection = mysql+pymysql://inventory:INVENTORY_DBPASS@controller/inventory diff --git a/inventory/inventory/doc/source/install/common_prerequisites.rst b/inventory/inventory/doc/source/install/common_prerequisites.rst new file mode 100644 index 00000000..2496034b --- /dev/null +++ b/inventory/inventory/doc/source/install/common_prerequisites.rst @@ -0,0 +1,75 @@ +Prerequisites +------------- + +Before you install and configure the inventory service, +you must create a database, service credentials, and API endpoints. + +#. To create the database, complete these steps: + + * Use the database access client to connect to the database + server as the ``root`` user: + + .. code-block:: console + + $ mysql -u root -p + + * Create the ``inventory`` database: + + .. code-block:: none + + CREATE DATABASE inventory; + + * Grant proper access to the ``inventory`` database: + + .. code-block:: none + + GRANT ALL PRIVILEGES ON inventory.* TO 'inventory'@'localhost' \ + IDENTIFIED BY 'INVENTORY_DBPASS'; + GRANT ALL PRIVILEGES ON inventory.* TO 'inventory'@'%' \ + IDENTIFIED BY 'INVENTORY_DBPASS'; + + Replace ``INVENTORY_DBPASS`` with a suitable password. + + * Exit the database access client. + + .. code-block:: none + + exit; + +#. Source the ``admin`` credentials to gain access to + admin-only CLI commands: + + .. code-block:: console + + $ . admin-openrc + +#. To create the service credentials, complete these steps: + + * Create the ``inventory`` user: + + .. code-block:: console + + $ openstack user create --domain default --password-prompt inventory + + * Add the ``admin`` role to the ``inventory`` user: + + .. code-block:: console + + $ openstack role add --project service --user inventory admin + + * Create the inventory service entities: + + .. code-block:: console + + $ openstack service create --name inventory --description "inventory" inventory + +#. Create the inventory service API endpoints: + + .. code-block:: console + + $ openstack endpoint create --region RegionOne \ + inventory public http://controller:XXXX/vY/%\(tenant_id\)s + $ openstack endpoint create --region RegionOne \ + inventory internal http://controller:XXXX/vY/%\(tenant_id\)s + $ openstack endpoint create --region RegionOne \ + inventory admin http://controller:XXXX/vY/%\(tenant_id\)s diff --git a/inventory/inventory/doc/source/install/get_started.rst b/inventory/inventory/doc/source/install/get_started.rst new file mode 100644 index 00000000..dbba1d78 --- /dev/null +++ b/inventory/inventory/doc/source/install/get_started.rst @@ -0,0 +1,9 @@ +========================== +inventory service overview +========================== +The inventory service provides host inventory of resources on the host. + +The inventory service consists of the following components: + +``inventory-api`` service + Accepts and responds to end user API calls... diff --git a/inventory/inventory/doc/source/install/index.rst b/inventory/inventory/doc/source/install/index.rst new file mode 100644 index 00000000..eefb242f --- /dev/null +++ b/inventory/inventory/doc/source/install/index.rst @@ -0,0 +1,17 @@ +==================================== +inventory service installation guide +==================================== + +.. toctree:: + :maxdepth: 2 + + get_started.rst + install.rst + verify.rst + next-steps.rst + +The inventory service (inventory) provides... + +This chapter assumes a working setup of StarlingX following the +`StarlingX Installation Guide +`_. diff --git a/inventory/inventory/doc/source/install/install-obs.rst b/inventory/inventory/doc/source/install/install-obs.rst new file mode 100644 index 00000000..c7c97a19 --- /dev/null +++ b/inventory/inventory/doc/source/install/install-obs.rst @@ -0,0 +1,34 @@ +.. _install-obs: + + +Install and configure for openSUSE and SUSE Linux Enterprise +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the inventory service +for openSUSE Leap 42.1 and SUSE Linux Enterprise Server 12 SP1. + +.. include:: common_prerequisites.rst + +Install and configure components +-------------------------------- + +#. Install the packages: + + .. code-block:: console + + # zypper --quiet --non-interactive install + +.. include:: common_configure.rst + + +Finalize installation +--------------------- + +Start the inventory services and configure them to start when +the system boots: + +.. code-block:: console + + # systemctl enable openstack-inventory-api.service + + # systemctl start openstack-inventory-api.service diff --git a/inventory/inventory/doc/source/install/install-rdo.rst b/inventory/inventory/doc/source/install/install-rdo.rst new file mode 100644 index 00000000..30bc134b --- /dev/null +++ b/inventory/inventory/doc/source/install/install-rdo.rst @@ -0,0 +1,33 @@ +.. _install-rdo: + +Install and configure for Red Hat Enterprise Linux and CentOS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +This section describes how to install and configure the inventory service +for Red Hat Enterprise Linux 7 and CentOS 7. + +.. include:: common_prerequisites.rst + +Install and configure components +-------------------------------- + +#. Install the packages: + + .. code-block:: console + + # yum install + +.. include:: common_configure.rst + +Finalize installation +--------------------- + +Start the inventory services and configure them to start when +the system boots: + +.. code-block:: console + + # systemctl enable openstack-inventory-api.service + + # systemctl start openstack-inventory-api.service diff --git a/inventory/inventory/doc/source/install/install-ubuntu.rst b/inventory/inventory/doc/source/install/install-ubuntu.rst new file mode 100644 index 00000000..0b36a42b --- /dev/null +++ b/inventory/inventory/doc/source/install/install-ubuntu.rst @@ -0,0 +1,31 @@ +.. _install-ubuntu: + +Install and configure for Ubuntu +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the inventory +service for Ubuntu 14.04 (LTS). + +.. include:: common_prerequisites.rst + +Install and configure components +-------------------------------- + +#. Install the packages: + + .. code-block:: console + + # apt-get update + + # apt-get install + +.. include:: common_configure.rst + +Finalize installation +--------------------- + +Restart the inventory services: + +.. code-block:: console + + # service openstack-inventory-api restart diff --git a/inventory/inventory/doc/source/install/install.rst b/inventory/inventory/doc/source/install/install.rst new file mode 100644 index 00000000..2d7c01d5 --- /dev/null +++ b/inventory/inventory/doc/source/install/install.rst @@ -0,0 +1,20 @@ +.. _install: + +Install and configure +~~~~~~~~~~~~~~~~~~~~~ + +This section describes how to install and configure the +inventory service, code-named inventory, on the controller node. + +This section assumes that you already have a working OpenStack +environment with at least the following components installed: +.. (add the appropriate services here and further notes) + +Note that installation and configuration vary by distribution. + +.. toctree:: + :maxdepth: 2 + + install-obs.rst + install-rdo.rst + install-ubuntu.rst diff --git a/inventory/inventory/doc/source/install/next-steps.rst b/inventory/inventory/doc/source/install/next-steps.rst new file mode 100644 index 00000000..435afbe0 --- /dev/null +++ b/inventory/inventory/doc/source/install/next-steps.rst @@ -0,0 +1,9 @@ +.. _next-steps: + +Next steps +~~~~~~~~~~ + +Your OpenStack environment now includes the inventory service. + +To add additional services, see +https://docs.openstack.org/project-install-guide/ocata/. diff --git a/inventory/inventory/doc/source/install/verify.rst b/inventory/inventory/doc/source/install/verify.rst new file mode 100644 index 00000000..91b2f528 --- /dev/null +++ b/inventory/inventory/doc/source/install/verify.rst @@ -0,0 +1,24 @@ +.. _verify: + +Verify operation +~~~~~~~~~~~~~~~~ + +Verify operation of the inventory service. + +.. note:: + + Perform these commands on the controller node. + +#. Source the ``admin`` project credentials to gain access to + admin-only CLI commands: + + .. code-block:: console + + $ . admin-openrc + +#. List service components to verify successful launch and registration + of each process: + + .. code-block:: console + + $ openstack inventory service list diff --git a/inventory/inventory/doc/source/library/index.rst b/inventory/inventory/doc/source/library/index.rst new file mode 100644 index 00000000..ad7fb71b --- /dev/null +++ b/inventory/inventory/doc/source/library/index.rst @@ -0,0 +1,7 @@ +===== +Usage +===== + +To use inventory in a project: + + import inventory diff --git a/inventory/inventory/doc/source/readme.rst b/inventory/inventory/doc/source/readme.rst new file mode 100644 index 00000000..a6210d3d --- /dev/null +++ b/inventory/inventory/doc/source/readme.rst @@ -0,0 +1 @@ +.. include:: ../../README.rst diff --git a/inventory/inventory/doc/source/reference/index.rst b/inventory/inventory/doc/source/reference/index.rst new file mode 100644 index 00000000..96df64c6 --- /dev/null +++ b/inventory/inventory/doc/source/reference/index.rst @@ -0,0 +1,5 @@ +========== +References +========== + +References of inventory. diff --git a/inventory/inventory/doc/source/user/index.rst b/inventory/inventory/doc/source/user/index.rst new file mode 100644 index 00000000..3dc33bf0 --- /dev/null +++ b/inventory/inventory/doc/source/user/index.rst @@ -0,0 +1,5 @@ +=========== +Users guide +=========== + +Users guide of inventory. diff --git a/inventory/inventory/etc/inventory/delete_load.sh b/inventory/inventory/etc/inventory/delete_load.sh new file mode 100644 index 00000000..a0d0155d --- /dev/null +++ b/inventory/inventory/etc/inventory/delete_load.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) 2015-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This script removes a load from a controller. +# The load version is passed in as the first variable. + +: ${1?"Usage $0 VERSION"} +VERSION=$1 + +FEED_DIR=/www/pages/feed/rel-$VERSION + +rm -f /pxeboot/pxelinux.cfg.files/*-$VERSION +rm -rf /pxeboot/rel-$VERSION + +rm -f /usr/sbin/pxeboot-update-$VERSION.sh + +rm -rf $FEED_DIR diff --git a/inventory/inventory/etc/inventory/inventory-agent-pmond.conf b/inventory/inventory/etc/inventory/inventory-agent-pmond.conf new file mode 100644 index 00000000..fd20eda5 --- /dev/null +++ b/inventory/inventory/etc/inventory/inventory-agent-pmond.conf @@ -0,0 +1,9 @@ +[process] +process = inventory-agent +pidfile = /var/run/inventory-agent.pid +script = /etc/init.d/inventory-agent +style = lsb ; ocf or lsb +severity = major ; minor, major, critical +restarts = 3 ; restarts before error assertion +interval = 5 ; number of seconds to wait between restarts +debounce = 20 ; number of seconds to wait before degrade clear diff --git a/inventory/inventory/etc/inventory/inventory_goenabled_check.sh b/inventory/inventory/etc/inventory/inventory_goenabled_check.sh new file mode 100644 index 00000000..2e57a594 --- /dev/null +++ b/inventory/inventory/etc/inventory/inventory_goenabled_check.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Inventory "goenabled" check. +# Wait for inventory information to be posted prior to allowing goenabled. + +NAME=$(basename $0) +INVENTORY_READY_FLAG=/var/run/.inventory_ready + +# logfile=/var/log/platform.log + +function LOG { + logger "$NAME: $*" + # echo "`date "+%FT%T"`: $NAME: $*" >> $logfile +} + +count=0 +while [ $count -le 45 ]; do + if [ -f $INVENTORY_READY_FLAG ]; then + LOG "Inventory is ready. Passing goenabled check." + echo "Inventory goenabled iterations PASS $count" + LOG "Inventory goenabled iterations PASS $count" + exit 0 + fi + sleep 1 + count=$(($count+1)) +done + +echo "Inventory goenabled iterations FAIL $count" + +LOG "Inventory is not ready. Continue." +exit 0 diff --git a/inventory/inventory/etc/inventory/motd-system b/inventory/inventory/etc/inventory/motd-system new file mode 100644 index 00000000..7ccde4e1 --- /dev/null +++ b/inventory/inventory/etc/inventory/motd-system @@ -0,0 +1,10 @@ +#!/bin/bash +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# update inventory MOTD if motd.system content present + +[ -f /etc/inventory/motd.system ] && cat /etc/inventory/motd.system || true diff --git a/inventory/inventory/etc/inventory/policy.json b/inventory/inventory/etc/inventory/policy.json new file mode 100644 index 00000000..94ac3a5b --- /dev/null +++ b/inventory/inventory/etc/inventory/policy.json @@ -0,0 +1,5 @@ +{ + "admin": "role:admin or role:administrator", + "admin_api": "is_admin:True", + "default": "rule:admin_api" +} diff --git a/inventory/inventory/inventory/__init__.py b/inventory/inventory/inventory/__init__.py new file mode 100644 index 00000000..2a16b190 --- /dev/null +++ b/inventory/inventory/inventory/__init__.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import pbr.version + + +__version__ = pbr.version.VersionInfo( + 'inventory').version_string() diff --git a/inventory/inventory/inventory/agent/__init__.py b/inventory/inventory/inventory/agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/agent/base_manager.py b/inventory/inventory/inventory/agent/base_manager.py new file mode 100644 index 00000000..88ac8eb8 --- /dev/null +++ b/inventory/inventory/inventory/agent/base_manager.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +"""Base agent manager functionality.""" + +import futurist +from futurist import periodics +from futurist import rejection +import inspect +from inventory.common import exception +from inventory.common.i18n import _ +from oslo_config import cfg +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class BaseAgentManager(object): + + def __init__(self, host, topic): + super(BaseAgentManager, self).__init__() + if not host: + host = cfg.CONF.host + self.host = host + self.topic = topic + self._started = False + + def init_host(self, admin_context=None): + """Initialize the agent host. + + :param admin_context: the admin context to pass to periodic tasks. + :raises: RuntimeError when agent is already running. + """ + if self._started: + raise RuntimeError(_('Attempt to start an already running ' + 'agent manager')) + + rejection_func = rejection.reject_when_reached(64) + # CONF.conductor.workers_pool_size) + self._executor = futurist.GreenThreadPoolExecutor( + 64, check_and_reject=rejection_func) + # JK max_workers=CONF.conductor.workers_pool_size, + """Executor for performing tasks async.""" + + # Collect driver-specific periodic tasks. + # Conductor periodic tasks accept context argument, + LOG.info('Collecting periodic tasks') + self._periodic_task_callables = [] + self._collect_periodic_tasks(self, (admin_context,)) + + self._periodic_tasks = periodics.PeriodicWorker( + self._periodic_task_callables, + executor_factory=periodics.ExistingExecutor(self._executor)) + + # Start periodic tasks + self._periodic_tasks_worker = self._executor.submit( + self._periodic_tasks.start, allow_empty=True) + self._periodic_tasks_worker.add_done_callback( + self._on_periodic_tasks_stop) + + self._started = True + + def del_host(self, deregister=True): + # Conductor deregistration fails if called on non-initialized + # agent (e.g. when rpc server is unreachable). + if not hasattr(self, 'agent'): + return + + self._periodic_tasks.stop() + self._periodic_tasks.wait() + self._executor.shutdown(wait=True) + self._started = False + + def _collect_periodic_tasks(self, obj, args): + """Collect periodic tasks from a given object. + + Populates self._periodic_task_callables with tuples + (callable, args, kwargs). + + :param obj: object containing periodic tasks as methods + :param args: tuple with arguments to pass to every task + """ + for name, member in inspect.getmembers(obj): + if periodics.is_periodic(member): + LOG.debug('Found periodic task %(owner)s.%(member)s', + {'owner': obj.__class__.__name__, + 'member': name}) + self._periodic_task_callables.append((member, args, {})) + + def _on_periodic_tasks_stop(self, fut): + try: + fut.result() + except Exception as exc: + LOG.critical('Periodic tasks worker has failed: %s', exc) + else: + LOG.info('Successfully shut down periodic tasks') + + def _spawn_worker(self, func, *args, **kwargs): + + """Create a greenthread to run func(*args, **kwargs). + + Spawns a greenthread if there are free slots in pool, otherwise raises + exception. Execution control returns immediately to the caller. + + :returns: Future object. + :raises: NoFreeConductorWorker if worker pool is currently full. + + """ + try: + return self._executor.submit(func, *args, **kwargs) + except futurist.RejectedSubmission: + raise exception.NoFreeConductorWorker() diff --git a/inventory/inventory/inventory/agent/disk.py b/inventory/inventory/inventory/agent/disk.py new file mode 100644 index 00000000..05f2a3f9 --- /dev/null +++ b/inventory/inventory/inventory/agent/disk.py @@ -0,0 +1,369 @@ +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +""" inventory idisk Utilities and helper functions.""" + +import os +import pyudev +import re +import subprocess +import sys + +from inventory.common import constants +from inventory.common import context +from inventory.common import utils +from inventory.conductor import rpcapi as conductor_rpcapi +from oslo_log import log + + +LOG = log.getLogger(__name__) + + +class DiskOperator(object): + '''Class to encapsulate Disk operations for System Inventory''' + + def __init__(self): + + self.num_cpus = 0 + self.num_nodes = 0 + self.float_cpuset = 0 + self.default_hugepage_size_kB = 0 + self.total_memory_MiB = 0 + self.free_memory_MiB = 0 + self.total_memory_nodes_MiB = [] + self.free_memory_nodes_MiB = [] + self.topology = {} + + def convert_range_string_to_list(self, s): + olist = [] + s = s.strip() + if s: + for part in s.split(','): + if '-' in part: + a, b = part.split('-') + a, b = int(a), int(b) + olist.extend(range(a, b + 1)) + else: + a = int(part) + olist.append(a) + olist.sort() + return olist + + def get_rootfs_node(self): + cmdline_file = '/proc/cmdline' + device = None + + with open(cmdline_file, 'r') as f: + for line in f: + for param in line.split(): + params = param.split("=", 1) + if params[0] == "root": + if "UUID=" in params[1]: + key, uuid = params[1].split("=") + symlink = "/dev/disk/by-uuid/%s" % uuid + device = os.path.basename(os.readlink(symlink)) + else: + device = os.path.basename(params[1]) + + if device is not None: + if constants.DEVICE_NAME_NVME in device: + re_line = re.compile(r'^(nvme[0-9]*n[0-9]*)') + else: + re_line = re.compile(r'^(\D*)') + match = re_line.search(device) + if match: + return os.path.join("/dev", match.group(1)) + + return + + @utils.skip_udev_partition_probe + def get_disk_available_mib(self, device_node): + # Check that partition table format is GPT. + # Return 0 if not. + if not utils.disk_is_gpt(device_node=device_node): + LOG.debug("Format of disk node %s is not GPT." % device_node) + return 0 + + pvs_command = '{} {}'.format('pvs | grep -w ', device_node) + pvs_process = subprocess.Popen(pvs_command, stdout=subprocess.PIPE, + shell=True) + pvs_output = pvs_process.stdout.read() + + if pvs_output: + LOG.debug("Disk %s is completely used by a PV => 0 available mib." + % device_node) + return 0 + + # Get sector size command. + sector_size_bytes_cmd = '{} {}'.format('blockdev --getss', device_node) + + # Get total free space in sectors command. + avail_space_sectors_cmd = '{} {} {}'.format( + 'sgdisk -p', device_node, "| grep \"Total free space\"") + + # Get the sector size. + sector_size_bytes_process = subprocess.Popen( + sector_size_bytes_cmd, stdout=subprocess.PIPE, shell=True) + sector_size_bytes = sector_size_bytes_process.stdout.read().rstrip() + + # Get the free space. + avail_space_sectors_process = subprocess.Popen( + avail_space_sectors_cmd, stdout=subprocess.PIPE, shell=True) + avail_space_sectors_output = avail_space_sectors_process.stdout.read() + avail_space_sectors = re.findall( + '\d+', avail_space_sectors_output)[0].rstrip() + + # Free space in MiB. + avail_space_mib = (int(sector_size_bytes) * int(avail_space_sectors) / + (1024 ** 2)) + + # Keep 2 MiB for partition table. + if avail_space_mib >= 2: + avail_space_mib = avail_space_mib - 2 + else: + avail_space_mib = 0 + + return avail_space_mib + + def disk_format_gpt(self, host_uuid, idisk_dict, is_cinder_device): + disk_node = idisk_dict.get('device_path') + + utils.disk_wipe(disk_node) + utils.execute('parted', disk_node, 'mklabel', 'gpt') + + if is_cinder_device: + LOG.debug("Removing .node_cinder_lvm_config_complete_file") + try: + os.remove(constants.NODE_CINDER_LVM_CONFIG_COMPLETE_FILE) + except OSError: + LOG.error(".node_cinder_lvm_config_complete_file not present.") + pass + + # On SX ensure wipe succeeds before DB is updated. + # Flag file is used to mark wiping in progress. + try: + os.remove(constants.DISK_WIPE_IN_PROGRESS_FLAG) + except OSError: + # it's ok if file is not present. + pass + + # We need to send the updated info about the host disks back to + # the conductor. + idisk_update = self.idisk_get() + ctxt = context.get_admin_context() + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + rpcapi.idisk_update_by_ihost(ctxt, + host_uuid, + idisk_update) + + def handle_exception(self, e): + traceback = sys.exc_info()[-1] + LOG.error("%s @ %s:%s" % ( + e, traceback.tb_frame.f_code.co_filename, traceback.tb_lineno)) + + def is_rotational(self, device_name): + """Find out if a certain disk is rotational or not. Mostly used for + determining if disk is HDD or SSD. + """ + + # Obtain the path to the rotational file for the current device. + device = device_name['DEVNAME'].split('/')[-1] + rotational_path = "/sys/block/{device}/queue/rotational"\ + .format(device=device) + + rotational = None + # Read file and remove trailing whitespaces. + if os.path.isfile(rotational_path): + with open(rotational_path, 'r') as rot_file: + rotational = rot_file.read() + rotational = rotational.rstrip() + + return rotational + + def get_device_id_wwn(self, device): + """Determine the ID and WWN of a disk from the value of the DEVLINKS + attribute. + + Note: This data is not currently being used for anything. We are + gathering this information so conductor can store for future use. + """ + # The ID and WWN default to None. + device_id = None + device_wwn = None + + # If there is no DEVLINKS attribute, return None. + if 'DEVLINKS' not in device: + return device_id, device_wwn + + # Extract the ID and the WWN. + LOG.debug("[DiskEnum] get_device_id_wwn: devlinks= %s" % + device['DEVLINKS']) + devlinks = device['DEVLINKS'].split() + for devlink in devlinks: + if "by-id" in devlink: + if "wwn" not in devlink: + device_id = devlink.split('/')[-1] + LOG.debug("[DiskEnum] by-id: %s id: %s" % (devlink, + device_id)) + else: + device_wwn = devlink.split('/')[-1] + LOG.debug("[DiskEnum] by-wwn: %s wwn: %s" % (devlink, + device_wwn)) + + return device_id, device_wwn + + def idisk_get(self): + """Enumerate disk topology based on: + + :param self + :returns list of disk and attributes + """ + idisk = [] + context = pyudev.Context() + + for device in context.list_devices(DEVTYPE='disk'): + if not utils.is_system_usable_block_device(device): + continue + + if device['MAJOR'] in constants.VALID_MAJOR_LIST: + if 'ID_PATH' in device: + device_path = "/dev/disk/by-path/" + device['ID_PATH'] + LOG.debug("[DiskEnum] device_path: %s ", device_path) + else: + # We should always have a udev supplied /dev/disk/by-path + # value as a matter of normal operation. We do not expect + # this to occur, thus the error. + # + # The kickstart files for the host install require the + # by-path value also to be present or the host install will + # fail. Since the installer and the runtime share the same + # kernel/udev we should not see this message on an + # installed system. + device_path = None + LOG.error("Device %s does not have an ID_PATH value " + "provided by udev" % device.device_node) + + size_mib = 0 + available_mib = 0 + model_num = '' + serial_id = '' + + # Can merge all try/except in one block but this allows + # at least attributes with no exception to be filled + try: + size_mib = utils.get_disk_capacity_mib(device.device_node) + except Exception as e: + self.handle_exception("Could not retrieve disk size - %s " + % e) + + try: + available_mib = self.get_disk_available_mib( + device_node=device.device_node) + except Exception as e: + self.handle_exception( + "Could not retrieve disk %s free space" % e) + + try: + # ID_MODEL received from udev is not correct for disks that + # are used entirely for LVM. LVM replaced the model ID with + # its own identifier that starts with "LVM PV".For this + # reason we will attempt to retrieve the correct model ID + # by using 2 different commands: hdparm and lsblk and + # hdparm. If one of them fails, the other one can attempt + # to retrieve the information. Else we use udev. + + # try hdparm command first + hdparm_command = 'hdparm -I %s |grep Model' % ( + device.get('DEVNAME')) + hdparm_process = subprocess.Popen( + hdparm_command, + stdout=subprocess.PIPE, + shell=True) + hdparm_output = hdparm_process.communicate()[0] + if hdparm_process.returncode == 0: + second_half = hdparm_output.split(':')[1] + model_num = second_half.strip() + else: + # try lsblk command + lsblk_command = 'lsblk -dn --output MODEL %s' % ( + device.get('DEVNAME')) + lsblk_process = subprocess.Popen( + lsblk_command, + stdout=subprocess.PIPE, + shell=True) + lsblk_output = lsblk_process.communicate()[0] + if lsblk_process.returncode == 0: + model_num = lsblk_output.strip() + else: + # both hdparm and lsblk commands failed, try udev + model_num = device.get('ID_MODEL') + if not model_num: + model_num = constants.DEVICE_MODEL_UNKNOWN + except Exception as e: + self.handle_exception("Could not retrieve disk model " + "for disk %s. Exception: %s" % + (device.get('DEVNAME'), e)) + try: + if 'ID_SCSI_SERIAL' in device: + serial_id = device['ID_SCSI_SERIAL'] + else: + serial_id = device['ID_SERIAL_SHORT'] + except Exception as e: + self.handle_exception("Could not retrieve disk " + "serial ID - %s " % e) + + capabilities = dict() + if model_num: + capabilities.update({'model_num': model_num}) + + if self.get_rootfs_node() == device.device_node: + capabilities.update({'stor_function': 'rootfs'}) + + rotational = self.is_rotational(device) + device_type = device.device_type + + rotation_rate = constants.DEVICE_TYPE_UNDETERMINED + if rotational is '1': + device_type = constants.DEVICE_TYPE_HDD + if 'ID_ATA_ROTATION_RATE_RPM' in device: + rotation_rate = device['ID_ATA_ROTATION_RATE_RPM'] + elif rotational is '0': + if constants.DEVICE_NAME_NVME in device.device_node: + device_type = constants.DEVICE_TYPE_NVME + else: + device_type = constants.DEVICE_TYPE_SSD + rotation_rate = constants.DEVICE_TYPE_NA + + # TODO(sc) else: what are other possible stor_function value? + # or do we just use pair { 'is_rootfs': True } instead? + # Obtain device ID and WWN. + device_id, device_wwn = self.get_device_id_wwn(device) + + attr = { + 'device_node': device.device_node, + 'device_num': device.device_number, + 'device_type': device_type, + 'device_path': device_path, + 'device_id': device_id, + 'device_wwn': device_wwn, + 'size_mib': size_mib, + 'available_mib': available_mib, + 'serial_id': serial_id, + 'capabilities': capabilities, + 'rpm': rotation_rate, + } + + idisk.append(attr) + + LOG.debug("idisk= %s" % idisk) + + return idisk diff --git a/inventory/inventory/inventory/agent/lldp/__init__.py b/inventory/inventory/inventory/agent/lldp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/agent/lldp/config.py b/inventory/inventory/inventory/agent/lldp/config.py new file mode 100644 index 00000000..40e63c58 --- /dev/null +++ b/inventory/inventory/inventory/agent/lldp/config.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from oslo_config import cfg +from oslo_utils._i18n import _ + +INVENTORY_LLDP_OPTS = [ + cfg.ListOpt('drivers', + default=['lldpd'], + help=_("An ordered list of inventory LLDP driver " + "entrypoints to be loaded from the " + "inventory.agent namespace.")), +] + +cfg.CONF.register_opts(INVENTORY_LLDP_OPTS, group="lldp") diff --git a/inventory/inventory/inventory/agent/lldp/drivers/__init__.py b/inventory/inventory/inventory/agent/lldp/drivers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/agent/lldp/drivers/base.py b/inventory/inventory/inventory/agent/lldp/drivers/base.py new file mode 100644 index 00000000..f69157d1 --- /dev/null +++ b/inventory/inventory/inventory/agent/lldp/drivers/base.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +import abc + +import six + + +@six.add_metaclass(abc.ABCMeta) +class InventoryLldpDriverBase(object): + """Inventory LLDP Driver Base Class.""" + + @abc.abstractmethod + def lldp_has_neighbour(self, name): + pass + + @abc.abstractmethod + def lldp_update(self): + pass + + @abc.abstractmethod + def lldp_agents_list(self): + pass + + @abc.abstractmethod + def lldp_neighbours_list(self): + pass + + @abc.abstractmethod + def lldp_agents_clear(self): + pass + + @abc.abstractmethod + def lldp_neighbours_clear(self): + pass + + @abc.abstractmethod + def lldp_update_systemname(self, systemname): + pass diff --git a/inventory/inventory/inventory/agent/lldp/drivers/lldpd/__init__.py b/inventory/inventory/inventory/agent/lldp/drivers/lldpd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/agent/lldp/drivers/lldpd/driver.py b/inventory/inventory/inventory/agent/lldp/drivers/lldpd/driver.py new file mode 100644 index 00000000..3f8d538a --- /dev/null +++ b/inventory/inventory/inventory/agent/lldp/drivers/lldpd/driver.py @@ -0,0 +1,321 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from oslo_log import log as logging + +import simplejson as json +import subprocess + +from inventory.agent.lldp.drivers import base +from inventory.agent.lldp import plugin +from inventory.common import k_lldp + +LOG = logging.getLogger(__name__) + + +class InventoryLldpdAgentDriver(base.InventoryLldpDriverBase): + + def __init__(self, **kwargs): + self.client = "" + self.agents = [] + self.neighbours = [] + self.current_neighbours = [] + self.previous_neighbours = [] + self.current_agents = [] + self.previous_agents = [] + self.agent_audit_count = 0 + self.neighbour_audit_count = 0 + + def initialize(self): + self.__init__() + + @staticmethod + def _lldpd_get_agent_status(): + json_obj = json + p = subprocess.Popen(["lldpcli", "-f", "json", "show", + "configuration"], + stdout=subprocess.PIPE) + data = json_obj.loads(p.communicate()[0]) + + configuration = data['configuration'][0] + config = configuration['config'][0] + rx_only = config['rx-only'][0] + + if rx_only.get("value") == "no": + return "rx=enabled,tx=enabled" + else: + return "rx=enabled,tx=disabled" + + @staticmethod + def _lldpd_get_attrs(iface): + name_or_uuid = None + chassis_id = None + system_name = None + system_desc = None + capability = None + management_address = None + port_desc = None + dot1_lag = None + dot1_port_vid = None + dot1_vid_digest = None + dot1_mgmt_vid = None + dot1_vlan_names = None + dot1_proto_vids = None + dot1_proto_ids = None + dot3_mac_status = None + dot3_max_frame = None + dot3_power_mdi = None + ttl = None + attrs = {} + + # Note: dot1_vid_digest, dot1_mgmt_vid are not currently supported + # by the lldpd daemon + + name_or_uuid = iface.get("name") + chassis = iface.get("chassis")[0] + port = iface.get("port")[0] + + if not chassis.get('id'): + return attrs + chassis_id = chassis['id'][0].get("value") + + if not port.get('id'): + return attrs + port_id = port["id"][0].get("value") + + if not port.get('ttl'): + return attrs + ttl = port['ttl'][0].get("value") + + if chassis.get("name"): + system_name = chassis['name'][0].get("value") + + if chassis.get("descr"): + system_desc = chassis['descr'][0].get("value") + + if chassis.get("capability"): + capability = "" + for cap in chassis["capability"]: + if cap.get("enabled"): + if capability: + capability += ", " + capability += cap.get("type").lower() + + if chassis.get("mgmt-ip"): + management_address = "" + for addr in chassis["mgmt-ip"]: + if management_address: + management_address += ", " + management_address += addr.get("value").lower() + + if port.get("descr"): + port_desc = port["descr"][0].get("value") + + if port.get("link-aggregation"): + dot1_lag_supported = port["link-aggregation"][0].get("supported") + dot1_lag_enabled = port["link-aggregation"][0].get("enabled") + dot1_lag = "capable=" + if dot1_lag_supported: + dot1_lag += "y," + else: + dot1_lag += "n," + dot1_lag += "enabled=" + if dot1_lag_enabled: + dot1_lag += "y" + else: + dot1_lag += "n" + + if port.get("auto-negotiation"): + port_auto_neg_support = port["auto-negotiation"][0].get( + "supported") + port_auto_neg_enabled = port["auto-negotiation"][0].get("enabled") + dot3_mac_status = "auto-negotiation-capable=" + if port_auto_neg_support: + dot3_mac_status += "y," + else: + dot3_mac_status += "n," + dot3_mac_status += "auto-negotiation-enabled=" + if port_auto_neg_enabled: + dot3_mac_status += "y," + else: + dot3_mac_status += "n," + advertised = "" + if port.get("auto-negotiation")[0].get("advertised"): + for adv in port["auto-negotiation"][0].get("advertised"): + if advertised: + advertised += ", " + type = adv.get("type").lower() + if adv.get("hd") and not adv.get("fd"): + type += "hd" + elif adv.get("fd"): + type += "fd" + advertised += type + dot3_mac_status += advertised + + if port.get("mfs"): + dot3_max_frame = port["mfs"][0].get("value") + + if port.get("power"): + power_mdi_support = port["power"][0].get("supported") + power_mdi_enabled = port["power"][0].get("enabled") + power_mdi_devicetype = port["power"][0].get("device-type")[0].get( + "value") + power_mdi_pairs = port["power"][0].get("pairs")[0].get("value") + power_mdi_class = port["power"][0].get("class")[0].get("value") + dot3_power_mdi = "power-mdi-supported=" + if power_mdi_support: + dot3_power_mdi += "y," + else: + dot3_power_mdi += "n," + dot3_power_mdi += "power-mdi-enabled=" + if power_mdi_enabled: + dot3_power_mdi += "y," + else: + dot3_power_mdi += "n," + if power_mdi_support and power_mdi_enabled: + dot3_power_mdi += "device-type=" + power_mdi_devicetype + dot3_power_mdi += ",pairs=" + power_mdi_pairs + dot3_power_mdi += ",class=" + power_mdi_class + + vlans = None + if iface.get("vlan"): + vlans = iface.get("vlan") + + if vlans: + dot1_vlan_names = "" + for vlan in vlans: + if vlan.get("pvid"): + dot1_port_vid = vlan.get("vlan-id") + continue + if dot1_vlan_names: + dot1_vlan_names += ", " + dot1_vlan_names += vlan.get("value") + + ppvids = None + if iface.get("ppvids"): + ppvids = iface.get("ppvid") + + if ppvids: + dot1_proto_vids = "" + for ppvid in ppvids: + if dot1_proto_vids: + dot1_proto_vids += ", " + dot1_proto_vids += ppvid.get("value") + + pids = None + if iface.get("pi"): + pids = iface.get('pi') + dot1_proto_ids = "" + for id in pids: + if dot1_proto_ids: + dot1_proto_ids += ", " + dot1_proto_ids += id.get("value") + + msap = chassis_id + "," + port_id + + attrs = {"name_or_uuid": name_or_uuid, + k_lldp.LLDP_TLV_TYPE_CHASSIS_ID: chassis_id, + k_lldp.LLDP_TLV_TYPE_PORT_ID: port_id, + k_lldp.LLDP_TLV_TYPE_TTL: ttl, + "msap": msap, + k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME: system_name, + k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC: system_desc, + k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP: capability, + k_lldp.LLDP_TLV_TYPE_MGMT_ADDR: management_address, + k_lldp.LLDP_TLV_TYPE_PORT_DESC: port_desc, + k_lldp.LLDP_TLV_TYPE_DOT1_LAG: dot1_lag, + k_lldp.LLDP_TLV_TYPE_DOT1_PORT_VID: dot1_port_vid, + k_lldp.LLDP_TLV_TYPE_DOT1_VID_DIGEST: dot1_vid_digest, + k_lldp.LLDP_TLV_TYPE_DOT1_MGMT_VID: dot1_mgmt_vid, + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES: dot1_vlan_names, + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_VIDS: dot1_proto_vids, + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_IDS: dot1_proto_ids, + k_lldp.LLDP_TLV_TYPE_DOT3_MAC_STATUS: dot3_mac_status, + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME: dot3_max_frame, + k_lldp.LLDP_TLV_TYPE_DOT3_POWER_MDI: dot3_power_mdi} + + return attrs + + def lldp_has_neighbour(self, name): + p = subprocess.check_output(["lldpcli", "-f", "keyvalue", "show", + "neighbors", "summary", "ports", name]) + return len(p) > 0 + + def lldp_update(self): + subprocess.call(['lldpcli', 'update']) + + def lldp_agents_list(self): + json_obj = json + lldp_agents = [] + + p = subprocess.Popen(["lldpcli", "-f", "json", "show", "interface", + "detail"], stdout=subprocess.PIPE) + data = json_obj.loads(p.communicate()[0]) + + lldp = data['lldp'][0] + + if not lldp.get('interface'): + return lldp_agents + + for iface in lldp['interface']: + agent_attrs = self._lldpd_get_attrs(iface) + status = self._lldpd_get_agent_status() + agent_attrs.update({"status": status}) + agent = plugin.Agent(**agent_attrs) + lldp_agents.append(agent) + + return lldp_agents + + def lldp_agents_clear(self): + self.current_agents = [] + self.previous_agents = [] + + def lldp_neighbours_list(self): + json_obj = json + lldp_neighbours = [] + p = subprocess.Popen(["lldpcli", "-f", "json", "show", "neighbor", + "detail"], stdout=subprocess.PIPE) + data = json_obj.loads(p.communicate()[0]) + + lldp = data['lldp'][0] + + if not lldp.get('interface'): + return lldp_neighbours + + for iface in lldp['interface']: + neighbour_attrs = self._lldpd_get_attrs(iface) + neighbour = plugin.Neighbour(**neighbour_attrs) + lldp_neighbours.append(neighbour) + + return lldp_neighbours + + def lldp_neighbours_clear(self): + self.current_neighbours = [] + self.previous_neighbours = [] + + def lldp_update_systemname(self, systemname): + p = subprocess.Popen(["lldpcli", "-f", "json", "show", "chassis"], + stdout=subprocess.PIPE) + data = json.loads(p.communicate()[0]) + + local_chassis = data['local-chassis'][0] + chassis = local_chassis['chassis'][0] + name = chassis.get('name', None) + if name is None or not name[0].get("value"): + return + name = name[0] + + hostname = name.get("value").partition(':')[0] + + newname = hostname + ":" + systemname + + p = subprocess.Popen(["lldpcli", "configure", "system", "hostname", + newname], stdout=subprocess.PIPE) diff --git a/inventory/inventory/inventory/agent/lldp/drivers/ovs/__init__.py b/inventory/inventory/inventory/agent/lldp/drivers/ovs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/agent/lldp/drivers/ovs/driver.py b/inventory/inventory/inventory/agent/lldp/drivers/ovs/driver.py new file mode 100644 index 00000000..70e71c27 --- /dev/null +++ b/inventory/inventory/inventory/agent/lldp/drivers/ovs/driver.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +import simplejson as json +import subprocess + +from oslo_log import log as logging + +from inventory.agent.lldp.drivers.lldpd import driver as lldpd_driver +from inventory.common import k_lldp + +LOG = logging.getLogger(__name__) + + +class InventoryOVSAgentDriver(lldpd_driver.InventoryLldpdAgentDriver): + + def run_cmd(self, cmd): + p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + p.wait() + output, error = p.communicate() + if p.returncode != 0: + LOG.error("Failed to run command %s: error: %s", cmd, error) + return None + return output + + def lldp_ovs_get_interface_port_map(self): + interface_port_map = {} + + cmd = "ovs-vsctl --timeout 10 --format json "\ + "--columns name,_uuid,interfaces list Port" + + output = self.run_cmd(cmd) + if not output: + return + + ports = json.loads(output) + ports = ports['data'] + + for port in ports: + port_uuid = port[1][1] + interfaces = port[2][1] + + if isinstance(interfaces, list): + for interface in interfaces: + interface_uuid = interface[1] + interface_port_map[interface_uuid] = port_uuid + else: + interface_uuid = interfaces + interface_port_map[interface_uuid] = port_uuid + + return interface_port_map + + def lldp_ovs_get_port_bridge_map(self): + port_bridge_map = {} + + cmd = "ovs-vsctl --timeout 10 --format json "\ + "--columns name,ports list Bridge" + output = self.run_cmd(cmd) + if not output: + return + + bridges = json.loads(output) + bridges = bridges['data'] + + for bridge in bridges: + bridge_name = bridge[0] + port_set = bridge[1][1] + for port in port_set: + value = port[1] + port_bridge_map[value] = bridge_name + + return port_bridge_map + + def lldp_ovs_lldp_flow_exists(self, brname, in_port): + + cmd = "ovs-ofctl dump-flows {} in_port={},dl_dst={},dl_type={}".format( + brname, in_port, k_lldp.LLDP_MULTICAST_ADDRESS, + k_lldp.LLDP_ETHER_TYPE) + output = self.run_cmd(cmd) + if not output: + return None + + return (output.count("\n") > 1) + + def lldp_ovs_add_flows(self, brname, in_port, out_port): + + cmd = ("ovs-ofctl add-flow {} in_port={},dl_dst={},dl_type={}," + "actions=output:{}".format( + brname, in_port, k_lldp.LLDP_MULTICAST_ADDRESS, + k_lldp.LLDP_ETHER_TYPE, out_port)) + output = self.run_cmd(cmd) + if not output: + return + + cmd = ("ovs-ofctl add-flow {} in_port={},dl_dst={},dl_type={}," + "actions=output:{}".format( + brname, out_port, k_lldp.LLDP_MULTICAST_ADDRESS, + k_lldp.LLDP_ETHER_TYPE, in_port)) + output = self.run_cmd(cmd) + if not output: + return + + def lldp_ovs_update_flows(self): + + port_bridge_map = self.lldp_ovs_get_port_bridge_map() + if not port_bridge_map: + return + + interface_port_map = self.lldp_ovs_get_interface_port_map() + if not interface_port_map: + return + + cmd = "ovs-vsctl --timeout 10 --format json "\ + "--columns name,_uuid,type,other_config list Interface" + + output = self.run_cmd(cmd) + if not output: + return + + data = json.loads(output) + data = data['data'] + + for interface in data: + name = interface[0] + uuid = interface[1][1] + type = interface[2] + other_config = interface[3] + + if type != 'internal': + continue + + config_map = other_config[1] + for config in config_map: + key = config[0] + value = config[1] + if key != 'lldp_phy_peer': + continue + + phy_peer = value + brname = port_bridge_map[interface_port_map[uuid]] + if not self.lldp_ovs_lldp_flow_exists(brname, name): + LOG.info("Adding missing LLDP flow from %s to %s", + name, phy_peer) + self.lldp_ovs_add_flows(brname, name, phy_peer) + + if not self.lldp_ovs_lldp_flow_exists(brname, value): + LOG.info("Adding missing LLDP flow from %s to %s", + phy_peer, name) + self.lldp_ovs_add_flows(brname, phy_peer, name) + + def lldp_agents_list(self): + self.lldp_ovs_update_flows() + return lldpd_driver.InventoryLldpdAgentDriver.lldp_agents_list(self) + + def lldp_neighbours_list(self): + self.lldp_ovs_update_flows() + return lldpd_driver.InventoryLldpdAgentDriver.lldp_neighbours_list( + self) diff --git a/inventory/inventory/inventory/agent/lldp/manager.py b/inventory/inventory/inventory/agent/lldp/manager.py new file mode 100644 index 00000000..1133fb54 --- /dev/null +++ b/inventory/inventory/inventory/agent/lldp/manager.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventory.common import exception +from oslo_config import cfg +from oslo_log import log +from stevedore.named import NamedExtensionManager + +LOG = log.getLogger(__name__) +cfg.CONF.import_opt('drivers', + 'inventory.agent.lldp.config', + group='lldp') + + +class InventoryLldpDriverManager(NamedExtensionManager): + """Implementation of Inventory LLDP drivers.""" + + def __init__(self, namespace='inventory.agent.lldp.drivers'): + + # Registered inventory lldp agent drivers, keyed by name. + self.drivers = {} + + # Ordered list of inventory lldp agent drivers, defining + # the order in which the drivers are called. + self.ordered_drivers = [] + + names = cfg.CONF.lldp.drivers + LOG.info("Configured inventory LLDP agent drivers: %s", names) + + super(InventoryLldpDriverManager, self).__init__( + namespace, + names, + invoke_on_load=True, + name_order=True) + + LOG.info("Loaded inventory LLDP agent drivers: %s", self.names()) + self._register_drivers() + + def _register_drivers(self): + """Register all inventory LLDP agent drivers. + + This method should only be called once in the + InventoryLldpDriverManager constructor. + """ + for ext in self: + self.drivers[ext.name] = ext + self.ordered_drivers.append(ext) + LOG.info("Registered inventory LLDP agent drivers: %s", + [driver.name for driver in self.ordered_drivers]) + + def _call_drivers_and_return_array(self, method_name, attr=None, + raise_orig_exc=False): + """Helper method for calling a method across all drivers. + + :param method_name: name of the method to call + :param attr: an optional attribute to provide to the drivers + :param raise_orig_exc: whether or not to raise the original + driver exception, or use a general one + """ + ret = [] + for driver in self.ordered_drivers: + try: + method = getattr(driver.obj, method_name) + if attr: + ret = ret + method(attr) + else: + ret = ret + method() + except Exception as e: + LOG.exception(e) + LOG.error( + "Inventory LLDP agent driver '%(name)s' " + "failed in %(method)s", + {'name': driver.name, 'method': method_name} + ) + if raise_orig_exc: + raise + else: + raise exception.LLDPDriverError( + method=method_name + ) + return list(set(ret)) + + def _call_drivers(self, method_name, attr=None, raise_orig_exc=False): + """Helper method for calling a method across all drivers. + + :param method_name: name of the method to call + :param attr: an optional attribute to provide to the drivers + :param raise_orig_exc: whether or not to raise the original + driver exception, or use a general one + """ + for driver in self.ordered_drivers: + try: + method = getattr(driver.obj, method_name) + if attr: + method(attr) + else: + method() + except Exception as e: + LOG.exception(e) + LOG.error( + "Inventory LLDP agent driver '%(name)s' " + "failed in %(method)s", + {'name': driver.name, 'method': method_name} + ) + if raise_orig_exc: + raise + else: + raise exception.LLDPDriverError( + method=method_name + ) + + def lldp_has_neighbour(self, name): + try: + return self._call_drivers("lldp_has_neighbour", + attr=name, + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return [] + + def lldp_update(self): + try: + return self._call_drivers("lldp_update", + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return [] + + def lldp_agents_list(self): + try: + return self._call_drivers_and_return_array("lldp_agents_list", + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return [] + + def lldp_neighbours_list(self): + try: + return self._call_drivers_and_return_array("lldp_neighbours_list", + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return [] + + def lldp_agents_clear(self): + try: + return self._call_drivers("lldp_agents_clear", + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return + + def lldp_neighbours_clear(self): + try: + return self._call_drivers("lldp_neighbours_clear", + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return + + def lldp_update_systemname(self, systemname): + try: + return self._call_drivers("lldp_update_systemname", + attr=systemname, + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + return diff --git a/inventory/inventory/inventory/agent/lldp/plugin.py b/inventory/inventory/inventory/agent/lldp/plugin.py new file mode 100644 index 00000000..6a3fca3a --- /dev/null +++ b/inventory/inventory/inventory/agent/lldp/plugin.py @@ -0,0 +1,246 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from oslo_log import log +from oslo_utils import excutils + +from inventory.agent.lldp import manager +from inventory.common import exception +from inventory.common import k_lldp +from inventory.common.utils import compare as cmp + +LOG = log.getLogger(__name__) + + +class Key(object): + def __init__(self, chassisid, portid, portname): + self.chassisid = chassisid + self.portid = portid + self.portname = portname + + def __hash__(self): + return hash((self.chassisid, self.portid, self.portname)) + + def __cmp__(self, rhs): + return (cmp(self.chassisid, rhs.chassisid) or + cmp(self.portid, rhs.portid) or + cmp(self.portname, rhs.portname)) + + def __eq__(self, rhs): + return (self.chassisid == rhs.chassisid and + self.portid == rhs.portid and + self.portname == rhs.portname) + + def __ne__(self, rhs): + return (self.chassisid != rhs.chassisid or + self.portid != rhs.portid or + self.portname != rhs.portname) + + def __str__(self): + return "%s [%s] [%s]" % (self.portname, self.chassisid, self.portid) + + def __repr__(self): + return "" % str(self) + + +class Agent(object): + '''Class to encapsulate LLDP agent data for System Inventory''' + + def __init__(self, **kwargs): + '''Construct an Agent object with the given values.''' + self.key = Key(kwargs.get(k_lldp.LLDP_TLV_TYPE_CHASSIS_ID), + kwargs.get(k_lldp.LLDP_TLV_TYPE_PORT_ID), + kwargs.get("name_or_uuid")) + self.status = kwargs.get('status') + self.ttl = kwargs.get(k_lldp.LLDP_TLV_TYPE_TTL) + self.system_name = kwargs.get(k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME) + self.system_desc = kwargs.get(k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC) + self.port_desc = kwargs.get(k_lldp.LLDP_TLV_TYPE_PORT_DESC) + self.capabilities = kwargs.get(k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP) + self.mgmt_addr = kwargs.get(k_lldp.LLDP_TLV_TYPE_MGMT_ADDR) + self.dot1_lag = kwargs.get(k_lldp.LLDP_TLV_TYPE_DOT1_LAG) + self.dot1_vlan_names = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES) + self.dot3_max_frame = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME) + self.state = None + + def __hash__(self): + return self.key.__hash__() + + def __eq__(self, rhs): + return (self.key == rhs.key) + + def __ne__(self, rhs): + return (self.key != rhs.key or + self.status != rhs.status or + self.ttl != rhs.ttl or + self.system_name != rhs.system_name or + self.system_desc != rhs.system_desc or + self.port_desc != rhs.port_desc or + self.capabilities != rhs.capabilities or + self.mgmt_addr != rhs.mgmt_addr or + self.dot1_lag != rhs.dot1_lag or + self.dot1_vlan_names != rhs.dot1_vlan_names or + self.dot3_max_frame != rhs.dot3_max_frame or + self.state != rhs.state) + + def __str__(self): + return "%s: [%s] [%s] [%s], [%s], [%s], [%s], [%s], [%s]" % ( + self.key, self.status, self.system_name, self.system_desc, + self.port_desc, self.capabilities, + self.mgmt_addr, self.dot1_lag, + self.dot3_max_frame) + + def __repr__(self): + return "" % str(self) + + +class Neighbour(object): + '''Class to encapsulate LLDP neighbour data for System Inventory''' + + def __init__(self, **kwargs): + '''Construct an Neighbour object with the given values.''' + self.key = Key(kwargs.get(k_lldp.LLDP_TLV_TYPE_CHASSIS_ID), + kwargs.get(k_lldp.LLDP_TLV_TYPE_PORT_ID), + kwargs.get("name_or_uuid")) + self.msap = kwargs.get('msap') + self.ttl = kwargs.get(k_lldp.LLDP_TLV_TYPE_TTL) + self.system_name = kwargs.get(k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME) + self.system_desc = kwargs.get(k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC) + self.port_desc = kwargs.get(k_lldp.LLDP_TLV_TYPE_PORT_DESC) + self.capabilities = kwargs.get(k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP) + self.mgmt_addr = kwargs.get(k_lldp.LLDP_TLV_TYPE_MGMT_ADDR) + self.dot1_port_vid = kwargs.get(k_lldp.LLDP_TLV_TYPE_DOT1_PORT_VID) + self.dot1_vid_digest = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT1_VID_DIGEST) + self.dot1_mgmt_vid = kwargs.get(k_lldp.LLDP_TLV_TYPE_DOT1_MGMT_VID) + self.dot1_vid_digest = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT1_VID_DIGEST) + self.dot1_mgmt_vid = kwargs.get(k_lldp.LLDP_TLV_TYPE_DOT1_MGMT_VID) + self.dot1_lag = kwargs.get(k_lldp.LLDP_TLV_TYPE_DOT1_LAG) + self.dot1_vlan_names = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES) + self.dot1_proto_vids = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_VIDS) + self.dot1_proto_ids = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_IDS) + self.dot3_mac_status = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT3_MAC_STATUS) + self.dot3_max_frame = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME) + self.dot3_power_mdi = kwargs.get( + k_lldp.LLDP_TLV_TYPE_DOT3_POWER_MDI) + + self.state = None + + def __hash__(self): + return self.key.__hash__() + + def __eq__(self, rhs): + return (self.key == rhs.key) + + def __ne__(self, rhs): + return (self.key != rhs.key or + self.msap != rhs.msap or + self.system_name != rhs.system_name or + self.system_desc != rhs.system_desc or + self.port_desc != rhs.port_desc or + self.capabilities != rhs.capabilities or + self.mgmt_addr != rhs.mgmt_addr or + self.dot1_port_vid != rhs.dot1_port_vid or + self.dot1_vid_digest != rhs.dot1_vid_digest or + self.dot1_mgmt_vid != rhs.dot1_mgmt_vid or + self.dot1_vid_digest != rhs.dot1_vid_digest or + self.dot1_mgmt_vid != rhs.dot1_mgmt_vid or + self.dot1_lag != rhs.dot1_lag or + self.dot1_vlan_names != rhs.dot1_vlan_names or + self.dot1_proto_vids != rhs.dot1_proto_vids or + self.dot1_proto_ids != rhs.dot1_proto_ids or + self.dot3_mac_status != rhs.dot3_mac_status or + self.dot3_max_frame != rhs.dot3_max_frame or + self.dot3_power_mdi != rhs.dot3_power_mdi) + + def __str__(self): + return "%s [%s] [%s] [%s], [%s]" % ( + self.key, self.system_name, self.system_desc, + self.port_desc, self.capabilities) + + def __repr__(self): + return "" % str(self) + + +class InventoryLldpPlugin(object): + + """Implementation of the Plugin.""" + + def __init__(self): + self.manager = manager.InventoryLldpDriverManager() + + def lldp_has_neighbour(self, name): + try: + return self.manager.lldp_has_neighbour(name) + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP has neighbour failed") + + def lldp_update(self): + try: + self.manager.lldp_update() + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP update failed") + + def lldp_agents_list(self): + try: + agents = self.manager.lldp_agents_list() + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP agents list failed") + + return agents + + def lldp_agents_clear(self): + try: + self.manager.lldp_agents_clear() + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP agents clear failed") + + def lldp_neighbours_list(self): + try: + neighbours = self.manager.lldp_neighbours_list() + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP neighbours list failed") + + return neighbours + + def lldp_neighbours_clear(self): + try: + self.manager.lldp_neighbours_clear() + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP neighbours clear failed") + + def lldp_update_systemname(self, systemname): + try: + self.manager.lldp_update_systemname(systemname) + except exception.LLDPDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("LLDP update systemname failed") diff --git a/inventory/inventory/inventory/agent/manager.py b/inventory/inventory/inventory/agent/manager.py new file mode 100644 index 00000000..b9955562 --- /dev/null +++ b/inventory/inventory/inventory/agent/manager.py @@ -0,0 +1,973 @@ +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +""" Perform activity related to local inventory. + +A single instance of :py:class:`inventory.agent.manager.AgentManager` is +created within the *inventory-agent* process, and is responsible for +performing all actions for this host managed by inventory . + +On start, collect and post inventory. + +Commands (from conductors) are received via RPC calls. + +""" + +import errno +import fcntl +import os +import oslo_messaging as messaging +import socket +import subprocess +import time + +from futurist import periodics +from oslo_config import cfg +from oslo_log import log + +# from inventory.agent import partition +from inventory.agent import base_manager +from inventory.agent.lldp import plugin as lldp_plugin +from inventory.agent import node +from inventory.agent import pci +from inventory.common import constants +from inventory.common import context as mycontext +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import k_lldp +from inventory.common import utils +from inventory.conductor import rpcapi as conductor_rpcapi +import tsconfig.tsconfig as tsc + +MANAGER_TOPIC = 'inventory.agent_manager' + +LOG = log.getLogger(__name__) + +agent_opts = [ + cfg.StrOpt('api_url', + default=None, + help=('Url of Inventory API service. If not set Inventory can ' + 'get current value from Keystone service catalog.')), + cfg.IntOpt('audit_interval', + default=60, + help='Maximum time since the last check-in of a agent'), +] + +CONF = cfg.CONF +CONF.register_opts(agent_opts, 'agent') + +MAXSLEEP = 300 # 5 minutes + +INVENTORY_READY_FLAG = os.path.join(tsc.VOLATILE_PATH, ".inventory_ready") + + +FIRST_BOOT_FLAG = os.path.join( + tsc.PLATFORM_CONF_PATH, ".first_boot") + + +class AgentManager(base_manager.BaseAgentManager): + """Inventory Agent service main class.""" + + # Must be in sync with rpcapi.AgentAPI's + RPC_API_VERSION = '1.0' + + target = messaging.Target(version=RPC_API_VERSION) + + def __init__(self, host, topic): + super(AgentManager, self).__init__(host, topic) + + self._report_to_conductor = False + self._report_to_conductor_iplatform_avail_flag = False + self._ipci_operator = pci.PCIOperator() + self._inode_operator = node.NodeOperator() + self._lldp_operator = lldp_plugin.InventoryLldpPlugin() + self._ihost_personality = None + self._ihost_uuid = "" + self._agent_throttle = 0 + self._subfunctions = None + self._subfunctions_configured = False + self._notify_subfunctions_alarm_clear = False + self._notify_subfunctions_alarm_raise = False + self._first_grub_update = False + + @property + def report_to_conductor_required(self): + return self._report_to_conductor + + @report_to_conductor_required.setter + def report_to_conductor_required(self, val): + if not isinstance(val, bool): + raise ValueError("report_to_conductor_required not bool %s" % + val) + self._report_to_conductor = val + + def start(self): + # Do not collect inventory and report to conductor at startup in + # order to eliminate two inventory reports + # (one from here and one from audit) being sent to the conductor + + super(AgentManager, self).start() + + if os.path.isfile('/etc/inventory/inventory.conf'): + LOG.info("inventory-agent started, " + "inventory to be reported by audit") + else: + LOG.info("No config file for inventory-agent found.") + + if tsc.system_mode == constants.SYSTEM_MODE_SIMPLEX: + utils.touch(INVENTORY_READY_FLAG) + + def init_host(self, admin_context=None): + super(AgentManager, self).init_host(admin_context) + if os.path.isfile('/etc/inventory/inventory.conf'): + LOG.info(_("inventory-agent started, " + "system config to be reported by audit")) + else: + LOG.info(_("No config file for inventory-agent found.")) + + if tsc.system_mode == constants.SYSTEM_MODE_SIMPLEX: + utils.touch(INVENTORY_READY_FLAG) + + def del_host(self, deregister=True): + return + + def periodic_tasks(self, context, raise_on_error=False): + """Periodic tasks are run at pre-specified intervals. """ + return self.run_periodic_tasks(context, + raise_on_error=raise_on_error) + + def _report_to_conductor_iplatform_avail(self): + utils.touch(INVENTORY_READY_FLAG) + time.sleep(1) # give time for conductor to process + self._report_to_conductor_iplatform_avail_flag = True + + def _update_ttys_dcd_status(self, context, host_id): + # Retrieve the serial line carrier detect flag + ttys_dcd = None + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + try: + ttys_dcd = rpcapi.get_host_ttys_dcd(context, host_id) + except exception.InventoryException: + LOG.exception("Inventory Agent exception getting host ttys_dcd.") + pass + if ttys_dcd is not None: + self._config_ttys_login(ttys_dcd) + else: + LOG.debug("ttys_dcd is not configured") + + @staticmethod + def _get_active_device(): + # the list of currently configured console devices, + # like 'tty1 ttyS0' or just 'ttyS0' + # The last entry in the file is the active device connected + # to /dev/console. + active_device = 'ttyS0' + try: + cmd = 'cat /sys/class/tty/console/active | grep ttyS' + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + output = proc.stdout.read().strip() + proc.communicate()[0] + if proc.returncode != 0: + LOG.info("Cannot find the current configured serial device, " + "return default %s" % active_device) + return active_device + # if more than one devices are found, take the last entry + if ' ' in output: + devs = output.split(' ') + active_device = devs[len(devs) - 1] + else: + active_device = output + except subprocess.CalledProcessError as e: + LOG.error("Failed to execute (%s) (%d)", cmd, e.returncode) + except OSError as e: + LOG.error("Failed to execute (%s) OS error (%d)", cmd, e.errno) + + return active_device + + @staticmethod + def _is_local_flag_disabled(device): + """ + :param device: + :return: boolean: True if the local flag is disabled 'i.e. -clocal is + set'. This means the serial data carrier detect + signal is significant + """ + try: + # uses -o for only-matching and -e for a pattern beginning with a + # hyphen (-), the following command returns 0 if the local flag + # is disabled + cmd = 'stty -a -F /dev/%s | grep -o -e -clocal' % device + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) + proc.communicate()[0] + return proc.returncode == 0 + except subprocess.CalledProcessError as e: + LOG.error("Failed to execute (%s) (%d)", cmd, e.returncode) + return False + except OSError as e: + LOG.error("Failed to execute (%s) OS error (%d)", cmd, e.errno) + return False + + def _config_ttys_login(self, ttys_dcd): + # agetty is now enabled by systemd + # we only need to disable the local flag to enable carrier detection + # and enable the local flag when the feature is turned off + toggle_flag = None + active_device = self._get_active_device() + local_flag_disabled = self._is_local_flag_disabled(active_device) + if str(ttys_dcd) in ['True', 'true']: + LOG.info("ttys_dcd is enabled") + # check if the local flag is disabled + if not local_flag_disabled: + LOG.info("Disable (%s) local line" % active_device) + toggle_flag = 'stty -clocal -F /dev/%s' % active_device + else: + if local_flag_disabled: + # enable local flag to ignore the carrier detection + LOG.info("Enable local flag for device :%s" % active_device) + toggle_flag = 'stty clocal -F /dev/%s' % active_device + + if toggle_flag: + try: + subprocess.Popen(toggle_flag, stdout=subprocess.PIPE, + shell=True) + # restart serial-getty + restart_cmd = ('systemctl restart serial-getty@%s.service' + % active_device) + subprocess.check_call(restart_cmd, shell=True) + except subprocess.CalledProcessError as e: + LOG.error("subprocess error: (%d)", e.returncode) + + def _force_grub_update(self): + """Force update the grub on the first AIO controller after the initial + config is completed + """ + if (not self._first_grub_update and + os.path.isfile(tsc.INITIAL_CONFIG_COMPLETE_FLAG)): + self._first_grub_update = True + return True + return False + + def host_lldp_get_and_report(self, context, rpcapi, host_uuid): + neighbour_dict_array = [] + agent_dict_array = [] + neighbours = [] + agents = [] + + try: + neighbours = self._lldp_operator.lldp_neighbours_list() + except Exception as e: + LOG.error("Failed to get LLDP neighbours: %s", str(e)) + + for neighbour in neighbours: + neighbour_dict = { + 'name_or_uuid': neighbour.key.portname, + 'msap': neighbour.msap, + 'state': neighbour.state, + k_lldp.LLDP_TLV_TYPE_CHASSIS_ID: neighbour.key.chassisid, + k_lldp.LLDP_TLV_TYPE_PORT_ID: neighbour.key.portid, + k_lldp.LLDP_TLV_TYPE_TTL: neighbour.ttl, + k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME: neighbour.system_name, + k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC: neighbour.system_desc, + k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP: neighbour.capabilities, + k_lldp.LLDP_TLV_TYPE_MGMT_ADDR: neighbour.mgmt_addr, + k_lldp.LLDP_TLV_TYPE_PORT_DESC: neighbour.port_desc, + k_lldp.LLDP_TLV_TYPE_DOT1_LAG: neighbour.dot1_lag, + k_lldp.LLDP_TLV_TYPE_DOT1_PORT_VID: neighbour.dot1_port_vid, + k_lldp.LLDP_TLV_TYPE_DOT1_VID_DIGEST: + neighbour.dot1_vid_digest, + k_lldp.LLDP_TLV_TYPE_DOT1_MGMT_VID: neighbour.dot1_mgmt_vid, + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_VIDS: + neighbour.dot1_proto_vids, + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_IDS: + neighbour.dot1_proto_ids, + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES: + neighbour.dot1_vlan_names, + k_lldp.LLDP_TLV_TYPE_DOT3_MAC_STATUS: + neighbour.dot3_mac_status, + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME: + neighbour.dot3_max_frame, + k_lldp.LLDP_TLV_TYPE_DOT3_POWER_MDI: + neighbour.dot3_power_mdi, + } + neighbour_dict_array.append(neighbour_dict) + + if neighbour_dict_array: + try: + rpcapi.lldp_neighbour_update_by_host(context, + host_uuid, + neighbour_dict_array) + except exception.InventoryException: + LOG.exception("Inventory Agent exception updating " + "lldp neighbours.") + self._lldp_operator.lldp_neighbours_clear() + pass + + try: + agents = self._lldp_operator.lldp_agents_list() + except Exception as e: + LOG.error("Failed to get LLDP agents: %s", str(e)) + + for agent in agents: + agent_dict = { + 'name_or_uuid': agent.key.portname, + 'state': agent.state, + 'status': agent.status, + k_lldp.LLDP_TLV_TYPE_CHASSIS_ID: agent.key.chassisid, + k_lldp.LLDP_TLV_TYPE_PORT_ID: agent.key.portid, + k_lldp.LLDP_TLV_TYPE_TTL: agent.ttl, + k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME: agent.system_name, + k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC: agent.system_desc, + k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP: agent.capabilities, + k_lldp.LLDP_TLV_TYPE_MGMT_ADDR: agent.mgmt_addr, + k_lldp.LLDP_TLV_TYPE_PORT_DESC: agent.port_desc, + k_lldp.LLDP_TLV_TYPE_DOT1_LAG: agent.dot1_lag, + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES: agent.dot1_vlan_names, + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME: agent.dot3_max_frame, + } + agent_dict_array.append(agent_dict) + + if agent_dict_array: + try: + rpcapi.lldp_agent_update_by_host(context, + host_uuid, + agent_dict_array) + except exception.InventoryException: + LOG.exception("Inventory Agent exception updating " + "lldp agents.") + self._lldp_operator.lldp_agents_clear() + pass + + def synchronized_network_config(func): + """Synchronization decorator to acquire and release + network_config_lock. + """ + + def wrap(self, *args, **kwargs): + try: + # Get lock to avoid conflict with apply_network_config.sh + lockfd = self._acquire_network_config_lock() + return func(self, *args, **kwargs) + finally: + self._release_network_config_lock(lockfd) + + return wrap + + @synchronized_network_config + def _lldp_enable_and_report(self, context, rpcapi, host_uuid): + """Temporarily enable interfaces and get lldp neighbor information. + This method should only be called before + INITIAL_CONFIG_COMPLETE_FLAG is set. + """ + links_down = [] + try: + # Turn on interfaces, so that lldpd can show all neighbors + for interface in self._ipci_operator.pci_get_net_names(): + flag = self._ipci_operator.pci_get_net_flags(interface) + # If administrative state is down, bring it up momentarily + if not (flag & pci.IFF_UP): + subprocess.call(['ip', 'link', 'set', interface, 'up']) + links_down.append(interface) + LOG.info('interface %s enabled to receive LLDP PDUs' % + interface) + self._lldp_operator.lldp_update() + + # delay maximum 30 seconds for lldpd to receive LLDP PDU + timeout = 0 + link_wait_for_lldp = True + while timeout < 30 and link_wait_for_lldp and links_down: + time.sleep(5) + timeout = timeout + 5 + link_wait_for_lldp = False + + for link in links_down: + if not self._lldp_operator.lldp_has_neighbour(link): + link_wait_for_lldp = True + break + self.host_lldp_get_and_report(context, rpcapi, host_uuid) + except Exception as e: + LOG.exception(e) + pass + finally: + # restore interface administrative state + for interface in links_down: + subprocess.call(['ip', 'link', 'set', interface, 'down']) + LOG.info('interface %s disabled after querying LLDP neighbors' + % interface) + + def platform_update_by_host(self, rpcapi, context, host_uuid, msg_dict): + """Update host platform information. + If this is the first boot (kickstart), then also update the Host + Action State to reinstalled, and remove the flag. + """ + if os.path.exists(FIRST_BOOT_FLAG): + msg_dict.update({k_host.HOST_ACTION_STATE: + k_host.HAS_REINSTALLED}) + + try: + rpcapi.platform_update_by_host(context, + host_uuid, + msg_dict) + if os.path.exists(FIRST_BOOT_FLAG): + os.remove(FIRST_BOOT_FLAG) + LOG.info("Removed %s" % FIRST_BOOT_FLAG) + except exception.InventoryException: + LOG.warn("platform_update_by_host exception " + "host_uuid=%s msg_dict=%s." % + (host_uuid, msg_dict)) + pass + + LOG.info("Inventory Agent platform update by host: %s" % msg_dict) + + def _acquire_network_config_lock(self): + """Synchronization with apply_network_config.sh + + This method is to acquire the lock to avoid + conflict with execution of apply_network_config.sh + during puppet manifest application. + + :returns: fd of the lock, if successful. 0 on error. + """ + lock_file_fd = os.open( + constants.NETWORK_CONFIG_LOCK_FILE, os.O_CREAT | os.O_RDONLY) + count = 1 + delay = 5 + max_count = 5 + while count <= max_count: + try: + fcntl.flock(lock_file_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return lock_file_fd + except IOError as e: + # raise on unrelated IOErrors + if e.errno != errno.EAGAIN: + raise + else: + LOG.info("Could not acquire lock({}): {} ({}/{}), " + "will retry".format(lock_file_fd, str(e), + count, max_count)) + time.sleep(delay) + count += 1 + LOG.error("Failed to acquire lock (fd={})".format(lock_file_fd)) + return 0 + + def _release_network_config_lock(self, lockfd): + """Release the lock guarding apply_network_config.sh """ + if lockfd: + fcntl.flock(lockfd, fcntl.LOCK_UN) + os.close(lockfd) + + def ihost_inv_get_and_report(self, icontext): + """Collect data for an ihost. + + This method allows an ihost data to be collected. + + :param: icontext: an admin context + :returns: updated ihost object, including all fields. + """ + + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + + ihost = None + + # find list of network related inics for this ihost + inics = self._ipci_operator.inics_get() + + # create an array of ports for each net entry of the NIC device + iports = [] + for inic in inics: + lockfd = 0 + try: + # Get lock to avoid conflict with apply_network_config.sh + lockfd = self._acquire_network_config_lock() + pci_net_array = \ + self._ipci_operator.pci_get_net_attrs(inic.pciaddr) + finally: + self._release_network_config_lock(lockfd) + for net in pci_net_array: + iports.append(pci.Port(inic, **net)) + + # find list of pci devices for this host + pci_devices = self._ipci_operator.pci_devices_get() + + # create an array of pci_devs for each net entry of the device + pci_devs = [] + for pci_dev in pci_devices: + pci_dev_array = \ + self._ipci_operator.pci_get_device_attrs(pci_dev.pciaddr) + for dev in pci_dev_array: + pci_devs.append(pci.PCIDevice(pci_dev, **dev)) + + # create a list of MAC addresses that will be used to identify the + # inventoried host (one of the MACs should be the management MAC) + host_macs = [port.mac for port in iports if port.mac] + + # get my ihost record which should be avail since booted + + LOG.debug('Inventory Agent iports={}, host_macs={}'.format( + iports, host_macs)) + + slept = 0 + while slept < MAXSLEEP: + # wait for controller to come up first may be a DOR + try: + ihost = rpcapi.get_host_by_macs(icontext, host_macs) + except messaging.MessagingTimeout: + LOG.info("get_host_by_macs Messaging Timeout.") + except Exception as ex: + LOG.warn("Conductor RPC get_host_by_macs exception " + "response %s" % ex) + + if not ihost: + hostname = socket.gethostname() + if hostname != k_host.LOCALHOST_HOSTNAME: + try: + ihost = rpcapi.get_host_by_hostname(icontext, + hostname) + except messaging.MessagingTimeout: + LOG.info("get_host_by_hostname Messaging Timeout.") + return # wait for next audit cycle + except Exception as ex: + LOG.warn("Conductor RPC get_host_by_hostname " + "exception response %s" % ex) + + if ihost and ihost.get('personality'): + self.report_to_conductor_required = True + self._ihost_uuid = ihost['uuid'] + self._ihost_personality = ihost['personality'] + + if os.path.isfile(tsc.PLATFORM_CONF_FILE): + # read the platform config file and check for UUID + found = False + with open(tsc.PLATFORM_CONF_FILE, "r") as fd: + for line in fd: + if line.find("UUID=") == 0: + found = True + if not found: + # the UUID is not found, append it + with open(tsc.PLATFORM_CONF_FILE, "a") as fd: + fd.write("UUID=" + self._ihost_uuid + "\n") + + # Report host install status + msg_dict = {} + self.platform_update_by_host(rpcapi, + icontext, + self._ihost_uuid, + msg_dict) + LOG.info("Agent found matching ihost: %s" % ihost['uuid']) + break + + time.sleep(30) + slept += 30 + + if not self.report_to_conductor_required: + # let the audit take care of it instead + LOG.info("Inventory no matching ihost found... await Audit") + return + + subfunctions = self.subfunctions_get() + try: + rpcapi.subfunctions_update_by_host(icontext, + ihost['uuid'], + subfunctions) + except exception.InventoryException: + LOG.exception("Inventory Agent exception updating " + "subfunctions conductor.") + pass + + # post to inventory db by ihost['uuid'] + iport_dict_array = [] + for port in iports: + inic_dict = {'pciaddr': port.ipci.pciaddr, + 'pclass': port.ipci.pclass, + 'pvendor': port.ipci.pvendor, + 'pdevice': port.ipci.pdevice, + 'prevision': port.ipci.prevision, + 'psvendor': port.ipci.psvendor, + 'psdevice': port.ipci.psdevice, + 'pname': port.name, + 'numa_node': port.numa_node, + 'sriov_totalvfs': port.sriov_totalvfs, + 'sriov_numvfs': port.sriov_numvfs, + 'sriov_vfs_pci_address': port.sriov_vfs_pci_address, + 'driver': port.driver, + 'mac': port.mac, + 'mtu': port.mtu, + 'speed': port.speed, + 'link_mode': port.link_mode, + 'dev_id': port.dev_id, + 'dpdksupport': port.dpdksupport} + + LOG.debug('Inventory Agent inic {}'.format(inic_dict)) + + iport_dict_array.append(inic_dict) + try: + # may get duplicate key if already sent on earlier init + rpcapi.port_update_by_host(icontext, + ihost['uuid'], + iport_dict_array) + except messaging.MessagingTimeout: + LOG.info("pci_device_update_by_host Messaging Timeout.") + self.report_to_conductor_required = False + return # wait for next audit cycle + + # post to inventory db by ihost['uuid'] + pci_device_dict_array = [] + for dev in pci_devs: + pci_dev_dict = {'name': dev.name, + 'pciaddr': dev.pci.pciaddr, + 'pclass_id': dev.pclass_id, + 'pvendor_id': dev.pvendor_id, + 'pdevice_id': dev.pdevice_id, + 'pclass': dev.pci.pclass, + 'pvendor': dev.pci.pvendor, + 'pdevice': dev.pci.pdevice, + 'prevision': dev.pci.prevision, + 'psvendor': dev.pci.psvendor, + 'psdevice': dev.pci.psdevice, + 'numa_node': dev.numa_node, + 'sriov_totalvfs': dev.sriov_totalvfs, + 'sriov_numvfs': dev.sriov_numvfs, + 'sriov_vfs_pci_address': dev.sriov_vfs_pci_address, + 'driver': dev.driver, + 'enabled': dev.enabled, + 'extra_info': dev.extra_info} + LOG.debug('Inventory Agent dev {}'.format(pci_dev_dict)) + + pci_device_dict_array.append(pci_dev_dict) + try: + # may get duplicate key if already sent on earlier init + rpcapi.pci_device_update_by_host(icontext, + ihost['uuid'], + pci_device_dict_array) + except messaging.MessagingTimeout: + LOG.info("pci_device_update_by_host Messaging Timeout.") + self.report_to_conductor_required = True + + # Find list of numa_nodes and cpus for this ihost + inumas, icpus = self._inode_operator.inodes_get_inumas_icpus() + + try: + # may get duplicate key if already sent on earlier init + rpcapi.numas_update_by_host(icontext, + ihost['uuid'], + inumas) + except messaging.RemoteError as e: + LOG.error("numas_update_by_host RemoteError exc_type=%s" % + e.exc_type) + except messaging.MessagingTimeout: + LOG.info("pci_device_update_by_host Messaging Timeout.") + self.report_to_conductor_required = True + except Exception as e: + LOG.exception("Inventory Agent exception updating inuma e=%s." % e) + pass + + force_grub_update = self._force_grub_update() + try: + # may get duplicate key if already sent on earlier init + rpcapi.cpus_update_by_host(icontext, + ihost['uuid'], + icpus, + force_grub_update) + except messaging.RemoteError as e: + LOG.error("cpus_update_by_host RemoteError exc_type=%s" % + e.exc_type) + except messaging.MessagingTimeout: + LOG.info("cpus_update_by_host Messaging Timeout.") + self.report_to_conductor_required = True + except Exception as e: + LOG.exception("Inventory exception updating cpus e=%s." % e) + self.report_to_conductor_required = True + pass + except exception.InventoryException: + LOG.exception("Inventory exception updating cpus conductor.") + pass + + imemory = self._inode_operator.inodes_get_imemory() + if imemory: + try: + # may get duplicate key if already sent on earlier init + rpcapi.memory_update_by_host(icontext, + ihost['uuid'], + imemory) + except messaging.MessagingTimeout: + LOG.info("memory_update_by_host Messaging Timeout.") + except messaging.RemoteError as e: + LOG.error("memory_update_by_host RemoteError exc_type=%s" % + e.exc_type) + except exception.InventoryException: + LOG.exception("Inventory Agent exception updating imemory " + "conductor.") + + if self._ihost_uuid and \ + os.path.isfile(tsc.INITIAL_CONFIG_COMPLETE_FLAG): + if not self._report_to_conductor_iplatform_avail_flag: + # and not self._wait_for_nova_lvg() + imsg_dict = {'availability': k_host.AVAILABILITY_AVAILABLE} + + iscsi_initiator_name = self.get_host_iscsi_initiator_name() + if iscsi_initiator_name is not None: + imsg_dict.update({'iscsi_initiator_name': + iscsi_initiator_name}) + + # Before setting the host to AVAILABILITY_AVAILABLE make + # sure that nova_local aggregates are correctly set + self.platform_update_by_host(rpcapi, + icontext, + self._ihost_uuid, + imsg_dict) + + self._report_to_conductor_iplatform_avail() + + def subfunctions_get(self): + """returns subfunctions on this host. + """ + + self._subfunctions = ','.join(tsc.subfunctions) + + return self._subfunctions + + @staticmethod + def subfunctions_list_get(): + """returns list of subfunctions on this host. + """ + subfunctions = ','.join(tsc.subfunctions) + subfunctions_list = subfunctions.split(',') + + return subfunctions_list + + def subfunctions_configured(self, subfunctions_list): + """Determines whether subfunctions configuration is completed. + return: Bool whether subfunctions configuration is completed. + """ + if (k_host.CONTROLLER in subfunctions_list and + k_host.COMPUTE in subfunctions_list): + if not os.path.exists(tsc.INITIAL_COMPUTE_CONFIG_COMPLETE): + self._subfunctions_configured = False + return False + + self._subfunctions_configured = True + return True + + @staticmethod + def _wait_for_nova_lvg(icontext, rpcapi, ihost_uuid, nova_lvgs=None): + """See if we wait for a provisioned nova-local volume group + + This method queries the conductor to see if we are provisioning + a nova-local volume group on this boot cycle. This check is used + to delay sending the platform availability to the conductor. + + :param: icontext: an admin context + :param: rpcapi: conductor rpc api + :param: ihost_uuid: an admin context + :returns: True if we are provisioning false otherwise + """ + + return True + LOG.info("TODO _wait_for_nova_lvg from systemconfig") + + def _is_config_complete(self): + """Check if this node has completed config + + This method queries node's config flag file to see if it has + complete config. + :return: True if the complete flag file exists false otherwise + """ + if not os.path.isfile(tsc.INITIAL_CONFIG_COMPLETE_FLAG): + return False + subfunctions = self.subfunctions_list_get() + if k_host.CONTROLLER in subfunctions: + if not os.path.isfile(tsc.INITIAL_CONTROLLER_CONFIG_COMPLETE): + return False + if k_host.COMPUTE in subfunctions: + if not os.path.isfile(tsc.INITIAL_COMPUTE_CONFIG_COMPLETE): + return False + if k_host.STORAGE in subfunctions: + if not os.path.isfile(tsc.INITIAL_STORAGE_CONFIG_COMPLETE): + return False + return True + + @periodics.periodic(spacing=CONF.agent.audit_interval, + run_immediately=True) + def _agent_audit(self, context): + # periodically, perform inventory audit + self.agent_audit(context, host_uuid=self._ihost_uuid, + force_updates=None) + + def agent_audit(self, context, + host_uuid, force_updates, cinder_device=None): + # perform inventory audit + if self._ihost_uuid != host_uuid: + # The function call is not for this host agent + return + + icontext = mycontext.get_admin_context() + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + + if not self.report_to_conductor_required: + LOG.info("Inventory Agent audit running inv_get_and_report.") + self.ihost_inv_get_and_report(icontext) + + if self._ihost_uuid and os.path.isfile( + tsc.INITIAL_CONFIG_COMPLETE_FLAG): + if (not self._report_to_conductor_iplatform_avail_flag and + not self._wait_for_nova_lvg( + icontext, rpcapi, self._ihost_uuid)): + imsg_dict = {'availability': k_host.AVAILABILITY_AVAILABLE} + + iscsi_initiator_name = self.get_host_iscsi_initiator_name() + if iscsi_initiator_name is not None: + imsg_dict.update({'iscsi_initiator_name': + iscsi_initiator_name}) + + # Before setting the host to AVAILABILITY_AVAILABLE make + # sure that nova_local aggregates are correctly set + self.platform_update_by_host(rpcapi, + icontext, + self._ihost_uuid, + imsg_dict) + + self._report_to_conductor_iplatform_avail() + + if (self._ihost_personality == k_host.CONTROLLER and + not self._notify_subfunctions_alarm_clear): + + subfunctions_list = self.subfunctions_list_get() + if ((k_host.CONTROLLER in subfunctions_list) and + (k_host.COMPUTE in subfunctions_list)): + if self.subfunctions_configured(subfunctions_list) and \ + not self._wait_for_nova_lvg( + icontext, rpcapi, self._ihost_uuid): + + ihost_notify_dict = {'subfunctions_configured': True} + rpcapi.notify_subfunctions_config(icontext, + self._ihost_uuid, + ihost_notify_dict) + self._notify_subfunctions_alarm_clear = True + else: + if not self._notify_subfunctions_alarm_raise: + ihost_notify_dict = {'subfunctions_configured': + False} + rpcapi.notify_subfunctions_config( + icontext, self._ihost_uuid, ihost_notify_dict) + self._notify_subfunctions_alarm_raise = True + else: + self._notify_subfunctions_alarm_clear = True + + if self._ihost_uuid: + LOG.debug("Inventory Agent Audit running.") + + if force_updates: + LOG.debug("Inventory Agent Audit force updates: (%s)" % + (', '.join(force_updates))) + + self._update_ttys_dcd_status(icontext, self._ihost_uuid) + if self._agent_throttle > 5: + # throttle updates + self._agent_throttle = 0 + imemory = self._inode_operator.inodes_get_imemory() + rpcapi.memory_update_by_host(icontext, + self._ihost_uuid, + imemory) + if self._is_config_complete(): + self.host_lldp_get_and_report( + icontext, rpcapi, self._ihost_uuid) + else: + self._lldp_enable_and_report( + icontext, rpcapi, self._ihost_uuid) + self._agent_throttle += 1 + + if os.path.isfile(tsc.PLATFORM_CONF_FILE): + # read the platform config file and check for UUID + if 'UUID' not in open(tsc.PLATFORM_CONF_FILE).read(): + # the UUID is not in found, append it + with open(tsc.PLATFORM_CONF_FILE, "a") as fd: + fd.write("UUID=" + self._ihost_uuid) + + def configure_lldp_systemname(self, context, systemname): + """Configure the systemname into the lldp agent with the supplied data. + + :param context: an admin context. + :param systemname: the systemname + """ + + # TODO(sc): This becomes an inventory-api call from + # via systemconfig: configure_isystemname + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + # Update the lldp agent + self._lldp_operator.lldp_update_systemname(systemname) + # Trigger an audit to ensure the db is up to date + self.host_lldp_get_and_report(context, rpcapi, self._ihost_uuid) + + def configure_ttys_dcd(self, context, uuid, ttys_dcd): + """Configure the getty on the serial device. + + :param context: an admin context. + :param uuid: the host uuid + :param ttys_dcd: the flag to enable/disable dcd + """ + + LOG.debug("AgentManager.configure_ttys_dcd: %s %s" % (uuid, ttys_dcd)) + if self._ihost_uuid and self._ihost_uuid == uuid: + LOG.debug("AgentManager configure getty on serial console") + self._config_ttys_login(ttys_dcd) + return + + def execute_command(self, context, host_uuid, command): + """Execute a command on behalf of inventory-conductor + + :param context: request context + :param host_uuid: the host uuid + :param command: the command to execute + """ + + LOG.debug("AgentManager.execute_command: (%s)" % command) + if self._ihost_uuid and self._ihost_uuid == host_uuid: + LOG.info("AgentManager execute_command: (%s)" % command) + with open(os.devnull, "w") as fnull: + try: + subprocess.check_call(command, stdout=fnull, stderr=fnull) + except subprocess.CalledProcessError as e: + LOG.error("Failed to execute (%s) (%d)", + command, e.returncode) + except OSError as e: + LOG.error("Failed to execute (%s), OS error:(%d)", + command, e.errno) + + LOG.info("(%s) executed.", command) + + def get_host_iscsi_initiator_name(self): + iscsi_initiator_name = None + try: + stdout, __ = utils.execute('cat', '/etc/iscsi/initiatorname.iscsi', + run_as_root=True) + if stdout: + stdout = stdout.strip() + iscsi_initiator_name = stdout.split('=')[-1] + LOG.info("iscsi initiator name = %s" % iscsi_initiator_name) + except Exception: + LOG.error("Failed retrieving iscsi initiator name") + + return iscsi_initiator_name + + def update_host_memory(self, context, host_uuid): + """update the host memory + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :return: None + """ + if self._ihost_uuid and self._ihost_uuid == host_uuid: + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + memory = self._inode_operator.inodes_get_imemory() + rpcapi.memory_update_by_host(context, + self._ihost_uuid, + memory, + force_update=True) diff --git a/inventory/inventory/inventory/agent/node.py b/inventory/inventory/inventory/agent/node.py new file mode 100644 index 00000000..8552a116 --- /dev/null +++ b/inventory/inventory/inventory/agent/node.py @@ -0,0 +1,608 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +""" inventory numa node Utilities and helper functions.""" + +import os +from os import listdir +from os.path import isfile +from os.path import join +from oslo_log import log +import re +import subprocess +import tsconfig.tsconfig as tsc + +LOG = log.getLogger(__name__) + +# Defines per-socket vswitch memory requirements (in MB) +VSWITCH_MEMORY_MB = 1024 + +# Defines the size of one kilobyte +SIZE_KB = 1024 + +# Defines the size of 2 megabytes in kilobyte units +SIZE_2M_KB = 2048 + +# Defines the size of 1 gigabyte in kilobyte units +SIZE_1G_KB = 1048576 + +# Defines the size of 2 megabytes in megabyte units +SIZE_2M_MB = int(SIZE_2M_KB / SIZE_KB) + +# Defines the size of 1 gigabyte in megabyte units +SIZE_1G_MB = int(SIZE_1G_KB / SIZE_KB) + +# Defines the minimum size of memory for a controller node in megabyte units +CONTROLLER_MIN_MB = 6000 + +# Defines the minimum size of memory for a compute node in megabyte units +COMPUTE_MIN_MB = 1600 + +# Defines the minimum size of memory for a secondary compute node in megabyte +# units +COMPUTE_MIN_NON_0_MB = 500 + + +class CPU(object): + '''Class to encapsulate CPU data for System Inventory''' + + def __init__(self, cpu, numa_node, core, thread, + cpu_family=None, cpu_model=None, revision=None): + '''Construct a cpu object with the given values.''' + + self.cpu = cpu + self.numa_node = numa_node + self.core = core + self.thread = thread + self.cpu_family = cpu_family + self.cpu_model = cpu_model + self.revision = revision + # self.allocated_functions = mgmt (usu. 0), vswitch + + def __eq__(self, rhs): + return (self.cpu == rhs.cpu and + self.numa_node == rhs.numa_node and + self.core == rhs.core and + self.thread == rhs.thread) + + def __ne__(self, rhs): + return (self.cpu != rhs.cpu or + self.numa_node != rhs.numa_node or + self.core != rhs.core or + self.thread != rhs.thread) + + def __str__(self): + return "%s [%s] [%s] [%s]" % (self.cpu, self.numa_node, + self.core, self.thread) + + def __repr__(self): + return "" % str(self) + + +class NodeOperator(object): + '''Class to encapsulate CPU operations for System Inventory''' + + def __init__(self): + + self.num_cpus = 0 + self.num_nodes = 0 + self.float_cpuset = 0 + self.total_memory_mb = 0 + self.free_memory_mb = 0 + self.total_memory_nodes_mb = [] + self.free_memory_nodes_mb = [] + self.topology = {} + + # self._get_cpu_topology() + # self._get_total_memory_mb() + # self._get_total_memory_nodes_mb() + # self._get_free_memory_mb() + # self._get_free_memory_nodes_mb() + + def _is_strict(self): + with open(os.devnull, "w") as fnull: + try: + output = subprocess.check_output( + ["cat", "/proc/sys/vm/overcommit_memory"], + stderr=fnull) + if int(output) == 2: + return True + except subprocess.CalledProcessError as e: + LOG.info("Failed to check for overcommit, error (%s)", + e.output) + return False + + def convert_range_string_to_list(self, s): + olist = [] + s = s.strip() + if s: + for part in s.split(','): + if '-' in part: + a, b = part.split('-') + a, b = int(a), int(b) + olist.extend(range(a, b + 1)) + else: + a = int(part) + olist.append(a) + olist.sort() + return olist + + def inodes_get_inumas_icpus(self): + '''Enumerate logical cpu topology based on parsing /proc/cpuinfo + as function of socket_id, core_id, and thread_id. This updates + topology. + + :param self + :updates self.num_cpus- number of logical cpus + :updates self.num_nodes- number of sockets;maps to number of numa nodes + :updates self.topology[socket_id][core_id][thread_id] = cpu + :returns None + ''' + self.num_cpus = 0 + self.num_nodes = 0 + self.topology = {} + + thread_cnt = {} + cpu = socket_id = core_id = thread_id = -1 + re_processor = re.compile(r'^[Pp]rocessor\s+:\s+(\d+)') + re_socket = re.compile(r'^physical id\s+:\s+(\d+)') + re_core = re.compile(r'^core id\s+:\s+(\d+)') + re_cpu_family = re.compile(r'^cpu family\s+:\s+(\d+)') + re_cpu_model = re.compile(r'^model name\s+:\s+(\w+)') + + inumas = [] + icpus = [] + sockets = [] + + with open('/proc/cpuinfo', 'r') as infile: + icpu_attrs = {} + + for line in infile: + match = re_processor.search(line) + if match: + cpu = int(match.group(1)) + socket_id = -1 + core_id = -1 + thread_id = -1 + self.num_cpus += 1 + continue + + match = re_cpu_family.search(line) + if match: + name_value = [s.strip() for s in line.split(':', 1)] + name, value = name_value + icpu_attrs.update({'cpu_family': value}) + continue + + match = re_cpu_model.search(line) + if match: + name_value = [s.strip() for s in line.split(':', 1)] + name, value = name_value + icpu_attrs.update({'cpu_model': value}) + continue + + match = re_socket.search(line) + if match: + socket_id = int(match.group(1)) + if socket_id not in sockets: + sockets.append(socket_id) + attrs = { + 'numa_node': socket_id, + 'capabilities': {}, + } + inumas.append(attrs) + continue + + match = re_core.search(line) + if match: + core_id = int(match.group(1)) + + if socket_id not in thread_cnt: + thread_cnt[socket_id] = {} + if core_id not in thread_cnt[socket_id]: + thread_cnt[socket_id][core_id] = 0 + else: + thread_cnt[socket_id][core_id] += 1 + thread_id = thread_cnt[socket_id][core_id] + + if socket_id not in self.topology: + self.topology[socket_id] = {} + if core_id not in self.topology[socket_id]: + self.topology[socket_id][core_id] = {} + + self.topology[socket_id][core_id][thread_id] = cpu + attrs = { + 'cpu': cpu, + 'numa_node': socket_id, + 'core': core_id, + 'thread': thread_id, + 'capabilities': {}, + } + icpu_attrs.update(attrs) + icpus.append(icpu_attrs) + icpu_attrs = {} + continue + + self.num_nodes = len(self.topology.keys()) + + # In the case topology not detected, hard-code structures + if self.num_nodes == 0: + n_sockets, n_cores, n_threads = (1, int(self.num_cpus), 1) + self.topology = {} + for socket_id in range(n_sockets): + self.topology[socket_id] = {} + if socket_id not in sockets: + sockets.append(socket_id) + attrs = { + 'numa_node': socket_id, + 'capabilities': {}, + } + inumas.append(attrs) + for core_id in range(n_cores): + self.topology[socket_id][core_id] = {} + for thread_id in range(n_threads): + self.topology[socket_id][core_id][thread_id] = 0 + attrs = { + 'cpu': cpu, + 'numa_node': socket_id, + 'core': core_id, + 'thread': thread_id, + 'capabilities': {}, + } + icpus.append(attrs) + + # Define Thread-Socket-Core order for logical cpu enumeration + cpu = 0 + for thread_id in range(n_threads): + for core_id in range(n_cores): + for socket_id in range(n_sockets): + if socket_id not in sockets: + sockets.append(socket_id) + attrs = { + 'numa_node': socket_id, + 'capabilities': {}, + } + inumas.append(attrs) + self.topology[socket_id][core_id][thread_id] = cpu + attrs = { + 'cpu': cpu, + 'numa_node': socket_id, + 'core': core_id, + 'thread': thread_id, + 'capabilities': {}, + } + icpus.append(attrs) + cpu += 1 + self.num_nodes = len(self.topology.keys()) + + LOG.debug("inumas= %s, cpus = %s" % (inumas, icpus)) + + return inumas, icpus + + def _get_immediate_subdirs(self, dir): + return [name for name in listdir(dir) + if os.path.isdir(join(dir, name))] + + def _inode_get_memory_hugepages(self): + """Collect hugepage info, including vswitch, and vm. + Collect platform reserved if config. + :param self + :returns list of memory nodes and attributes + """ + + imemory = [] + + initial_compute_config_completed = \ + os.path.exists(tsc.INITIAL_COMPUTE_CONFIG_COMPLETE) + + # check if it is initial report before the huge pages are allocated + initial_report = not initial_compute_config_completed + + # do not send report if the initial compute config is completed and + # compute config has not finished, i.e.during subsequent + # reboot before the manifest allocates the huge pages + compute_config_completed = \ + os.path.exists(tsc.VOLATILE_COMPUTE_CONFIG_COMPLETE) + if (initial_compute_config_completed and + not compute_config_completed): + return imemory + + for node in range(self.num_nodes): + attr = {} + total_hp_mb = 0 # Total memory (MB) currently configured in HPs + free_hp_mb = 0 + + # Check vswitch and libvirt memory + # Loop through configured hugepage sizes of this node and record + # total number and number free + hugepages = "/sys/devices/system/node/node%d/hugepages" % node + + try: + subdirs = self._get_immediate_subdirs(hugepages) + + for subdir in subdirs: + hp_attr = {} + sizesplit = subdir.split('-') + if sizesplit[1].startswith("1048576kB"): + size = SIZE_1G_MB + else: + size = SIZE_2M_MB + + nr_hugepages = 0 + free_hugepages = 0 + + mydir = hugepages + '/' + subdir + files = [f for f in listdir(mydir) + if isfile(join(mydir, f))] + + if files: + for file in files: + with open(mydir + '/' + file, 'r') as f: + if file.startswith("nr_hugepages"): + nr_hugepages = int(f.readline()) + if file.startswith("free_hugepages"): + free_hugepages = int(f.readline()) + + total_hp_mb = total_hp_mb + int(nr_hugepages * size) + free_hp_mb = free_hp_mb + int(free_hugepages * size) + + # Libvirt hugepages can be 1G and 2M + if size == SIZE_1G_MB: + vswitch_hugepages_nr = VSWITCH_MEMORY_MB / size + hp_attr = { + 'vswitch_hugepages_size_mib': size, + 'vswitch_hugepages_nr': vswitch_hugepages_nr, + 'vswitch_hugepages_avail': 0, + 'vm_hugepages_nr_1G': + (nr_hugepages - vswitch_hugepages_nr), + 'vm_hugepages_avail_1G': free_hugepages, + 'vm_hugepages_use_1G': 'True' + } + else: + if len(subdirs) == 1: + # No 1G hugepage support. + vswitch_hugepages_nr = VSWITCH_MEMORY_MB / size + hp_attr = { + 'vswitch_hugepages_size_mib': size, + 'vswitch_hugepages_nr': vswitch_hugepages_nr, + 'vswitch_hugepages_avail': 0, + } + hp_attr.update({'vm_hugepages_use_1G': 'False'}) + else: + # vswitch will use 1G hugpages + vswitch_hugepages_nr = 0 + + hp_attr.update({ + 'vm_hugepages_avail_2M': free_hugepages, + 'vm_hugepages_nr_2M': + (nr_hugepages - vswitch_hugepages_nr) + }) + + attr.update(hp_attr) + + except IOError: + # silently ignore IO errors (eg. file missing) + pass + + # Get the free and total memory from meminfo for this node + re_node_memtotal = re.compile(r'^Node\s+\d+\s+\MemTotal:\s+(\d+)') + re_node_memfree = re.compile(r'^Node\s+\d+\s+\MemFree:\s+(\d+)') + re_node_filepages = \ + re.compile(r'^Node\s+\d+\s+\FilePages:\s+(\d+)') + re_node_sreclaim = \ + re.compile(r'^Node\s+\d+\s+\SReclaimable:\s+(\d+)') + re_node_commitlimit = \ + re.compile(r'^Node\s+\d+\s+\CommitLimit:\s+(\d+)') + re_node_committed_as = \ + re.compile(r'^Node\s+\d+\s+\'Committed_AS:\s+(\d+)') + + free_kb = 0 # Free Memory (KB) available + total_kb = 0 # Total Memory (KB) + limit = 0 # only used in strict accounting + committed = 0 # only used in strict accounting + + meminfo = "/sys/devices/system/node/node%d/meminfo" % node + try: + with open(meminfo, 'r') as infile: + for line in infile: + match = re_node_memtotal.search(line) + if match: + total_kb += int(match.group(1)) + continue + match = re_node_memfree.search(line) + if match: + free_kb += int(match.group(1)) + continue + match = re_node_filepages.search(line) + if match: + free_kb += int(match.group(1)) + continue + match = re_node_sreclaim.search(line) + if match: + free_kb += int(match.group(1)) + continue + match = re_node_commitlimit.search(line) + if match: + limit = int(match.group(1)) + continue + match = re_node_committed_as.search(line) + if match: + committed = int(match.group(1)) + continue + + if self._is_strict(): + free_kb = limit - committed + + except IOError: + # silently ignore IO errors (eg. file missing) + pass + + # Calculate PSS + pss_mb = 0 + if node == 0: + cmd = 'cat /proc/*/smaps 2>/dev/null | awk \'/^Pss:/ ' \ + '{a += $2;} END {printf "%d\\n", a/1024.0;}\'' + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + shell=True) + result = proc.stdout.read().strip() + pss_mb = int(result) + except subprocess.CalledProcessError as e: + LOG.error("Cannot calculate PSS (%s) (%d)", cmd, + e.returncode) + except OSError as e: + LOG.error("Failed to execute (%s) OS error (%d)", cmd, + e.errno) + + # need to multiply total_mb by 1024 to match compute_huge + node_total_kb = total_hp_mb * SIZE_KB + free_kb + pss_mb * SIZE_KB + + # Read base memory from compute_reserved.conf + base_mem_mb = 0 + with open('/etc/nova/compute_reserved.conf', 'r') as infile: + for line in infile: + if "COMPUTE_BASE_RESERVED" in line: + val = line.split("=") + base_reserves = val[1].strip('\n')[1:-1] + for reserve in base_reserves.split(): + reserve = reserve.split(":") + if reserve[0].strip('"') == "node%d" % node: + base_mem_mb = int(reserve[1].strip('MB')) + + # On small systems, clip memory overhead to more reasonable minimal + # settings + if (total_kb / SIZE_KB - base_mem_mb) < 1000: + if node == 0: + base_mem_mb = COMPUTE_MIN_MB + if tsc.nodetype == 'controller': + base_mem_mb += CONTROLLER_MIN_MB + else: + base_mem_mb = COMPUTE_MIN_NON_0_MB + + eng_kb = node_total_kb - base_mem_mb * SIZE_KB + + vswitch_mem_kb = (attr.get('vswitch_hugepages_size_mib', 0) * + attr.get('vswitch_hugepages_nr', 0) * SIZE_KB) + + vm_kb = (eng_kb - vswitch_mem_kb) + + max_vm_pages_2mb = vm_kb / SIZE_2M_KB + max_vm_pages_1gb = vm_kb / SIZE_1G_KB + + attr.update({ + 'vm_hugepages_possible_2M': max_vm_pages_2mb, + 'vm_hugepages_possible_1G': max_vm_pages_1gb, + }) + + # calculate 90% 2M pages if it is initial report and the huge + # pages have not been allocated + if initial_report: + max_vm_pages_2mb = max_vm_pages_2mb * 0.9 + total_hp_mb += int(max_vm_pages_2mb * (SIZE_2M_KB / SIZE_KB)) + free_hp_mb = total_hp_mb + attr.update({ + 'vm_hugepages_nr_2M': max_vm_pages_2mb, + 'vm_hugepages_avail_2M': max_vm_pages_2mb, + 'vm_hugepages_nr_1G': 0 + }) + + attr.update({ + 'numa_node': node, + 'memtotal_mib': total_hp_mb, + 'memavail_mib': free_hp_mb, + 'hugepages_configured': 'True', + 'node_memtotal_mib': node_total_kb / 1024, + }) + + imemory.append(attr) + + return imemory + + def _inode_get_memory_nonhugepages(self): + '''Collect nonhugepage info, including platform reserved if config. + :param self + :returns list of memory nodes and attributes + ''' + + imemory = [] + self.total_memory_mb = 0 + + re_node_memtotal = re.compile(r'^Node\s+\d+\s+\MemTotal:\s+(\d+)') + re_node_memfree = re.compile(r'^Node\s+\d+\s+\MemFree:\s+(\d+)') + re_node_filepages = re.compile(r'^Node\s+\d+\s+\FilePages:\s+(\d+)') + re_node_sreclaim = re.compile(r'^Node\s+\d+\s+\SReclaimable:\s+(\d+)') + + for node in range(self.num_nodes): + attr = {} + total_mb = 0 + free_mb = 0 + + meminfo = "/sys/devices/system/node/node%d/meminfo" % node + try: + with open(meminfo, 'r') as infile: + for line in infile: + match = re_node_memtotal.search(line) + if match: + total_mb += int(match.group(1)) + continue + + match = re_node_memfree.search(line) + if match: + free_mb += int(match.group(1)) + continue + match = re_node_filepages.search(line) + if match: + free_mb += int(match.group(1)) + continue + match = re_node_sreclaim.search(line) + if match: + free_mb += int(match.group(1)) + continue + + except IOError: + # silently ignore IO errors (eg. file missing) + pass + + total_mb /= 1024 + free_mb /= 1024 + self.total_memory_nodes_mb.append(total_mb) + attr = { + 'numa_node': node, + 'memtotal_mib': total_mb, + 'memavail_mib': free_mb, + 'hugepages_configured': 'False', + } + + imemory.append(attr) + + return imemory + + def inodes_get_imemory(self): + '''Enumerate logical memory topology based on: + if CONF.compute_hugepages: + self._inode_get_memory_hugepages() + else: + self._inode_get_memory_nonhugepages() + + :param self + :returns list of memory nodes and attributes + ''' + imemory = [] + + # if CONF.compute_hugepages: + if os.path.isfile("/etc/nova/compute_reserved.conf"): + imemory = self._inode_get_memory_hugepages() + else: + imemory = self._inode_get_memory_nonhugepages() + + LOG.debug("imemory= %s" % imemory) + + return imemory diff --git a/inventory/inventory/inventory/agent/pci.py b/inventory/inventory/inventory/agent/pci.py new file mode 100644 index 00000000..db9daa4a --- /dev/null +++ b/inventory/inventory/inventory/agent/pci.py @@ -0,0 +1,621 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +""" inventory pci Utilities and helper functions.""" + +import glob +import os +import shlex +import subprocess + +from inventory.common import k_pci +from inventory.common import utils +from oslo_log import log + +LOG = log.getLogger(__name__) + +# Look for PCI class 0x0200 and 0x0280 so that we get generic ethernet +# controllers and those that may report as "other" network controllers. +ETHERNET_PCI_CLASSES = ['ethernet controller', 'network controller'] + +# Look for other devices we may want to inventory. +KNOWN_PCI_DEVICES = [ + {"vendor_id": k_pci.NOVA_PCI_ALIAS_QAT_PF_VENDOR, + "device_id": k_pci.NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE, + "class_id": k_pci.NOVA_PCI_ALIAS_QAT_CLASS}, + {"vendor_id": k_pci.NOVA_PCI_ALIAS_QAT_PF_VENDOR, + "device_id": k_pci.NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE, + "class_id": k_pci.NOVA_PCI_ALIAS_QAT_CLASS}, + {"class_id": k_pci.NOVA_PCI_ALIAS_GPU_CLASS}] + +# PCI-SIG 0x06 bridge devices to not inventory. +IGNORE_BRIDGE_PCI_CLASSES = ['bridge', 'isa bridge', 'host bridge'] + +# PCI-SIG 0x08 generic peripheral devices to not inventory. +IGNORE_PERIPHERAL_PCI_CLASSES = ['system peripheral', 'pic', 'dma controller', + 'iommu', 'rtc'] + +# PCI-SIG 0x11 signal processing devices to not inventory. +IGNORE_SIGNAL_PROCESSING_PCI_CLASSES = ['performance counters'] + +# Blacklist of devices we do not want to inventory, because they are dealt +# with separately (ie. Ethernet devices), or do not make sense to expose +# to a guest. +IGNORE_PCI_CLASSES = ETHERNET_PCI_CLASSES + IGNORE_BRIDGE_PCI_CLASSES + \ + IGNORE_PERIPHERAL_PCI_CLASSES + IGNORE_SIGNAL_PROCESSING_PCI_CLASSES + +pciaddr = 0 +pclass = 1 +pvendor = 2 +pdevice = 3 +prevision = 4 +psvendor = 5 +psdevice = 6 + +VALID_PORT_SPEED = ['10', '100', '1000', '10000', '40000', '100000'] + +# Network device flags (from include/uapi/linux/if.h) +IFF_UP = 1 << 0 +IFF_BROADCAST = 1 << 1 +IFF_DEBUG = 1 << 2 +IFF_LOOPBACK = 1 << 3 +IFF_POINTOPOINT = 1 << 4 +IFF_NOTRAILERS = 1 << 5 +IFF_RUNNING = 1 << 6 +IFF_NOARP = 1 << 7 +IFF_PROMISC = 1 << 8 +IFF_ALLMULTI = 1 << 9 +IFF_MASTER = 1 << 10 +IFF_SLAVE = 1 << 11 +IFF_MULTICAST = 1 << 12 +IFF_PORTSEL = 1 << 13 +IFF_AUTOMEDIA = 1 << 14 +IFF_DYNAMIC = 1 << 15 + + +class PCI(object): + '''Class to encapsulate PCI data for System Inventory''' + + def __init__(self, pciaddr, pclass, pvendor, pdevice, prevision, + psvendor, psdevice): + '''Construct a pci object with the given values.''' + + self.pciaddr = pciaddr + self.pclass = pclass + self.pvendor = pvendor + self.pdevice = pdevice + self.prevision = prevision + self.psvendor = psvendor + self.psdevice = psdevice + + def __eq__(self, rhs): + return (self.pvendor == rhs.pvendor and + self.pdevice == rhs.pdevice) + + def __ne__(self, rhs): + return (self.pvendor != rhs.pvendor or + self.pdevice != rhs.pdevice) + + def __str__(self): + return "%s [%s] [%s]" % (self.pciaddr, self.pvendor, self.pdevice) + + def __repr__(self): + return "" % str(self) + + +class Port(object): + '''Class to encapsulate PCI data for System Inventory''' + + def __init__(self, ipci, **kwargs): + '''Construct an port object with the given values.''' + self.ipci = ipci + self.name = kwargs.get('name') + self.mac = kwargs.get('mac') + self.mtu = kwargs.get('mtu') + self.speed = kwargs.get('speed') + self.link_mode = kwargs.get('link_mode') + self.numa_node = kwargs.get('numa_node') + self.dev_id = kwargs.get('dev_id') + self.sriov_totalvfs = kwargs.get('sriov_totalvfs') + self.sriov_numvfs = kwargs.get('sriov_numvfs') + self.sriov_vfs_pci_address = kwargs.get('sriov_vfs_pci_address') + self.driver = kwargs.get('driver') + self.dpdksupport = kwargs.get('dpdksupport') + + def __str__(self): + return "%s %s: [%s] [%s] [%s], [%s], [%s], [%s], [%s]" % ( + self.ipci, self.name, self.mac, self.mtu, self.speed, + self.link_mode, self.numa_node, self.dev_id, self.dpdksupport) + + def __repr__(self): + return "" % str(self) + + +class PCIDevice(object): + '''Class to encapsulate extended PCI data for System Inventory''' + + def __init__(self, pci, **kwargs): + '''Construct a PciDevice object with the given values.''' + self.pci = pci + self.name = kwargs.get('name') + self.pclass_id = kwargs.get('pclass_id') + self.pvendor_id = kwargs.get('pvendor_id') + self.pdevice_id = kwargs.get('pdevice_id') + self.numa_node = kwargs.get('numa_node') + self.sriov_totalvfs = kwargs.get('sriov_totalvfs') + self.sriov_numvfs = kwargs.get('sriov_numvfs') + self.sriov_vfs_pci_address = kwargs.get('sriov_vfs_pci_address') + self.driver = kwargs.get('driver') + self.enabled = kwargs.get('enabled') + self.extra_info = kwargs.get('extra_info') + + def __str__(self): + return "%s %s: [%s]" % ( + self.pci, self.numa_node, self.driver) + + def __repr__(self): + return "" % str(self) + + +class PCIOperator(object): + '''Class to encapsulate PCI operations for System Inventory''' + + def format_lspci_output(self, device): + # hack for now + if device[prevision].strip() == device[pvendor].strip(): + # no revision info + device.append(device[psvendor]) + device[psvendor] = device[prevision] + device[prevision] = "0" + elif len(device) <= 6: # one less entry, no revision + LOG.debug("update psdevice length=%s" % len(device)) + device.append(device[psvendor]) + return device + + def get_pci_numa_node(self, pciaddr): + fnuma_node = '/sys/bus/pci/devices/' + pciaddr + '/numa_node' + try: + with open(fnuma_node, 'r') as f: + numa_node = f.readline().strip() + LOG.debug("ATTR numa_node: %s " % numa_node) + except Exception: + LOG.debug("ATTR numa_node unknown for: %s " % pciaddr) + numa_node = None + return numa_node + + def get_pci_sriov_totalvfs(self, pciaddr): + fsriov_totalvfs = '/sys/bus/pci/devices/' + pciaddr + '/sriov_totalvfs' + try: + with open(fsriov_totalvfs, 'r') as f: + sriov_totalvfs = f.readline() + LOG.debug("ATTR sriov_totalvfs: %s " % sriov_totalvfs) + f.close() + except Exception: + LOG.debug("ATTR sriov_totalvfs unknown for: %s " % pciaddr) + sriov_totalvfs = None + pass + return sriov_totalvfs + + def get_pci_sriov_numvfs(self, pciaddr): + fsriov_numvfs = '/sys/bus/pci/devices/' + pciaddr + '/sriov_numvfs' + try: + with open(fsriov_numvfs, 'r') as f: + sriov_numvfs = f.readline() + LOG.debug("ATTR sriov_numvfs: %s " % sriov_numvfs) + f.close() + except Exception: + LOG.debug("ATTR sriov_numvfs unknown for: %s " % pciaddr) + sriov_numvfs = 0 + pass + LOG.debug("sriov_numvfs: %s" % sriov_numvfs) + return sriov_numvfs + + def get_pci_sriov_vfs_pci_address(self, pciaddr, sriov_numvfs): + dirpcidev = '/sys/bus/pci/devices/' + pciaddr + sriov_vfs_pci_address = [] + i = 0 + while i < int(sriov_numvfs): + lvf = dirpcidev + '/virtfn' + str(i) + try: + sriov_vfs_pci_address.append( + os.path.basename(os.readlink(lvf))) + except Exception: + LOG.warning("virtfn link %s non-existent (sriov_numvfs=%s)" + % (lvf, sriov_numvfs)) + pass + i += 1 + LOG.debug("sriov_vfs_pci_address: %s" % sriov_vfs_pci_address) + return sriov_vfs_pci_address + + def get_pci_driver_name(self, pciaddr): + ddriver = '/sys/bus/pci/devices/' + pciaddr + '/driver/module/drivers' + try: + drivers = [ + os.path.basename(os.readlink(ddriver + '/' + d)) + for d in os.listdir(ddriver)] + driver = str(','.join(str(d) for d in drivers)) + + except Exception: + LOG.debug("ATTR driver unknown for: %s " % pciaddr) + driver = None + pass + LOG.debug("driver: %s" % driver) + return driver + + def pci_devices_get(self): + + p = subprocess.Popen(["lspci", "-Dm"], stdout=subprocess.PIPE) + + pci_devices = [] + for line in p.stdout: + pci_device = shlex.split(line.strip()) + pci_device = self.format_lspci_output(pci_device) + + if any(x in pci_device[pclass].lower() for x in + IGNORE_PCI_CLASSES): + continue + + dirpcidev = '/sys/bus/pci/devices/' + physfn = dirpcidev + pci_device[pciaddr] + '/physfn' + if not os.path.isdir(physfn): + # Do not report VFs + pci_devices.append(PCI(pci_device[pciaddr], + pci_device[pclass], + pci_device[pvendor], + pci_device[pdevice], + pci_device[prevision], + pci_device[psvendor], + pci_device[psdevice])) + + p.wait() + + return pci_devices + + def inics_get(self): + + p = subprocess.Popen(["lspci", "-Dm"], stdout=subprocess.PIPE) + + pci_inics = [] + for line in p.stdout: + inic = shlex.split(line.strip()) + if any(x in inic[pclass].lower() for x in ETHERNET_PCI_CLASSES): + # hack for now + if inic[prevision].strip() == inic[pvendor].strip(): + # no revision info + inic.append(inic[psvendor]) + inic[psvendor] = inic[prevision] + inic[prevision] = "0" + elif len(inic) <= 6: # one less entry, no revision + LOG.debug("update psdevice length=%s" % len(inic)) + inic.append(inic[psvendor]) + + dirpcidev = '/sys/bus/pci/devices/' + physfn = dirpcidev + inic[pciaddr] + '/physfn' + if os.path.isdir(physfn): + # Do not report VFs + continue + pci_inics.append(PCI(inic[pciaddr], inic[pclass], + inic[pvendor], inic[pdevice], + inic[prevision], inic[psvendor], + inic[psdevice])) + + p.wait() + + return pci_inics + + def pci_get_enabled_attr(self, class_id, vendor_id, product_id): + for known_device in KNOWN_PCI_DEVICES: + if (class_id == known_device.get("class_id", None) or + (vendor_id == known_device.get("vendor_id", None) and + product_id == known_device.get("device_id", None))): + return True + return False + + def pci_get_device_attrs(self, pciaddr): + """For this pciaddr, build a list of device attributes """ + pci_attrs_array = [] + + dirpcidev = '/sys/bus/pci/devices/' + pciaddrs = os.listdir(dirpcidev) + + for a in pciaddrs: + if ((a == pciaddr) or (a == ("0000:" + pciaddr))): + LOG.debug("Found device pci bus: %s " % a) + + dirpcideva = dirpcidev + a + + numa_node = self.get_pci_numa_node(a) + sriov_totalvfs = self.get_pci_sriov_totalvfs(a) + sriov_numvfs = self.get_pci_sriov_numvfs(a) + sriov_vfs_pci_address = \ + self.get_pci_sriov_vfs_pci_address(a, sriov_numvfs) + driver = self.get_pci_driver_name(a) + + fclass = dirpcideva + '/class' + fvendor = dirpcideva + '/vendor' + fdevice = dirpcideva + '/device' + try: + with open(fvendor, 'r') as f: + pvendor_id = f.readline().strip('0x').strip() + except Exception: + LOG.debug("ATTR vendor unknown for: %s " % a) + pvendor_id = None + + try: + with open(fdevice, 'r') as f: + pdevice_id = f.readline().replace('0x', '').strip() + except Exception: + LOG.debug("ATTR device unknown for: %s " % a) + pdevice_id = None + + try: + with open(fclass, 'r') as f: + pclass_id = f.readline().replace('0x', '').strip() + except Exception: + LOG.debug("ATTR class unknown for: %s " % a) + pclass_id = None + + name = "pci_" + a.replace(':', '_').replace('.', '_') + + attrs = { + "name": name, + "pci_address": a, + "pclass_id": pclass_id, + "pvendor_id": pvendor_id, + "pdevice_id": pdevice_id, + "numa_node": numa_node, + "sriov_totalvfs": sriov_totalvfs, + "sriov_numvfs": sriov_numvfs, + "sriov_vfs_pci_address": + ','.join(str(x) for x in sriov_vfs_pci_address), + "driver": driver, + "enabled": self.pci_get_enabled_attr( + pclass_id, pvendor_id, pdevice_id), + } + + pci_attrs_array.append(attrs) + + return pci_attrs_array + + def get_pci_net_directory(self, pciaddr): + device_directory = '/sys/bus/pci/devices/' + pciaddr + # Look for the standard device 'net' directory + net_directory = device_directory + '/net/' + if os.path.exists(net_directory): + return net_directory + # Otherwise check whether this is a virtio based device + net_pattern = device_directory + '/virtio*/net/' + results = glob.glob(net_pattern) + if not results: + return None + if len(results) > 1: + LOG.warning("PCI device {} has multiple virtio " + "sub-directories".format(pciaddr)) + return results[0] + + def _read_flags(self, fflags): + try: + with open(fflags, 'r') as f: + hex_str = f.readline().rstrip() + flags = int(hex_str, 16) + except Exception: + flags = None + return flags + + def _get_netdev_flags(self, dirpcinet, pci): + fflags = dirpcinet + pci + '/flags' + return self._read_flags(fflags) + + def pci_get_net_flags(self, name): + fflags = '/sys/class/net/' + name + '/flags' + return self._read_flags(fflags) + + def pci_get_net_names(self): + '''build a list of network device names.''' + names = [] + for name in os.listdir('/sys/class/net/'): + if os.path.isdir('/sys/class/net/' + name): + names.append(name) + return names + + def pci_get_net_attrs(self, pciaddr): + """For this pciaddr, build a list of network attributes per port""" + pci_attrs_array = [] + + dirpcidev = '/sys/bus/pci/devices/' + pciaddrs = os.listdir(dirpcidev) + + for a in pciaddrs: + if ((a == pciaddr) or (a == ("0000:" + pciaddr))): + # Look inside net expect to find address,speed,mtu etc. info + # There may be more than 1 net device for this NIC. + LOG.debug("Found NIC pci bus: %s " % a) + + dirpcideva = dirpcidev + a + + numa_node = self.get_pci_numa_node(a) + sriov_totalvfs = self.get_pci_sriov_totalvfs(a) + sriov_numvfs = self.get_pci_sriov_numvfs(a) + sriov_vfs_pci_address = \ + self.get_pci_sriov_vfs_pci_address(a, sriov_numvfs) + driver = self.get_pci_driver_name(a) + + # Determine DPDK support + dpdksupport = False + fvendor = dirpcideva + '/vendor' + fdevice = dirpcideva + '/device' + try: + with open(fvendor, 'r') as f: + vendor = f.readline().strip() + except Exception: + LOG.debug("ATTR vendor unknown for: %s " % a) + vendor = None + + try: + with open(fdevice, 'r') as f: + device = f.readline().strip() + except Exception: + LOG.debug("ATTR device unknown for: %s " % a) + device = None + + try: + with open(os.devnull, "w") as fnull: + subprocess.check_call( + ["query_pci_id", "-v " + str(vendor), + "-d " + str(device)], + stdout=fnull, stderr=fnull) + dpdksupport = True + LOG.debug("DPDK does support NIC " + "(vendor: %s device: %s)", + vendor, device) + except subprocess.CalledProcessError as e: + dpdksupport = False + if e.returncode == 1: + # NIC is not supprted + LOG.debug("DPDK does not support NIC " + "(vendor: %s device: %s)", + vendor, device) + else: + # command failed, default to DPDK support to False + LOG.info("Could not determine DPDK support for " + "NIC (vendor %s device: %s), defaulting " + "to False", vendor, device) + + # determine the net directory for this device + dirpcinet = self.get_pci_net_directory(a) + if dirpcinet is None: + LOG.warning("no /net for PCI device: %s " % a) + continue # go to next PCI device + + # determine which netdevs are associated to this device + netdevs = os.listdir(dirpcinet) + for n in netdevs: + mac = None + fmac = dirpcinet + n + '/' + "address" + fmaster = dirpcinet + n + '/' + "master" + # if a port is a member of a bond the port MAC address + # must be retrieved from /proc/net/bonding/ + if os.path.exists(fmaster): + dirmaster = os.path.realpath(fmaster) + master_name = os.path.basename(dirmaster) + procnetbonding = '/proc/net/bonding/' + master_name + found_interface = False + + try: + with open(procnetbonding, 'r') as f: + for line in f: + if 'Slave Interface: ' + n in line: + found_interface = True + if (found_interface and + 'Permanent HW addr:' in line): + mac = line.split(': ')[1].rstrip() + mac = utils.validate_and_normalize_mac( + mac) + break + if not mac: + LOG.info("ATTR mac could not be determined" + " for slave interface %s" % n) + except Exception: + LOG.info("ATTR mac could not be determined, " + "could not open %s" % procnetbonding) + else: + try: + with open(fmac, 'r') as f: + mac = f.readline().rstrip() + mac = utils.validate_and_normalize_mac(mac) + except Exception: + LOG.info("ATTR mac unknown for: %s " % n) + + fmtu = dirpcinet + n + '/' + "mtu" + try: + with open(fmtu, 'r') as f: + mtu = f.readline().rstrip() + except Exception: + LOG.debug("ATTR mtu unknown for: %s " % n) + mtu = None + + # Check the administrative state before reading the speed + flags = self._get_netdev_flags(dirpcinet, n) + + # If administrative state is down, bring it up momentarily + if not(flags & IFF_UP): + LOG.warning("Enabling device %s to query link speed" % + n) + cmd = 'ip link set dev %s up' % n + subprocess.Popen(cmd, stdout=subprocess.PIPE, + shell=True) + # Read the speed + fspeed = dirpcinet + n + '/' + "speed" + try: + with open(fspeed, 'r') as f: + speed = f.readline().rstrip() + if speed not in VALID_PORT_SPEED: + LOG.error("Invalid port speed = %s for %s " % + (speed, n)) + speed = None + except Exception: + LOG.warning("ATTR speed unknown for: %s " + "(flags: %s)" % (n, hex(flags))) + speed = None + # If the administrative state was down, take it back down + if not(flags & IFF_UP): + LOG.warning("Disabling device %s after querying " + "link speed" % n) + cmd = 'ip link set dev %s down' % n + subprocess.Popen(cmd, stdout=subprocess.PIPE, + shell=True) + + flink_mode = dirpcinet + n + '/' + "link_mode" + try: + with open(flink_mode, 'r') as f: + link_mode = f.readline().rstrip() + except Exception: + LOG.debug("ATTR link_mode unknown for: %s " % n) + link_mode = None + + fdevport = dirpcinet + n + '/' + "dev_port" + try: + with open(fdevport, 'r') as f: + dev_port = int(f.readline().rstrip(), 0) + except Exception: + LOG.debug("ATTR dev_port unknown for: %s " % n) + # Kernel versions older than 3.15 used dev_id + # (incorrectly) to identify the network devices, + # therefore support the fallback if dev_port is not + # available + try: + fdevid = dirpcinet + n + '/' + "dev_id" + with open(fdevid, 'r') as f: + dev_port = int(f.readline().rstrip(), 0) + except Exception: + LOG.debug("ATTR dev_id unknown for: %s " % n) + dev_port = 0 + + attrs = { + "name": n, + "numa_node": numa_node, + "sriov_totalvfs": sriov_totalvfs, + "sriov_numvfs": sriov_numvfs, + "sriov_vfs_pci_address": + ','.join(str(x) for x in sriov_vfs_pci_address), + "driver": driver, + "pci_address": a, + "mac": mac, + "mtu": mtu, + "speed": speed, + "link_mode": link_mode, + "dev_id": dev_port, + "dpdksupport": dpdksupport + } + + pci_attrs_array.append(attrs) + + return pci_attrs_array diff --git a/inventory/inventory/inventory/agent/rpcapi.py b/inventory/inventory/inventory/agent/rpcapi.py new file mode 100644 index 00000000..bd836f20 --- /dev/null +++ b/inventory/inventory/inventory/agent/rpcapi.py @@ -0,0 +1,161 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 + +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +""" +Client side of the agent RPC API. +""" + +from oslo_log import log +import oslo_messaging as messaging + +from inventory.common import rpc +from inventory.objects import base as objects_base + + +LOG = log.getLogger(__name__) + +MANAGER_TOPIC = 'inventory.agent_manager' + + +class AgentAPI(object): + """Client side of the agent RPC API. + + API version history: + + 1.0 - Initial version. + """ + + RPC_API_VERSION = '1.0' + + def __init__(self, topic=None): + + super(AgentAPI, self).__init__() + self.topic = topic + if self.topic is None: + self.topic = MANAGER_TOPIC + target = messaging.Target(topic=self.topic, + version='1.0') + serializer = objects_base.InventoryObjectSerializer() + version_cap = self.RPC_API_VERSION + self.client = rpc.get_client(target, + version_cap=version_cap, + serializer=serializer) + + def host_inventory(self, context, values, topic=None): + """Synchronously, have a agent collect inventory for this host. + + Collect ihost inventory and report to conductor. + + :param context: request context. + :param values: dictionary with initial values for new host object + :returns: created ihost object, including all fields. + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'host_inventory', + values=values) + + def configure_ttys_dcd(self, context, uuid, ttys_dcd, topic=None): + """Asynchronously, have the agent configure the getty on the serial + console. + + :param context: request context. + :param uuid: the host uuid + :param ttys_dcd: the flag to enable/disable dcd + :returns: none ... uses asynchronous cast(). + """ + # fanout / broadcast message to all inventory agents + LOG.debug("AgentApi.configure_ttys_dcd: fanout_cast: sending " + "dcd update to agent: (%s) (%s" % (uuid, ttys_dcd)) + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0', + fanout=True) + retval = cctxt.cast(context, + 'configure_ttys_dcd', + uuid=uuid, + ttys_dcd=ttys_dcd) + + return retval + + def execute_command(self, context, host_uuid, command, topic=None): + """Asynchronously, have the agent execute a command + + :param context: request context. + :param host_uuid: the host uuid + :param command: the command to execute + :returns: none ... uses asynchronous cast(). + """ + # fanout / broadcast message to all inventory agents + LOG.debug("AgentApi.update_cpu_config: fanout_cast: sending " + "host uuid: (%s) " % host_uuid) + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0', + fanout=True) + retval = cctxt.cast(context, + 'execute_command', + host_uuid=host_uuid, + command=command) + return retval + + def agent_update(self, context, host_uuid, force_updates, + cinder_device=None, + topic=None): + """ + Asynchronously, have the agent update partitions, ipv and ilvg state + + :param context: request context + :param host_uuid: the host uuid + :param force_updates: list of inventory objects to update + :param cinder_device: device by path of cinder volumes + :return: none ... uses asynchronous cast(). + """ + + # fanout / broadcast message to all inventory agents + LOG.info("AgentApi.agent_update: fanout_cast: sending " + "update request to agent for: (%s)" % + (', '.join(force_updates))) + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0', + fanout=True) + retval = cctxt.cast(context, + 'agent_audit', + host_uuid=host_uuid, + force_updates=force_updates, + cinder_device=cinder_device) + return retval + + def disk_format_gpt(self, context, host_uuid, idisk_dict, + is_cinder_device, topic=None): + """Asynchronously, GPT format a disk. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param idisk_dict: values for disk object + :param is_cinder_device: bool value tells if the idisk is for cinder + :returns: pass or fail + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0', + fanout=True) + + return cctxt.cast(context, + 'disk_format_gpt', + host_uuid=host_uuid, + idisk_dict=idisk_dict, + is_cinder_device=is_cinder_device) diff --git a/inventory/inventory/inventory/api/__init__.py b/inventory/inventory/inventory/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/api/app.py b/inventory/inventory/inventory/api/app.py new file mode 100644 index 00000000..8d27ffd2 --- /dev/null +++ b/inventory/inventory/inventory/api/app.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from oslo_config import cfg +from oslo_log import log +from oslo_service import service +from oslo_service import wsgi +import pecan + +from inventory.api import config +from inventory.api import middleware +from inventory.common.i18n import _ +from inventory.common import policy + +CONF = cfg.CONF + +LOG = log.getLogger(__name__) + +_launcher = None +_launcher_pxe = None + + +def get_pecan_config(): + # Set up the pecan configuration + filename = config.__file__.replace('.pyc', '.py') + return pecan.configuration.conf_from_file(filename) + + +def setup_app(config=None): + policy.init_enforcer() + + if not config: + config = get_pecan_config() + + pecan.configuration.set_config(dict(config), overwrite=True) + app_conf = dict(config.app) + + app = pecan.make_app( + app_conf.pop('root'), + debug=CONF.debug, + logging=getattr(config, 'logging', {}), + force_canonical=getattr(config.app, 'force_canonical', True), + guess_content_type_from_ext=False, + wrap_app=middleware.ParsableErrorMiddleware, + **app_conf + ) + return app + + +def load_paste_app(app_name=None): + """Loads a WSGI app from a paste config file.""" + if app_name is None: + app_name = cfg.CONF.prog + + loader = wsgi.Loader(cfg.CONF) + app = loader.load_app(app_name) + return app + + +def app_factory(global_config, **local_conf): + return setup_app() + + +def serve(api_service, conf, workers=1): + global _launcher + + if _launcher: + raise RuntimeError(_('serve() _launcher can only be called once')) + + _launcher = service.launch(conf, api_service, workers=workers) + + +def serve_pxe(api_service, conf, workers=1): + global _launcher_pxe + + if _launcher_pxe: + raise RuntimeError(_('serve() _launcher_pxe can only be called once')) + + _launcher_pxe = service.launch(conf, api_service, workers=workers) + + +def wait(): + _launcher.wait() + + +def wait_pxe(): + _launcher_pxe.wait() diff --git a/inventory/inventory/inventory/api/config.py b/inventory/inventory/inventory/api/config.py new file mode 100644 index 00000000..4f2aaecc --- /dev/null +++ b/inventory/inventory/inventory/api/config.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from inventory.api import hooks +from inventory.common import config +from inventory import objects +from keystoneauth1 import loading as ks_loading +from oslo_config import cfg +from oslo_log import log as logging +import pbr.version +import sys + +LOG = logging.getLogger(__name__) + +sysinv_group = cfg.OptGroup( + 'sysinv', + title='Sysinv Options', + help="Configuration options for the platform service") + +sysinv_opts = [ + cfg.StrOpt('catalog_info', + default='platform:sysinv:internalURL', + help="Service catalog Look up info."), + cfg.StrOpt('os_region_name', + default='RegionOne', + help="Region name of this node. It is used for catalog lookup"), +] + +version_info = pbr.version.VersionInfo('inventory') + +# Pecan Application Configurations +app = { + 'root': 'inventory.api.controllers.root.RootController', + 'modules': ['inventory.api'], + 'hooks': [ + hooks.DBHook(), + hooks.ContextHook(), + hooks.RPCHook(), + hooks.SystemConfigHook(), + ], + 'acl_public_routes': [ + '/', + '/v1', + ], +} + + +def init(args, **kwargs): + cfg.CONF.register_group(sysinv_group) + cfg.CONF.register_opts(sysinv_opts, group=sysinv_group) + ks_loading.register_session_conf_options(cfg.CONF, + sysinv_group.name) + logging.register_options(cfg.CONF) + + cfg.CONF(args=args, project='inventory', + version='%%(prog)s %s' % version_info.release_string(), + **kwargs) + objects.register_all() + config.parse_args(args) + + +def setup_logging(): + """Sets up the logging options for a log with supplied name.""" + logging.setup(cfg.CONF, "inventory") + LOG.debug("Logging enabled!") + LOG.debug("%(prog)s version %(version)s", + {'prog': sys.argv[0], + 'version': version_info.release_string()}) + LOG.debug("command line: %s", " ".join(sys.argv)) diff --git a/inventory/inventory/inventory/api/controllers/__init__.py b/inventory/inventory/inventory/api/controllers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/api/controllers/root.py b/inventory/inventory/inventory/api/controllers/root.py new file mode 100644 index 00000000..01c26715 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/root.py @@ -0,0 +1,115 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import pecan +from pecan import rest +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers import v1 +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import link + +ID_VERSION = 'v1' + + +def expose(*args, **kwargs): + """Ensure that only JSON, and not XML, is supported.""" + if 'rest_content_types' not in kwargs: + kwargs['rest_content_types'] = ('json',) + return wsme_pecan.wsexpose(*args, **kwargs) + + +class Version(base.APIBase): + """An API version representation. + + This class represents an API version, including the minimum and + maximum minor versions that are supported within the major version. + """ + + id = wtypes.text + """The ID of the (major) version, also acts as the release number""" + + links = [link.Link] + """A Link that point to a specific version of the API""" + + @classmethod + def convert(cls, vid): + version = Version() + version.id = vid + version.links = [link.Link.make_link('self', pecan.request.host_url, + vid, '', bookmark=True)] + return version + + +class Root(base.APIBase): + + name = wtypes.text + """The name of the API""" + + description = wtypes.text + """Some information about this API""" + + versions = [Version] + """Links to all the versions available in this API""" + + default_version = Version + """A link to the default version of the API""" + + @staticmethod + def convert(): + root = Root() + root.name = "Inventory API" + root.description = ("Inventory is an OpenStack project which " + "provides REST API services for " + "system configuration.") + root.default_version = Version.convert(ID_VERSION) + root.versions = [root.default_version] + return root + + +class RootController(rest.RestController): + + _versions = [ID_VERSION] + """All supported API versions""" + + _default_version = ID_VERSION + """The default API version""" + + v1 = v1.Controller() + + @expose(Root) + def get(self): + # NOTE: The reason why convert() it's being called for every + # request is because we need to get the host url from + # the request object to make the links. + return Root.convert() + + @pecan.expose() + def _route(self, args, request=None): + """Overrides the default routing behavior. + + It redirects the request to the default version of the Inventory API + if the version number is not specified in the url. + """ + + if args[0] and args[0] not in self._versions: + args = [self._default_version] + args + return super(RootController, self)._route(args, request) diff --git a/inventory/inventory/inventory/api/controllers/v1/__init__.py b/inventory/inventory/inventory/api/controllers/v1/__init__.py new file mode 100644 index 00000000..74b752df --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/__init__.py @@ -0,0 +1,198 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import pecan +from pecan import rest + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import cpu +from inventory.api.controllers.v1 import ethernet_port +from inventory.api.controllers.v1 import host +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import lldp_agent +from inventory.api.controllers.v1 import lldp_neighbour +from inventory.api.controllers.v1 import memory +from inventory.api.controllers.v1 import node +from inventory.api.controllers.v1 import pci_device +from inventory.api.controllers.v1 import port +from inventory.api.controllers.v1 import sensor +from inventory.api.controllers.v1 import sensorgroup + +from inventory.api.controllers.v1 import system +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + + +class MediaType(base.APIBase): + """A media type representation.""" + + base = wtypes.text + type = wtypes.text + + def __init__(self, base, type): + self.base = base + self.type = type + + +class V1(base.APIBase): + """The representation of the version 1 of the API.""" + + id = wtypes.text + "The ID of the version, also acts as the release number" + + media_types = [MediaType] + "An array of supported media types for this version" + + links = [link.Link] + "Links that point to a specific URL for this version and documentation" + + systems = [link.Link] + "Links to the system resource" + + hosts = [link.Link] + "Links to the host resource" + + lldp_agents = [link.Link] + "Links to the lldp agents resource" + + lldp_neighbours = [link.Link] + "Links to the lldp neighbours resource" + + @classmethod + def convert(self): + v1 = V1() + v1.id = "v1" + v1.links = [link.Link.make_link('self', pecan.request.host_url, + 'v1', '', bookmark=True), + link.Link.make_link('describedby', + 'http://www.starlingx.io/', + 'developer/inventory/dev', + 'api-spec-v1.html', + bookmark=True, type='text/html') + ] + v1.media_types = [MediaType('application/json', + 'application/vnd.openstack.inventory.v1+json')] + + v1.systems = [link.Link.make_link('self', pecan.request.host_url, + 'systems', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'systems', '', + bookmark=True) + ] + + v1.hosts = [link.Link.make_link('self', pecan.request.host_url, + 'hosts', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', '', + bookmark=True) + ] + + v1.nodes = [link.Link.make_link('self', pecan.request.host_url, + 'nodes', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'nodes', '', + bookmark=True) + ] + + v1.cpus = [link.Link.make_link('self', pecan.request.host_url, + 'cpus', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'cpus', '', + bookmark=True) + ] + + v1.memory = [link.Link.make_link('self', pecan.request.host_url, + 'memory', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'memory', '', + bookmark=True) + ] + + v1.ports = [link.Link.make_link('self', + pecan.request.host_url, + 'ports', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'ports', '', + bookmark=True) + ] + + v1.ethernet_ports = [link.Link.make_link('self', + pecan.request.host_url, + 'ethernet_ports', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'ethernet_ports', '', + bookmark=True) + ] + + v1.lldp_agents = [link.Link.make_link('self', + pecan.request.host_url, + 'lldp_agents', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'lldp_agents', '', + bookmark=True) + ] + + v1.lldp_neighbours = [link.Link.make_link('self', + pecan.request.host_url, + 'lldp_neighbours', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'lldp_neighbours', '', + bookmark=True) + ] + + v1.sensors = [link.Link.make_link('self', + pecan.request.host_url, + 'sensors', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'sensors', '', + bookmark=True) + ] + + v1.sensorgroups = [link.Link.make_link('self', + pecan.request.host_url, + 'sensorgroups', ''), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'sensorgroups', '', + bookmark=True) + ] + + return v1 + + +class Controller(rest.RestController): + """Version 1 API controller root.""" + + systems = system.SystemController() + hosts = host.HostController() + nodes = node.NodeController() + cpus = cpu.CPUController() + memorys = memory.MemoryController() + ports = port.PortController() + ethernet_ports = ethernet_port.EthernetPortController() + lldp_agents = lldp_agent.LLDPAgentController() + lldp_neighbours = lldp_neighbour.LLDPNeighbourController() + pci_devices = pci_device.PCIDeviceController() + sensors = sensor.SensorController() + sensorgroups = sensorgroup.SensorGroupController() + + @wsme_pecan.wsexpose(V1) + def get(self): + return V1.convert() + + +__all__ = ('Controller',) diff --git a/inventory/inventory/inventory/api/controllers/v1/base.py b/inventory/inventory/inventory/api/controllers/v1/base.py new file mode 100644 index 00000000..f795e93a --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/base.py @@ -0,0 +1,130 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import datetime +import functools +from oslo_utils._i18n import _ +from webob import exc +import wsme +from wsme import types as wtypes + + +class APIBase(wtypes.Base): + + created_at = wsme.wsattr(datetime.datetime, readonly=True) + """The time in UTC at which the object is created""" + + updated_at = wsme.wsattr(datetime.datetime, readonly=True) + """The time in UTC at which the object is updated""" + + def as_dict(self): + """Render this object as a dict of its fields.""" + return dict((k, getattr(self, k)) + for k in self.fields + if hasattr(self, k) and + getattr(self, k) != wsme.Unset) + + def unset_fields_except(self, except_list=None): + """Unset fields so they don't appear in the message body. + + :param except_list: A list of fields that won't be touched. + + """ + if except_list is None: + except_list = [] + + for k in self.as_dict(): + if k not in except_list: + setattr(self, k, wsme.Unset) + + @classmethod + def from_rpc_object(cls, m, fields=None): + """Convert a RPC object to an API object.""" + obj_dict = m.as_dict() + # Unset non-required fields so they do not appear + # in the message body + obj_dict.update(dict((k, wsme.Unset) + for k in obj_dict.keys() + if fields and k not in fields)) + return cls(**obj_dict) + + +@functools.total_ordering +class Version(object): + """API Version object.""" + + string = 'X-OpenStack-Inventory-API-Version' + """HTTP Header string carrying the requested version""" + + min_string = 'X-OpenStack-Inventory-API-Minimum-Version' + """HTTP response header""" + + max_string = 'X-OpenStack-Inventory-API-Maximum-Version' + """HTTP response header""" + + def __init__(self, headers, default_version, latest_version): + """Create an API Version object from the supplied headers. + + :param headers: webob headers + :param default_version: version to use if not specified in headers + :param latest_version: version to use if latest is requested + :raises: webob.HTTPNotAcceptable + """ + (self.major, self.minor) = Version.parse_headers( + headers, default_version, latest_version) + + def __repr__(self): + return '%s.%s' % (self.major, self.minor) + + @staticmethod + def parse_headers(headers, default_version, latest_version): + """Determine the API version requested based on the headers supplied. + + :param headers: webob headers + :param default_version: version to use if not specified in headers + :param latest_version: version to use if latest is requested + :returns: a tupe of (major, minor) version numbers + :raises: webob.HTTPNotAcceptable + """ + version_str = headers.get(Version.string, default_version) + + if version_str.lower() == 'latest': + parse_str = latest_version + else: + parse_str = version_str + + try: + version = tuple(int(i) for i in parse_str.split('.')) + except ValueError: + version = () + + if len(version) != 2: + raise exc.HTTPNotAcceptable(_( + "Invalid value for %s header") % Version.string) + return version + + def __gt__(self, other): + return (self.major, self.minor) > (other.major, other.minor) + + def __eq__(self, other): + return (self.major, self.minor) == (other.major, other.minor) + + def __ne__(self, other): + return not self.__eq__(other) diff --git a/inventory/inventory/inventory/api/controllers/v1/collection.py b/inventory/inventory/inventory/api/controllers/v1/collection.py new file mode 100644 index 00000000..4c18fdaa --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/collection.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import pecan +from wsme import types as wtypes + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import link + + +class Collection(base.APIBase): + + next = wtypes.text + "A link to retrieve the next subset of the collection" + + @property + def collection(self): + return getattr(self, self._type) + + def has_next(self, limit): + """Return whether collection has more items.""" + return len(self.collection) and len(self.collection) == limit + + def get_next(self, limit, url=None, **kwargs): + """Return a link to the next subset of the collection.""" + if not self.has_next(limit): + return wtypes.Unset + + resource_url = url or self._type + q_args = ''.join(['%s=%s&' % (key, kwargs[key]) for key in kwargs]) + next_args = '?%(args)slimit=%(limit)d&marker=%(marker)s' % { + 'args': q_args, 'limit': limit, + 'marker': self.collection[-1].uuid} + + return link.Link.make_link('next', pecan.request.host_url, + resource_url, next_args).href diff --git a/inventory/inventory/inventory/api/controllers/v1/cpu.py b/inventory/inventory/inventory/api/controllers/v1/cpu.py new file mode 100644 index 00000000..df4cdfc2 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/cpu.py @@ -0,0 +1,303 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import six + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory import objects + +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class CPUPatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class CPU(base.APIBase): + """API representation of a host CPU. + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of a cpu. + """ + + uuid = types.uuid + "Unique UUID for this cpu" + + cpu = int + "Represent the cpu id cpu" + + core = int + "Represent the core id cpu" + + thread = int + "Represent the thread id cpu" + + cpu_family = wtypes.text + "Represent the cpu family of the cpu" + + cpu_model = wtypes.text + "Represent the cpu model of the cpu" + + function = wtypes.text + "Represent the function of the cpu" + + num_cores_on_processor0 = wtypes.text + "The number of cores on processors 0" + + num_cores_on_processor1 = wtypes.text + "The number of cores on processors 1" + + num_cores_on_processor2 = wtypes.text + "The number of cores on processors 2" + + num_cores_on_processor3 = wtypes.text + "The number of cores on processors 3" + + numa_node = int + "The numa node or zone the cpu. API only attribute" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "This cpu's meta data" + + host_id = int + "The hostid that this cpu belongs to" + + node_id = int + "The nodeId that this cpu belongs to" + + host_uuid = types.uuid + "The UUID of the host this cpu belongs to" + + node_uuid = types.uuid + "The UUID of the node this cpu belongs to" + + links = [link.Link] + "A list containing a self link and associated cpu links" + + def __init__(self, **kwargs): + self.fields = objects.CPU.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + # API only attributes + self.fields.append('function') + setattr(self, 'function', kwargs.get('function', None)) + self.fields.append('num_cores_on_processor0') + setattr(self, 'num_cores_on_processor0', + kwargs.get('num_cores_on_processor0', None)) + self.fields.append('num_cores_on_processor1') + setattr(self, 'num_cores_on_processor1', + kwargs.get('num_cores_on_processor1', None)) + self.fields.append('num_cores_on_processor2') + setattr(self, 'num_cores_on_processor2', + kwargs.get('num_cores_on_processor2', None)) + self.fields.append('num_cores_on_processor3') + setattr(self, 'num_cores_on_processor3', + kwargs.get('num_cores_on_processor3', None)) + + @classmethod + def convert_with_links(cls, rpc_port, expand=True): + cpu = CPU(**rpc_port.as_dict()) + if not expand: + cpu.unset_fields_except( + ['uuid', 'cpu', 'core', 'thread', + 'cpu_family', 'cpu_model', + 'numa_node', 'host_uuid', 'node_uuid', + 'host_id', 'node_id', + 'capabilities', + 'created_at', 'updated_at']) + + # never expose the id attribute + cpu.host_id = wtypes.Unset + cpu.node_id = wtypes.Unset + + cpu.links = [link.Link.make_link('self', pecan.request.host_url, + 'cpus', cpu.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'cpus', cpu.uuid, + bookmark=True) + ] + return cpu + + +class CPUCollection(collection.Collection): + """API representation of a collection of cpus.""" + + cpus = [CPU] + "A list containing cpu objects" + + def __init__(self, **kwargs): + self._type = 'cpus' + + @classmethod + def convert_with_links(cls, rpc_ports, limit, url=None, + expand=False, **kwargs): + collection = CPUCollection() + collection.cpus = [ + CPU.convert_with_links(p, expand) for p in rpc_ports] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class CPUController(rest.RestController): + """REST controller for cpus.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_node=False): + self._from_hosts = from_hosts + self._from_node = from_node + + def _get_cpus_collection(self, i_uuid, node_uuid, marker, + limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not i_uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + if self._from_node and not i_uuid: + raise exception.InvalidParameterValue(_( + "Node id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.CPU.get_by_uuid(pecan.request.context, + marker) + + if self._from_hosts: + # cpus = pecan.request.dbapi.cpu_get_by_host( + cpus = objects.CPU.get_by_host( + pecan.request.context, + i_uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + elif self._from_node: + # cpus = pecan.request.dbapi.cpu_get_by_node( + cpus = objects.CPU.get_by_node( + pecan.request.context, + i_uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if i_uuid and not node_uuid: + # cpus = pecan.request.dbapi.cpu_get_by_host( + cpus = objects.CPU.get_by_host( + pecan.request.context, + i_uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + elif i_uuid and node_uuid: + # cpus = pecan.request.dbapi.cpu_get_by_host_node( + cpus = objects.CPU.get_by_host_node( + pecan.request.context, + i_uuid, + node_uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + elif node_uuid: + # cpus = pecan.request.dbapi.cpu_get_by_host_node( + cpus = objects.CPU.get_by_node( + pecan.request.context, + i_uuid, + node_uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + else: + # cpus = pecan.request.dbapi.icpu_get_list( + cpus = objects.CPU.list( + pecan.request.context, + limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return CPUCollection.convert_with_links(cpus, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(CPUCollection, types.uuid, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, host_uuid=None, node_uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of cpus.""" + return self._get_cpus_collection(host_uuid, node_uuid, + marker, limit, + sort_key, sort_dir) + + @wsme_pecan.wsexpose(CPUCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, host_uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of cpus with detail.""" + # NOTE(lucasagomes): /detail should only work agaist collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "cpus": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['cpus', 'detail']) + return self._get_cpus_collection(host_uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(CPU, types.uuid) + def get_one(self, cpu_uuid): + """Retrieve information about the given cpu.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_port = objects.CPU.get_by_uuid(pecan.request.context, cpu_uuid) + return CPU.convert_with_links(rpc_port) diff --git a/inventory/inventory/inventory/api/controllers/v1/cpu_utils.py b/inventory/inventory/inventory/api/controllers/v1/cpu_utils.py new file mode 100644 index 00000000..b378ca64 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/cpu_utils.py @@ -0,0 +1,330 @@ +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import pecan + +from inventory.common import constants +from inventory.common import k_host +from oslo_log import log + +LOG = log.getLogger(__name__) + +CORE_FUNCTIONS = [ + constants.PLATFORM_FUNCTION, + constants.VSWITCH_FUNCTION, + constants.SHARED_FUNCTION, + constants.VM_FUNCTION, + constants.NO_FUNCTION +] + +VSWITCH_MIN_CORES = 1 +VSWITCH_MAX_CORES = 8 + + +class CpuProfile(object): + class CpuConfigure(object): + def __init__(self): + self.platform = 0 + self.vswitch = 0 + self.shared = 0 + self.vms = 0 + self.numa_node = 0 + + # cpus is a list of cpu sorted by numa_node, core and thread + # if not, provide a node list sorted by numa_node + # (id might not be reliable) + def __init__(self, cpus, nodes=None): + if nodes is not None: + cpus = CpuProfile.sort_cpu_by_numa_node(cpus, nodes) + cores = [] + + self.number_of_cpu = 0 + self.cores_per_cpu = 0 + self.hyper_thread = False + self.processors = [] + cur_processor = None + + for cpu in cpus: + key = '{0}-{1}'.format(cpu.numa_node, cpu.core) + if key not in cores: + cores.append(key) + else: + self.hyper_thread = True + continue + + if (cur_processor is None or + cur_processor.numa_node != cpu.numa_node): + cur_processor = CpuProfile.CpuConfigure() + cur_processor.numa_node = cpu.numa_node + self.processors.append(cur_processor) + + if cpu.allocated_function == constants.PLATFORM_FUNCTION: + cur_processor.platform += 1 + elif cpu.allocated_function == constants.VSWITCH_FUNCTION: + cur_processor.vswitch += 1 + elif cpu.allocated_function == constants.SHARED_FUNCTION: + cur_processor.shared += 1 + elif cpu.allocated_function == constants.VM_FUNCTION: + cur_processor.vms += 1 + + self.number_of_cpu = len(self.processors) + self.cores_per_cpu = len(cores) / self.number_of_cpu + + @staticmethod + def sort_cpu_by_numa_node(cpus, nodes): + newlist = [] + for node in nodes: + for cpu in cpus: + if cpu.node_id == node.id: + cpu.numa_node = node.numa_node + newlist.append(cpu) + return newlist + + +class HostCpuProfile(CpuProfile): + def __init__(self, subfunctions, cpus, nodes=None): + super(HostCpuProfile, self).__init__(cpus, nodes) + self.subfunctions = subfunctions + + # see if a cpu profile is applicable to this host + def profile_applicable(self, profile): + if self.number_of_cpu == profile.number_of_cpu and \ + self.cores_per_cpu == profile.cores_per_cpu: + return self.check_profile_core_functions(profile) + return False # Profile is not applicable to host + + def check_profile_core_functions(self, profile): + platform_cores = 0 + vswitch_cores = 0 + shared_cores = 0 + vm_cores = 0 + for cpu in profile.processors: + platform_cores += cpu.platform + vswitch_cores += cpu.vswitch + shared_cores += cpu.shared + vm_cores += cpu.vms + + error_string = "" + if platform_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.PLATFORM_FUNCTION + elif k_host.COMPUTE in self.subfunctions and vswitch_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.VSWITCH_FUNCTION + elif k_host.COMPUTE in self.subfunctions and vm_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.VM_FUNCTION + return error_string + + +def lookup_function(s): + for f in CORE_FUNCTIONS: + if s.lower() == f.lower(): + return f + return s + + +def check_profile_core_functions(personality, profile): + + platform_cores = 0 + vswitch_cores = 0 + shared_cores = 0 + vm_cores = 0 + for cpu in profile.processors: + platform_cores += cpu.platform + vswitch_cores += cpu.vswitch + shared_cores += cpu.shared + vm_cores += cpu.vms + + error_string = "" + if platform_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.PLATFORM_FUNCTION + elif k_host.COMPUTE in personality and vswitch_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.VSWITCH_FUNCTION + elif k_host.COMPUTE in personality and vm_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.VM_FUNCTION + return error_string + + +def check_core_functions(personality, icpus): + platform_cores = 0 + vswitch_cores = 0 + shared_cores = 0 + vm_cores = 0 + for cpu in icpus: + allocated_function = cpu.allocated_function + if allocated_function == constants.PLATFORM_FUNCTION: + platform_cores += 1 + elif allocated_function == constants.VSWITCH_FUNCTION: + vswitch_cores += 1 + elif allocated_function == constants.SHARED_FUNCTION: + shared_cores += 1 + elif allocated_function == constants.VM_FUNCTION: + vm_cores += 1 + + error_string = "" + if platform_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.PLATFORM_FUNCTION + elif k_host.COMPUTE in personality and vswitch_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.VSWITCH_FUNCTION + elif k_host.COMPUTE in personality and vm_cores == 0: + error_string = "There must be at least one core for %s." % \ + constants.VM_FUNCTION + return error_string + + +def get_default_function(host): + """Return the default function to be assigned to cpus on this host""" + if k_host.COMPUTE in host.subfunctions: + return constants.VM_FUNCTION + return constants.PLATFORM_FUNCTION + + +def get_cpu_function(host, cpu): + """Return the function that is assigned to the specified cpu""" + for s in range(0, len(host.nodes)): + functions = host.cpu_functions[s] + for f in CORE_FUNCTIONS: + if cpu.cpu in functions[f]: + return f + return constants.NO_FUNCTION + + +def get_cpu_counts(host): + """Return the CPU counts for this host by socket and function.""" + counts = {} + for s in range(0, len(host.nodes)): + counts[s] = {} + for f in CORE_FUNCTIONS: + counts[s][f] = len(host.cpu_functions[s][f]) + return counts + + +def init_cpu_counts(host): + """Create empty data structures to track CPU assignments by socket and + function. + """ + host.cpu_functions = {} + host.cpu_lists = {} + for s in range(0, len(host.nodes)): + host.cpu_functions[s] = {} + for f in CORE_FUNCTIONS: + host.cpu_functions[s][f] = [] + host.cpu_lists[s] = [] + + +def _sort_by_coreid(cpu): + """Sort a list of cpu database objects such that threads of the same core + are adjacent in the list with the lowest thread number appearing first. + """ + return (int(cpu.core), int(cpu.thread)) + + +def restructure_host_cpu_data(host): + """Reorganize the cpu list by socket and function so that it can more + easily be consumed by other utilities. + """ + init_cpu_counts(host) + host.sockets = len(host.nodes or []) + host.hyperthreading = False + host.physical_cores = 0 + if not host.cpus: + return + host.cpu_model = host.cpus[0].cpu_model + cpu_list = sorted(host.cpus, key=_sort_by_coreid) + for cpu in cpu_list: + inode = pecan.request.dbapi.inode_get(inode_id=cpu.node_id) + cpu.numa_node = inode.numa_node + if cpu.thread == 0: + host.physical_cores += 1 + elif cpu.thread > 0: + host.hyperthreading = True + function = cpu.allocated_function or get_default_function(host) + host.cpu_functions[cpu.numa_node][function].append(int(cpu.cpu)) + host.cpu_lists[cpu.numa_node].append(int(cpu.cpu)) + + +def check_core_allocations(host, cpu_counts, func): + """Check that minimum and maximum core values are respected.""" + total_platform_cores = 0 + total_vswitch_cores = 0 + total_shared_cores = 0 + for s in range(0, len(host.nodes)): + available_cores = len(host.cpu_lists[s]) + platform_cores = cpu_counts[s][constants.PLATFORM_FUNCTION] + vswitch_cores = cpu_counts[s][constants.VSWITCH_FUNCTION] + shared_cores = cpu_counts[s][constants.SHARED_FUNCTION] + requested_cores = platform_cores + vswitch_cores + shared_cores + if requested_cores > available_cores: + return ("More total logical cores requested than present on " + "'Processor %s' (%s cores)." % (s, available_cores)) + total_platform_cores += platform_cores + total_vswitch_cores += vswitch_cores + total_shared_cores += shared_cores + if func.lower() == constants.PLATFORM_FUNCTION.lower(): + if ((k_host.CONTROLLER in host.subfunctions) and + (k_host.COMPUTE in host.subfunctions)): + if total_platform_cores < 2: + return "%s must have at least two cores." % \ + constants.PLATFORM_FUNCTION + elif total_platform_cores == 0: + return "%s must have at least one core." % \ + constants.PLATFORM_FUNCTION + if k_host.COMPUTE in (host.subfunctions or host.personality): + if func.lower() == constants.VSWITCH_FUNCTION.lower(): + if host.hyperthreading: + total_physical_cores = total_vswitch_cores / 2 + else: + total_physical_cores = total_vswitch_cores + if total_physical_cores < VSWITCH_MIN_CORES: + return ("The %s function must have at least %s core(s)." % + (constants.VSWITCH_FUNCTION.lower(), + VSWITCH_MIN_CORES)) + elif total_physical_cores > VSWITCH_MAX_CORES: + return ("The %s function can only be assigned up to %s cores." + % (constants.VSWITCH_FUNCTION.lower(), + VSWITCH_MAX_CORES)) + reserved_for_vms = \ + len(host.cpus) - total_platform_cores - total_vswitch_cores + if reserved_for_vms <= 0: + return "There must be at least one unused core for %s." % \ + constants. VM_FUNCTION + else: + if total_platform_cores != len(host.cpus): + return "All logical cores must be reserved for platform use" + return "" + + +def update_core_allocations(host, cpu_counts): + """Update the per socket/function cpu list based on the newly requested + counts. + """ + # Remove any previous assignments + for s in range(0, len(host.nodes)): + for f in CORE_FUNCTIONS: + host.cpu_functions[s][f] = [] + # Set new assignments + for s in range(0, len(host.nodes)): + cpu_list = host.cpu_lists[s] if s in host.cpu_lists else [] + # Reserve for the platform first + for i in range(0, cpu_counts[s][constants.PLATFORM_FUNCTION]): + host.cpu_functions[s][constants.PLATFORM_FUNCTION].append( + cpu_list.pop(0)) + # Reserve for the vswitch next + for i in range(0, cpu_counts[s][constants.VSWITCH_FUNCTION]): + host.cpu_functions[s][constants.VSWITCH_FUNCTION].append( + cpu_list.pop(0)) + # Reserve for the shared next + for i in range(0, cpu_counts[s][constants.SHARED_FUNCTION]): + host.cpu_functions[s][constants.SHARED_FUNCTION].append( + cpu_list.pop(0)) + # Assign the remaining cpus to the default function for this host + host.cpu_functions[s][get_default_function(host)] += cpu_list + return diff --git a/inventory/inventory/inventory/api/controllers/v1/ethernet_port.py b/inventory/inventory/inventory/api/controllers/v1/ethernet_port.py new file mode 100644 index 00000000..f397923e --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/ethernet_port.py @@ -0,0 +1,310 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import six + +import pecan +from pecan import rest + +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory import objects + +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class EthernetPortPatchType(types.JsonPatchType): + @staticmethod + def mandatory_attrs(): + return [] + + +class EthernetPort(base.APIBase): + """API representation of an Ethernet port + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + Ethernet port. + """ + + uuid = types.uuid + "Unique UUID for this port" + + type = wtypes.text + "Represent the type of port" + + name = wtypes.text + "Represent the name of the port. Unique per host" + + namedisplay = wtypes.text + "Represent the display name of the port. Unique per host" + + pciaddr = wtypes.text + "Represent the pci address of the port" + + dev_id = int + "The unique identifier of PCI device" + + pclass = wtypes.text + "Represent the pci class of the port" + + pvendor = wtypes.text + "Represent the pci vendor of the port" + + pdevice = wtypes.text + "Represent the pci device of the port" + + psvendor = wtypes.text + "Represent the pci svendor of the port" + + psdevice = wtypes.text + "Represent the pci sdevice of the port" + + numa_node = int + "Represent the numa node or zone sdevice of the port" + + sriov_totalvfs = int + "The total number of available SR-IOV VFs" + + sriov_numvfs = int + "The number of configured SR-IOV VFs" + + sriov_vfs_pci_address = wtypes.text + "The PCI Addresses of the VFs" + + driver = wtypes.text + "The kernel driver for this device" + + mac = wsme.wsattr(types.macaddress, mandatory=False) + "Represent the MAC Address of the port" + + mtu = int + "Represent the MTU size (bytes) of the port" + + speed = int + "Represent the speed (MBytes/sec) of the port" + + link_mode = int + "Represent the link mode of the port" + + duplex = wtypes.text + "Represent the duplex mode of the port" + + autoneg = wtypes.text + "Represent the auto-negotiation mode of the port" + + bootp = wtypes.text + "Represent the bootp port of the host" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "Represent meta data of the port" + + host_id = int + "Represent the host_id the port belongs to" + + bootif = wtypes.text + "Represent whether the port is a boot port" + + dpdksupport = bool + "Represent whether or not the port supports DPDK acceleration" + + host_uuid = types.uuid + "Represent the UUID of the host the port belongs to" + + node_uuid = types.uuid + "Represent the UUID of the node the port belongs to" + + links = [link.Link] + "Represent a list containing a self link and associated port links" + + def __init__(self, **kwargs): + self.fields = objects.EthernetPort.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_port, expand=True): + port = EthernetPort(**rpc_port.as_dict()) + if not expand: + port.unset_fields_except(['uuid', 'host_id', 'node_id', + 'type', 'name', + 'namedisplay', 'pciaddr', 'dev_id', + 'pclass', 'pvendor', 'pdevice', + 'psvendor', 'psdevice', 'numa_node', + 'mac', 'sriov_totalvfs', 'sriov_numvfs', + 'sriov_vfs_pci_address', 'driver', + 'mtu', 'speed', 'link_mode', + 'duplex', 'autoneg', 'bootp', + 'capabilities', + 'host_uuid', + 'node_uuid', 'dpdksupport', + 'created_at', 'updated_at']) + + # never expose the id attribute + port.host_id = wtypes.Unset + port.node_id = wtypes.Unset + + port.links = [link.Link.make_link('self', pecan.request.host_url, + 'ethernet_ports', port.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'ethernet_ports', port.uuid, + bookmark=True) + ] + return port + + +class EthernetPortCollection(collection.Collection): + """API representation of a collection of EthernetPort objects.""" + + ethernet_ports = [EthernetPort] + "A list containing EthernetPort objects" + + def __init__(self, **kwargs): + self._type = 'ethernet_ports' + + @classmethod + def convert_with_links(cls, rpc_ports, limit, url=None, + expand=False, **kwargs): + collection = EthernetPortCollection() + collection.ethernet_ports = [EthernetPort.convert_with_links(p, expand) + for p in rpc_ports] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'EthernetPortController' + + +class EthernetPortController(rest.RestController): + """REST controller for EthernetPorts.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_node=False): + self._from_hosts = from_hosts + self._from_node = from_node + + def _get_ports_collection(self, uuid, node_uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + if self._from_node and not uuid: + raise exception.InvalidParameterValue(_( + "node id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.EthernetPort.get_by_uuid( + pecan.request.context, + marker) + + if self._from_hosts: + ports = objects.EthernetPort.get_by_host( + pecan.request.context, + uuid, limit, + marker=marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + elif self._from_node: + ports = objects.EthernetPort.get_by_numa_node( + pecan.request.context, + uuid, limit, + marker=marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if uuid: + ports = objects.EthernetPort.get_by_host( + pecan.request.context, + uuid, limit, + marker=marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + ports = objects.EthernetPort.list( + pecan.request.context, + limit, marker=marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return EthernetPortCollection.convert_with_links( + ports, limit, url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(EthernetPortCollection, types.uuid, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, node_uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of ports.""" + + return self._get_ports_collection(uuid, + node_uuid, + marker, limit, sort_key, sort_dir) + + @wsme_pecan.wsexpose(EthernetPortCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of ports with detail.""" + + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "ethernet_ports": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['ethernet_ports', 'detail']) + return self._get_ports_collection(uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(EthernetPort, types.uuid) + def get_one(self, port_uuid): + """Retrieve information about the given port.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_port = objects.EthernetPort.get_by_uuid( + pecan.request.context, port_uuid) + return EthernetPort.convert_with_links(rpc_port) diff --git a/inventory/inventory/inventory/api/controllers/v1/host.py b/inventory/inventory/inventory/api/controllers/v1/host.py new file mode 100644 index 00000000..c262dce7 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/host.py @@ -0,0 +1,3585 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import ast +import cgi +from configutilities import HOST_XML_ATTRIBUTES +import copy +from fm_api import constants as fm_constants +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import cpu as cpu_api +# TODO(LK) from inventory.api.controllers.v1 import disk +# TODO(LK) from inventory.api.controllers.v1 import partition +from inventory.api.controllers.v1 import ethernet_port +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import lldp_agent +from inventory.api.controllers.v1 import lldp_neighbour +from inventory.api.controllers.v1 import memory +from inventory.api.controllers.v1 import node as node_api +from inventory.api.controllers.v1 import pci_device +from inventory.api.controllers.v1 import port +from inventory.api.controllers.v1.query import Query +from inventory.api.controllers.v1 import sensor as sensor_api +from inventory.api.controllers.v1 import sensorgroup +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import ceph +from inventory.common import constants +from inventory.common import exception +from inventory.common import health +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import mtce_api +from inventory.common import patch_api +from inventory.common import sm_api +from inventory.common.storage_backend_conf import StorageBackendConfig +# TODO(sc) to be removed StorageBackendConfig +from inventory.common import utils as cutils +from inventory.common import vim_api +from inventory import objects +import json +import jsonpatch +from oslo_log import log +from oslo_utils import uuidutils +import pecan +from pecan import expose +from pecan import rest +import psutil +import re +import six +from six import text_type as unicode +import tsconfig.tsconfig as tsc +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan +from xml.dom import minidom as dom +import xml.etree.ElementTree as ET +import xml.etree.ElementTree as et + +LOG = log.getLogger(__name__) +KEYRING_BM_SERVICE = "BM" +ERR_CODE_LOCK_SOLE_SERVICE_PROVIDER = "-1003" + + +class Host(base.APIBase): + """API representation of a host. + + This class enforces type checking and value constraints, and + converts between the internal object model and + the API representation of a host. + """ + + id = int + + uuid = wtypes.text + + hostname = wtypes.text + "Represent the hostname of the host" + + invprovision = wtypes.text + "Represent the current provision state of the host" + + administrative = wtypes.text + "Represent the administrative state of the host" + + operational = wtypes.text + "Represent the operational state of the host" + + availability = wtypes.text + "Represent the availability status of the host" + + mgmt_mac = wtypes.text + "Represent the boot mgmt MAC address of the host." + + mgmt_ip = wtypes.text + "Represent the boot mgmt IP address of the host." + + infra_ip = wtypes.text + "Represent the infrastructure IP address of the host." + + bm_ip = wtypes.text + "Represent the board management IP address of the host." + + bm_type = wtypes.text + "Represent the board management type of the host." + + bm_username = wtypes.text + "Represent the board management username of the host." + + bm_password = wtypes.text + "Represent the board management password of the host." + + personality = wtypes.text + "Represent the personality of the host" + + subfunctions = wtypes.text + "Represent the subfunctions of the host" + + subfunction_oper = wtypes.text + "Represent the subfunction operational state of the host" + + subfunction_avail = wtypes.text + "Represent the subfunction availability status of the host" + + serialid = wtypes.text + "Represent the serial id of the host" + + action = wtypes.text + 'Represent the action on the host' + + host_action = wtypes.text + 'Represent the current action task in progress' + + vim_progress_status = wtypes.text + 'Represent the vim progress status' + + task = wtypes.text + "Represent the mtce task state" + + mtce_info = wtypes.text + "Represent the mtce info" + + uptime = int + "Represent the uptime, in seconds, of the host." + + location = {wtypes.text: utils.ValidTypes(wtypes.text, six.integer_types)} + "Represent the location of the host" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "Represent the capabilities of the host" + + system_uuid = types.uuid + "The UUID of the system this host belongs to" + + boot_device = wtypes.text + "Represent the boot device of the host" + + rootfs_device = wtypes.text + "Represent the rootfs device of the host" + + install_output = wtypes.text + "Represent the install_output of the host" + + console = wtypes.text + "Represent the console of the host" + + tboot = wtypes.text + "Represent the tboot of the host" + + ttys_dcd = wtypes.text + "Enable or disable serial console carrier detect" + + install_state = wtypes.text + "Represent the install state" + + install_state_info = wtypes.text + "Represent install state extra information if there is any" + + iscsi_initiator_name = wtypes.text + "The iscsi initiator name (only used for compute hosts)" + + links = [link.Link] + "A list containing a self link and associated host links" + + ports = [link.Link] + "Links to the collection of Ports on this host" + + ethernet_ports = [link.Link] + "Links to the collection of EthernetPorts on this host" + + nodes = [link.Link] + "Links to the collection of nodes on this host" + + cpus = [link.Link] + "Links to the collection of cpus on this host" + + memorys = [link.Link] + "Links to the collection of memorys on this host" + + # idisks = [link.Link] + # "Links to the collection of idisks on this ihost" + + sensors = [link.Link] + "Links to the collection of sensors on this host" + + sensorgroups = [link.Link] + "Links to the collection of sensorgruops on this host" + + pci_devices = [link.Link] + "Links to the collection of pci_devices on this host" + + lldp_agents = [link.Link] + "Links to the collection of LldpAgents on this ihost" + + lldp_neighbours = [link.Link] + "Links to the collection of LldpNeighbours on this ihost" + + def __init__(self, **kwargs): + self.fields = objects.Host.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_ihost, expand=True): + minimum_fields = [ + 'id', 'uuid', 'hostname', + 'personality', 'subfunctions', + 'subfunction_oper', 'subfunction_avail', + 'administrative', 'operational', 'availability', + 'invprovision', 'task', 'mtce_info', 'action', 'uptime', + 'host_action', 'mgmt_mac', 'mgmt_ip', 'infra_ip', 'location', + 'bm_ip', 'bm_type', 'bm_username', + 'system_uuid', 'capabilities', 'serialid', + 'created_at', 'updated_at', 'boot_device', + 'rootfs_device', 'install_output', 'console', + 'tboot', 'ttys_dcd', + 'install_state', 'install_state_info', + 'iscsi_initiator_name'] + + fields = minimum_fields if not expand else None + uhost = Host.from_rpc_object(rpc_ihost, fields) + uhost.links = [link.Link.make_link('self', pecan.request.host_url, + 'hosts', uhost.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', uhost.uuid, + bookmark=True) + ] + if expand: + uhost.ports = [link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/ports"), + link.Link.make_link( + 'bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/ports", + bookmark=True) + ] + uhost.ethernet_ports = [ + link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/ethernet_ports"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/ethernet_ports", + bookmark=True) + ] + uhost.nodes = [link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/nodes"), + link.Link.make_link( + 'bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/nodes", + bookmark=True) + ] + uhost.cpus = [link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/cpus"), + link.Link.make_link( + 'bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/cpus", + bookmark=True) + ] + + uhost.memorys = [link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/memorys"), + link.Link.make_link( + 'bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/memorys", + bookmark=True) + ] + + uhost.disks = [link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/disks"), + link.Link.make_link( + 'bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/disks", + bookmark=True) + ] + + uhost.sensors = [link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/sensors"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/sensors", + bookmark=True) + ] + + uhost.sensorgroups = [ + link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/sensorgroups"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/sensorgroups", + bookmark=True) + ] + + uhost.pci_devices = [ + link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/pci_devices"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/pci_devices", + bookmark=True) + ] + + uhost.lldp_agents = [ + link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/lldp_agents"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/lldp_agents", + bookmark=True) + ] + + uhost.lldp_neighbours = [ + link.Link.make_link('self', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/lldp_neighbors"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'hosts', + uhost.uuid + "/lldp_neighbors", + bookmark=True) + ] + + return uhost + + +class HostCollection(collection.Collection): + """API representation of a collection of hosts.""" + + hosts = [Host] + "A list containing hosts objects" + + def __init__(self, **kwargs): + self._type = 'hosts' + + @classmethod + def convert_with_links(cls, ihosts, limit, url=None, + expand=False, **kwargs): + collection = HostCollection() + collection.hosts = [ + Host.convert_with_links(n, expand) for n in ihosts] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class HostUpdate(object): + """Host update helper class. + """ + + CONTINUE = "continue" + EXIT_RETURN_HOST = "exit_return_host" + EXIT_UPDATE_PREVAL = "exit_update_preval" + FAILED = "failed" + PASSED = "passed" + + ACTIONS_TO_TASK_DISPLAY_CHOICES = ( + (None, ""), + ("", ""), + (k_host.ACTION_UNLOCK, _("Unlocking")), + (k_host.ACTION_FORCE_UNLOCK, _("Force Unlocking")), + (k_host.ACTION_LOCK, _("Locking")), + (k_host.ACTION_FORCE_LOCK, _("Force Locking")), + (k_host.ACTION_RESET, _("Resetting")), + (k_host.ACTION_REBOOT, _("Rebooting")), + (k_host.ACTION_REINSTALL, _("Reinstalling")), + (k_host.ACTION_POWERON, _("Powering-on")), + (k_host.ACTION_POWEROFF, _("Powering-off")), + (k_host.ACTION_SWACT, _("Swacting")), + (k_host.ACTION_FORCE_SWACT, _("Force-Swacting")), + ) + + def __init__(self, host_orig, host_patch, delta): + self.ihost_orig = dict(host_orig) + self.ihost_patch = dict(host_patch) + self._delta = list(delta) + self._ihost_val_prenotify = {} + self._ihost_val = {} + + self._configure_required = False + self._notify_vim = False + self._notify_mtce = False + self._notify_availability = None + self._notify_vim_add_host = False + self._notify_action_lock = False + self._notify_action_lock_force = False + self._skip_notify_mtce = False + self._bm_type_changed_to_none = False + self._nextstep = self.CONTINUE + + self._action = None + self.displayid = host_patch.get('hostname') + if not self.displayid: + self.displayid = host_patch.get('uuid') + + LOG.debug("host_orig=%s, host_patch=%s, delta=%s" % + (self.ihost_orig, self.ihost_patch, self.delta)) + + @property + def action(self): + return self._action + + @action.setter + def action(self, val): + self._action = val + + @property + def delta(self): + return self._delta + + @property + def nextstep(self): + return self._nextstep + + @nextstep.setter + def nextstep(self, val): + self._nextstep = val + + @property + def configure_required(self): + return self._configure_required + + @configure_required.setter + def configure_required(self, val): + self._configure_required = val + + @property + def bm_type_changed_to_none(self): + return self._bm_type_changed_to_none + + @bm_type_changed_to_none.setter + def bm_type_changed_to_none(self, val): + self._bm_type_changed_to_none = val + + @property + def notify_vim_add_host(self): + return self._notify_vim_add_host + + @notify_vim_add_host.setter + def notify_vim_add_host(self, val): + self._notify_vim_add_host = val + + @property + def skip_notify_mtce(self): + return self._skip_notify_mtce + + @skip_notify_mtce.setter + def skip_notify_mtce(self, val): + self._skip_notify_mtce = val + + @property + def notify_action_lock(self): + return self._notify_action_lock + + @notify_action_lock.setter + def notify_action_lock(self, val): + self._notify_action_lock = val + + @property + def notify_action_lock_force(self): + return self._notify_action_lock_force + + @notify_action_lock_force.setter + def notify_action_lock_force(self, val): + self._notify_action_lock_force = val + + @property + def ihost_val_prenotify(self): + return self._ihost_val_prenotify + + def ihost_val_prenotify_update(self, val): + self._ihost_val_prenotify.update(val) + + @property + def ihost_val(self): + return self._ihost_val + + def ihost_val_update(self, val): + self._ihost_val.update(val) + + @property + def notify_vim(self): + return self._notify_vim + + @notify_vim.setter + def notify_vim(self, val): + self._notify_vim = val + + @property + def notify_mtce(self): + return self._notify_mtce + + @notify_mtce.setter + def notify_mtce(self, val): + self._notify_mtce = val + + @property + def notify_availability(self): + return self._notify_availability + + @notify_availability.setter + def notify_availability(self, val): + self._notify_availability = val + + def get_task_from_action(self, action): + """Lookup the task value in the action to task dictionary.""" + + display_choices = self.ACTIONS_TO_TASK_DISPLAY_CHOICES + + display_value = [display for (value, display) in display_choices + if value and value.lower() == (action or '').lower()] + + if display_value: + return display_value[0] + return None + + +LOCK_NAME = 'HostController' +LOCK_NAME_SYS = 'HostControllerSys' + + +class HostController(rest.RestController): + """REST controller for hosts.""" + + ports = port.PortController( + from_hosts=True) + "Expose ports as a sub-element of hosts" + + ethernet_ports = ethernet_port.EthernetPortController( + from_hosts=True) + "Expose ethernet_ports as a sub-element of hosts" + + nodes = node_api.NodeController(from_hosts=True) + "Expose nodes as a sub-element of hosts" + + cpus = cpu_api.CPUController(from_hosts=True) + "Expose cpus as a sub-element of hosts" + + memorys = memory.MemoryController(from_hosts=True) + "Expose memorys as a sub-element of hosts" + + # TODO(LK) idisks = disk.DiskController(from_hosts=True) + # "Expose idisks as a sub-element of hosts" + + sensors = sensor_api.SensorController(from_hosts=True) + "Expose sensors as a sub-element of hosts" + + sensorgroups = sensorgroup.SensorGroupController(from_hosts=True) + "Expose sensorgroups as a sub-element of hosts" + + pci_devices = pci_device.PCIDeviceController(from_hosts=True) + "Expose pci_devices as a sub-element of hosts" + + lldp_agents = lldp_agent.LLDPAgentController( + from_hosts=True) + "Expose lldp_agents as a sub-element of hosts" + + lldp_neighbours = lldp_neighbour.LLDPNeighbourController( + from_hosts=True) + "Expose lldp_neighbours as a sub-element of hosts" + + _custom_actions = { + 'detail': ['GET'], + 'bulk_add': ['POST'], + 'bulk_export': ['GET'], + 'install_progress': ['POST'], + } + + def __init__(self, from_system=False): + self._from_system = from_system + self._mtc_address = k_host.LOCALHOST_HOSTNAME + self._mtc_port = 2112 + self._ceph = ceph.CephApiOperator() + self._api_token = None + + def _ihosts_get(self, isystem_id, marker, limit, personality, + sort_key, sort_dir, q=None): + if self._from_system and not isystem_id: + raise exception.InvalidParameterValue(_( + "System id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + filters = {} + if q is not None: + for i in q: + if i.op == 'eq': + filters[i.field] = i.value + + marker_obj = None + if marker: + marker_obj = objects.Host.get_by_uuid(pecan.request.context, + marker) + + if isystem_id: + ihosts = pecan.request.dbapi.host_get_by_isystem( + isystem_id, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if personality: + ihosts = objects.Host.list( + pecan.request.context, + limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir, + filters={'personality': personality}) + else: + ihosts = objects.Host.list( + pecan.request.context, + limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir, + filters=filters) + + for h in ihosts: + self._update_controller_personality(h) + + return ihosts + + @staticmethod + def _get_controller_address(hostname): + networktype = constants.NETWORK_TYPE_MGMT + name = '%s-%s' % (hostname, networktype) + address = pecan.request.systemconfig.address_get_by_name(name) + LOG.info("systemconfig _get_controller_address=%s" % address) + return address + + @staticmethod + def _get_storage_address(hostname): + networktype = constants.NETWORK_TYPE_MGMT + name = '%s-%s' % (hostname, networktype) + return pecan.request.systemconfig.address_get_by_name(name) + + @staticmethod + def _update_subfunctions(ihost): + subfunctions = ihost.get('subfunctions') or "" + personality = ihost.get('personality') or "" + # handle race condition with subfunctions being updated late. + if not subfunctions: + LOG.info("update_subfunctions: subfunctions not set. " + "personality=%s" % personality) + if personality == k_host.CONTROLLER: + subfunctions = ','.join(tsc.subfunctions) + else: + subfunctions = personality + ihost['subfunctions'] = subfunctions + + subfunctions_set = set(subfunctions.split(',')) + if personality not in subfunctions_set: + # Automatically add it + subfunctions_list = list(subfunctions_set) + subfunctions_list.insert(0, personality) + subfunctions = ','.join(subfunctions_list) + LOG.info("%s personality=%s update subfunctions=%s" % + (ihost.get('hostname'), personality, subfunctions)) + LOG.debug("update_subfunctions: personality=%s subfunctions=%s" % + (personality, subfunctions)) + return subfunctions + + @staticmethod + def _update_controller_personality(host): + if host['personality'] == k_host.CONTROLLER: + if utils.is_host_active_controller(host): + activity = 'Controller-Active' + else: + activity = 'Controller-Standby' + host['capabilities'].update({'Personality': activity}) + + @wsme_pecan.wsexpose(HostCollection, [Query], unicode, unicode, int, + unicode, unicode, unicode) + def get_all(self, q=[], isystem_id=None, marker=None, limit=None, + personality=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of hosts.""" + ihosts = self._ihosts_get( + isystem_id, marker, limit, personality, sort_key, sort_dir, q=q) + return HostCollection.convert_with_links(ihosts, limit, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(unicode, unicode, body=unicode) + def install_progress(self, uuid, install_state, + install_state_info=None): + """Update the install status for the given host.""" + LOG.debug("Update host uuid %s with install_state=%s " + "and install_state_info=%s" % + (uuid, install_state, install_state_info)) + if install_state == constants.INSTALL_STATE_INSTALLED: + # After an install a node will reboot right away. Change the state + # to reflect this. + install_state = constants.INSTALL_STATE_BOOTING + + host = objects.Host.get_by_uuid(pecan.request.context, uuid) + pecan.request.dbapi.host_update(host['uuid'], + {'install_state': install_state, + 'install_state_info': + install_state_info}) + + @wsme_pecan.wsexpose(HostCollection, unicode, unicode, int, unicode, + unicode, unicode) + def detail(self, isystem_id=None, marker=None, limit=None, + personality=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of hosts with detail.""" + # /detail should only work against collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "hosts": + raise exception.HTTPNotFound + + ihosts = self._ihosts_get( + isystem_id, marker, limit, personality, sort_key, sort_dir) + resource_url = '/'.join(['hosts', 'detail']) + return HostCollection.convert_with_links(ihosts, limit, + url=resource_url, + expand=True, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(Host, unicode) + def get_one(self, uuid): + """Retrieve information about the given ihost.""" + if self._from_system: + raise exception.OperationNotPermitted + + rpc_ihost = objects.Host.get_by_uuid(pecan.request.context, uuid) + self._update_controller_personality(rpc_ihost) + + return Host.convert_with_links(rpc_ihost) + + def _add_host_semantic_checks(self, ihost_dict): + chosts = self._get_controllers() + if chosts and ihost_dict.get('personality') is None: + # Prevent adding any new host(s) until there is + # an unlocked-enabled controller to manage them. + for c in chosts: + if (c.administrative == k_host.ADMIN_UNLOCKED and + c.operational == k_host.OPERATIONAL_ENABLED): + break + else: + raise wsme.exc.ClientSideError( + _("Provisioning request for new host '%s' is not permitted" + " while there is no unlocked-enabled controller. Unlock " + "controller-0, wait for it to enable and then retry.") % + ihost_dict.get('mgmt_mac')) + + def _new_host_semantic_checks(self, ihost_dict): + + if self._get_controllers(): + self._add_host_semantic_checks(ihost_dict) + + mgmt_network = pecan.request.systemconfig.network_get_by_type( + constants.NETWORK_TYPE_MGMT) + LOG.info("systemconfig mgmt_network={}".format(mgmt_network)) + + if mgmt_network.dynamic and ihost_dict.get('mgmt_ip'): + # raise wsme.exc.ClientSideError(_( + LOG.info(_( + "Host-add Allowed: Specifying a mgmt_ip when dynamic " + "address allocation is configured")) + elif (not mgmt_network.dynamic and + not ihost_dict.get('mgmt_ip') and + ihost_dict.get('personality') not in + [k_host.STORAGE, k_host.CONTROLLER]): + raise wsme.exc.ClientSideError(_( + "Host-add Rejected: Cannot add a compute host without " + "specifying a mgmt_ip when static address allocation is " + "configured.")) + + # Check whether the system mode is simplex + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError(_( + "Host-add Rejected: Adding a host on a simplex system " + "is not allowed.")) + + personality = ihost_dict['personality'] + if not ihost_dict['hostname']: + if personality not in (k_host.CONTROLLER, k_host.STORAGE): + raise wsme.exc.ClientSideError(_( + "Host-add Rejected. Must provide a hostname for a node of " + "personality %s") % personality) + else: + self._validate_hostname(ihost_dict['hostname'], personality) + + HostController._personality_license_check(personality) + + def _do_post(self, ihost_dict): + """Create a new ihost based off a dictionary of attributes """ + + log_start = cutils.timestamped("ihost_post_start") + LOG.info("SYS_I host %s %s add" % (ihost_dict['hostname'], + log_start)) + + power_on = ihost_dict.get('power_on', None) + + ihost_obj = None + + # Semantic checks for adding a new node + if self._from_system: + raise exception.OperationNotPermitted + + self._new_host_semantic_checks(ihost_dict) + current_ihosts = objects.Host.list(pecan.request.context) + + # Check for missing/invalid hostname + # ips/hostnames are automatic for controller & storage nodes + if ihost_dict['personality'] not in (k_host.CONTROLLER, + k_host.STORAGE): + host_names = [h.hostname for h in current_ihosts] + if ihost_dict['hostname'] in host_names: + raise wsme.exc.ClientSideError( + _("Host-add Rejected: Hostname already exists")) + host_ips = [h.mgmt_ip for h in current_ihosts] + if (ihost_dict.get('mgmt_ip') and + ihost_dict['mgmt_ip'] in host_ips): + raise wsme.exc.ClientSideError( + _("Host-add Rejected: Host with mgmt_ip %s already " + "exists") % ihost_dict['mgmt_ip']) + + try: + ihost_obj = objects.Host.get_by_filters_one( + pecan.request.context, + {'mgmt_mac': ihost_dict['mgmt_mac']}) + # A host with this MAC already exists. We will allow it to be + # added if the hostname and personality have not been set. + if ihost_obj['hostname'] or ihost_obj['personality']: + raise wsme.exc.ClientSideError( + _("Host-add Rejected: Host with mgmt_mac {} already " + "exists").format(ihost_dict['mgmt_mac'])) + # Check DNSMASQ for ip/mac already existing + # -> node in use by someone else or has already been booted + elif (not ihost_obj and self._dnsmasq_mac_exists( + ihost_dict['mgmt_mac'])): + raise wsme.exc.ClientSideError( + _("Host-add Rejected: mgmt_mac {} has already been " + "active").format(ihost_dict['mgmt_mac'])) + + # Use the uuid from the existing host + ihost_dict['uuid'] = ihost_obj['uuid'] + except exception.HostNotFound: + ihost_dict['mgmt_mac'] = cutils.validate_and_normalize_mac( + ihost_dict['mgmt_mac']) + # This is a new host + pass + + if not ihost_dict.get('uuid'): + ihost_dict['uuid'] = uuidutils.generate_uuid() + + # BM handling + ihost_orig = copy.deepcopy(ihost_dict) + + subfunctions = self._update_subfunctions(ihost_dict) + ihost_dict['subfunctions'] = subfunctions + + changed_paths = [] + delta = set() + + for key in objects.Host.fields: + # Internal values that aren't being modified + if key in ['id', 'updated_at', 'created_at']: + continue + + # Update only the new fields + if key in ihost_dict and ihost_dict[key] != ihost_orig[key]: + delta.add(key) + ihost_orig[key] = ihost_dict[key] + + bm_list = ['bm_type', 'bm_ip', 'bm_username', 'bm_password'] + for bmi in bm_list: + if bmi in ihost_dict: + delta.add(bmi) + changed_paths.append({'path': '/' + str(bmi), + 'value': ihost_dict[bmi], + 'op': 'replace'}) + + self._bm_semantic_check_and_update(ihost_orig, ihost_dict, + delta, changed_paths, + current_ihosts) + + if not ihost_dict.get('capabilities', {}): + ihost_dict['capabilities'] = {} + + # If this is the first controller being set up, + # configure and return + if ihost_dict['personality'] == k_host.CONTROLLER: + if not self._get_controllers(): + pecan.request.rpcapi.create_controller_filesystems( + pecan.request.context, ihost_dict['rootfs_device']) + controller_ihost = pecan.request.rpcapi.create_host( + pecan.request.context, ihost_dict) + pecan.request.rpcapi.configure_host( + pecan.request.context, + controller_ihost) + return Host.convert_with_links(controller_ihost) + + if ihost_dict['personality'] in ( + k_host.CONTROLLER, k_host.STORAGE): + self._controller_storage_node_setup(ihost_dict) + + # Validate that management name and IP do not already exist + # If one exists, other value must match in addresses table + mgmt_address_name = cutils.format_address_name( + ihost_dict['hostname'], constants.NETWORK_TYPE_MGMT) + self._validate_address_not_allocated(mgmt_address_name, + ihost_dict.get('mgmt_ip')) + + if ihost_dict.get('mgmt_ip'): + self._validate_ip_in_mgmt_network(ihost_dict['mgmt_ip']) + else: + del ihost_dict['mgmt_ip'] + + # Set host to reinstalling + ihost_dict.update({k_host.HOST_ACTION_STATE: + k_host.HAS_REINSTALLING}) + + # Creation/Configuration + if ihost_obj: + # The host exists - do an update. + for key in objects.Host.fields: + # Internal values that shouldn't be updated + if key in ['id', 'uuid', 'updated_at', 'created_at']: + continue + + # Update only the fields that are not empty and have changed + if (key in ihost_dict and ihost_dict[key] and + (ihost_obj[key] != ihost_dict[key])): + ihost_obj[key] = ihost_dict[key] + ihost_obj = pecan.request.rpcapi.update_host( + pecan.request.context, ihost_obj) + else: + # The host doesn't exist - do an add. + LOG.info("create_host=%s" % ihost_dict.get('hostname')) + ihost_obj = pecan.request.rpcapi.create_host( + pecan.request.context, ihost_dict) + + ihost_obj = objects.Host.get_by_uuid(pecan.request.context, + ihost_obj.uuid) + + # mgmt_network = pecan.request.systemconfig.network_get_by_type( + # constants.NETWORK_TYPE_MGMT) + + # Configure the new ihost, gets info about its addresses + host = pecan.request.rpcapi.configure_host( + pecan.request.context, + ihost_obj) + + if not host: + raise wsme.exc.ClientSideError( + _("Host-add Rejected: Host configure {} rejected ").format( + ihost_obj.hostname)) + + # Add host to mtc + ihost_obj['mgmt_ip'] = host.get('mgmt_ip') + new_ihost_mtc = ihost_obj.as_dict() + new_ihost_mtc.update({'operation': 'add'}) + new_ihost_mtc = cutils.removekeys_nonmtce(new_ihost_mtc) + # new_ihost_mtc.update( + # {'infra_ip': self._get_infra_ip_by_ihost(ihost_obj['uuid'])}) + + mtce_response = mtce_api.host_add( + self._api_token, + self._mtc_address, + self._mtc_port, + new_ihost_mtc, + constants.MTC_ADD_TIMEOUT_IN_SECS) + + self._handle_mtce_response('host_add', mtce_response) + + # once the host is added to mtc, attempt to power it on if requested + if power_on is not None and ihost_obj['bm_type'] is not None: + new_ihost_mtc.update({'action': k_host.ACTION_POWERON}) + + mtce_response = mtce_api.host_modify( + self._api_token, + self._mtc_address, + self._mtc_port, + new_ihost_mtc, + constants.MTC_ADD_TIMEOUT_IN_SECS) + + self._handle_mtce_response('power_on', mtce_response) + + # Notify the VIM that the host has been added - must be done after + # the host has been added to mtc and saved to the DB. + LOG.info("VIM notify add host add %s subfunctions={}").format(( + ihost_obj['hostname'], subfunctions)) + try: + self._vim_host_add(ihost_obj) + except Exception as e: + LOG.warn(_("No response from vim_api {} e={}").format( + ihost_obj['hostname'], e)) + self._api_token = None + pass # VIM audit will pickup + + log_end = cutils.timestamped("ihost_post_end") + LOG.info("SYS_I host %s %s" % (ihost_obj.hostname, log_end)) + + return Host.convert_with_links(ihost_obj) + + @cutils.synchronized(LOCK_NAME) + @expose('json') + def bulk_add(self): + pending_creation = [] + success_str = "" + error_str = "" + + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + return dict( + success="", + error="Bulk add on a simplex system is not allowed." + ) + + # Semantic Check: Prevent bulk add until there is an unlocked + # and enabled controller to manage them. + controller_list = objects.Host.list( + pecan.request.context, + filters={'personality': k_host.CONTROLLER}) + + have_unlocked_enabled_controller = False + for c in controller_list: + if (c['administrative'] == k_host.ADMIN_UNLOCKED and + c['operational'] == k_host.OPERATIONAL_ENABLED): + have_unlocked_enabled_controller = True + break + + if not have_unlocked_enabled_controller: + return dict( + success="", + error="Bulk_add requires enabled controller. " + "Please unlock controller-0, wait for it to enable " + "and then retry." + ) + + LOG.info("Starting ihost bulk_add operation") + assert isinstance(pecan.request.POST['file'], cgi.FieldStorage) + fileitem = pecan.request.POST['file'] + if not fileitem.filename: + return dict(success="", error="Error: No file uploaded") + + try: + contents = fileitem.file.read() + # Generate an array of hosts' attributes to be used in creation + root = ET.fromstring(contents) + except Exception: + return dict( + success="", + error="No hosts have been added, invalid XML document" + ) + + for idx, xmlhost in enumerate(root.findall('host')): + + new_ihost = {} + for attr in HOST_XML_ATTRIBUTES: + elem = xmlhost.find(attr) + if elem is not None: + # If the element is found, set the attribute. + # If the text field is empty, set it to the empty string. + new_ihost[attr] = elem.text or "" + else: + # If the element is not found, set the attribute to None. + new_ihost[attr] = None + + # This is the expected format of the location field + if new_ihost['location'] is not None: + new_ihost['location'] = {"locn": new_ihost['location']} + + # Semantic checks + try: + LOG.debug(new_ihost) + self._new_host_semantic_checks(new_ihost) + except Exception as ex: + culprit = new_ihost.get('hostname') or "with index " + str(idx) + return dict( + success="", + error=" No hosts have been added, error parsing host %s: " + "%s" % (culprit, ex) + ) + pending_creation.append(new_ihost) + + # Find local network adapter MACs + my_macs = list() + for liSnics in psutil.net_if_addrs().values(): + for snic in liSnics: + if snic.family == psutil.AF_LINK: + my_macs.append(snic.address) + + # Perform the actual creations + for new_host in pending_creation: + try: + # Configuring for the setup controller, only uses BMC fields + if new_host['mgmt_mac'].lower() in my_macs: + changed_paths = list() + + bm_list = ['bm_type', 'bm_ip', + 'bm_username', 'bm_password'] + for bmi in bm_list: + if bmi in new_host: + changed_paths.append({ + 'path': '/' + str(bmi), + 'value': new_host[bmi], + 'op': 'replace' + }) + + ihost_obj = [ihost for ihost in + objects.Host.list(pecan.request.context) + if ihost['mgmt_mac'] in my_macs] + if len(ihost_obj) != 1: + raise Exception( + "Unexpected: no/more_than_one host(s) contain(s) " + "a management mac address from " + "local network adapters") + self._patch(ihost_obj[0]['uuid'], + changed_paths, None) + else: + self._do_post(new_host) + + if (new_host['power_on'] is not None and + new_host['bm_type'] is None): + success_str = ( + "%s\n %s Warning: Ignoring due to " + "insufficient board management (bm) data." % + (success_str, new_host['hostname'])) + else: + success_str = "%s\n %s" % (success_str, + new_host['hostname']) + except Exception as ex: + LOG.exception(ex) + error_str += " " + (new_host.get('hostname') or + new_host.get('personality')) + \ + ": " + str(ex) + "\n" + + return dict( + success=success_str, + error=error_str + ) + + @expose('json') + def bulk_export(self): + def host_personality_name_sort_key(host_obj): + if host_obj.personality == k_host.CONTROLLER: + rank = 0 + elif host_obj.personality == k_host.STORAGE: + rank = 1 + elif host_obj.personality == k_host.COMPUTE: + rank = 2 + else: + rank = 3 + return rank, host_obj.hostname + + xml_host_node = et.Element('hosts', + {'version': cutils.get_sw_version()}) + mgmt_network = pecan.request.systemconfig.network_get_by_type( + constants.NETWORK_TYPE_MGMT) + + host_list = objects.Host.list(pecan.request.context) + sorted_hosts = sorted(host_list, key=host_personality_name_sort_key) + + for host in sorted_hosts: + _create_node(host, xml_host_node, host.personality, + mgmt_network.dynamic) + + xml_text = dom.parseString(et.tostring(xml_host_node)).toprettyxml() + result = {'content': xml_text} + return result + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(Host, body=Host) + def post(self, host): + """Create a new ihost.""" + ihost_dict = host.as_dict() + + # bm_password is not a part of ihost, so retrieve it from the body + body = json.loads(pecan.request.body) + if 'bm_password' in body: + ihost_dict['bm_password'] = body['bm_password'] + else: + ihost_dict['bm_password'] = '' + + return self._do_post(ihost_dict) + + @wsme_pecan.wsexpose(Host, unicode, body=[unicode]) + def patch(self, uuid, patch): + """Update an existing ihost. + """ + utils.validate_patch(patch) + + optimizable = 0 + optimize_list = ['/uptime', '/location', '/serialid', '/task'] + for p in patch: + path = p['path'] + if path in optimize_list: + optimizable += 1 + + if len(patch) == optimizable: + return self._patch(uuid, patch) + elif (pecan.request.user_agent.startswith('mtce') or + pecan.request.user_agent.startswith('vim')): + return self._patch_sys(uuid, patch) + else: + return self._patch_gen(uuid, patch) + + @cutils.synchronized(LOCK_NAME_SYS) + def _patch_sys(self, uuid, patch): + return self._patch(uuid, patch) + + @cutils.synchronized(LOCK_NAME) + def _patch_gen(self, uuid, patch): + return self._patch(uuid, patch) + + @staticmethod + def _validate_capability_is_not_set(old, new): + is_set, _ = new + return not is_set + + @staticmethod + def _validate_capability_is_equal(old, new): + return old == new + + def _validate_capabilities(self, old_caps, new_caps): + """Reject updating read-only host capabilities: + 1. stor_function. This field is set to 'monitor' for hosts that are + running ceph monitor process: + controller-0, controller-1, storage-0. + 2. Personality. This field is "virtual": + not saved in the database but + returned via API and displayed via "system host-show". + + :param old_caps: current host capabilities + :type old_caps: dict + :param new_caps: updated host capabilies (to be set) + :type new_caps: str + :raises: wsme.exc.ClientSideError when attempting to + change read-only capabilities + """ + if type(new_caps) == str: + try: + new_caps = ast.literal_eval(new_caps) + except SyntaxError: + pass + if type(new_caps) != dict: + raise wsme.exc.ClientSideError( + _("Changing capabilities type is not allowed: " + "old_value={}, new_value={}").format( + old_caps, new_caps)) + PROTECTED_CAPABILITIES = [ + ('Personality', + self._validate_capability_is_not_set), + (k_host.HOST_STOR_FUNCTION, + self._validate_capability_is_equal)] + for capability, validate in PROTECTED_CAPABILITIES: + old_is_set, old_value = ( + capability in old_caps, old_caps.get(capability)) + new_is_set, new_value = ( + capability in new_caps, new_caps.get(capability)) + if not validate((old_is_set, old_value), + (new_is_set, new_value)): + if old_is_set: + raise wsme.exc.ClientSideError( + _("Changing capability not allowed: " + "name={}, old_value={}, new_value={}. ").format( + capability, old_value, new_value)) + else: + raise wsme.exc.ClientSideError( + _("Setting capability not allowed: " + "name={}, value={}. ").format( + capability, new_value)) + + def _patch(self, uuid, patch): + log_start = cutils.timestamped("host_patch_start") + + patch_obj = jsonpatch.JsonPatch(patch) + + ihost_obj = objects.Host.get_by_uuid(pecan.request.context, uuid) + ihost_dict = ihost_obj.as_dict() + + self._add_host_semantic_checks(ihost_dict) + + # Add transient fields that are not stored in the database + ihost_dict['bm_password'] = None + + try: + patched_ihost = jsonpatch.apply_patch(ihost_dict, + patch_obj) + except jsonpatch.JsonPatchException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Patching Error: %s") % e) + + self._validate_capabilities( + ihost_dict['capabilities'], patched_ihost['capabilities']) + + ihost_dict_orig = dict(ihost_obj.as_dict()) + # defaults = objects.Host.get_defaults() + for key in objects.Host.fields: + # Internal values that shouldn't be part of the patch + if key in ['id', 'updated_at', 'created_at', 'infra_ip']: + continue + + # In case of a remove operation, add the missing fields back + # to the document with their default value + if key in ihost_dict and key not in patched_ihost: + # patched_ihost[key] = defaults[key] + patched_ihost[key] = ihost_obj[key] + + # Update only the fields that have changed + if ihost_obj[key] != patched_ihost[key]: + ihost_obj[key] = patched_ihost[key] + + delta = ihost_obj.obj_what_changed() + delta_handle = list(delta) + + uptime_update = False + if 'uptime' in delta_handle: + # There is a log of uptime updates, so just do a debug log + uptime_update = True + LOG.debug("%s %s patch" % (ihost_obj.hostname, + log_start)) + else: + LOG.info("%s %s patch" % (ihost_obj.hostname, + log_start)) + + hostupdate = HostUpdate(ihost_dict_orig, patched_ihost, delta) + if delta_handle: + self._validate_delta(delta_handle) + if delta_handle == ['uptime']: + LOG.debug("%s 1. delta_handle %s" % + (hostupdate.displayid, delta_handle)) + else: + LOG.info("%s 1. delta_handle %s" % + (hostupdate.displayid, delta_handle)) + else: + LOG.info("%s ihost_patch_end. No changes from %s." % + (hostupdate.displayid, pecan.request.user_agent)) + return Host.convert_with_links(ihost_obj) + + myaction = patched_ihost.get('action') + if self.action_check(myaction, hostupdate): + LOG.info("%s post action_check hostupdate " + "action=%s notify_vim=%s notify_mtc=%s " + "skip_notify_mtce=%s" % + (hostupdate.displayid, + hostupdate.action, + hostupdate.notify_vim, + hostupdate.notify_mtce, + hostupdate.skip_notify_mtce)) + + if self.stage_action(myaction, hostupdate): + LOG.info("%s Action staged: %s" % + (hostupdate.displayid, myaction)) + else: + LOG.info("%s ihost_patch_end stage_action rc %s" % + (hostupdate.displayid, hostupdate.nextstep)) + if hostupdate.nextstep == hostupdate.EXIT_RETURN_HOST: + return Host.convert_with_links(ihost_obj) + elif hostupdate.nextstep == hostupdate.EXIT_UPDATE_PREVAL: + if hostupdate.ihost_val_prenotify: + # update value in db prior to notifications + LOG.info("update ihost_val_prenotify: %s" % + hostupdate.ihost_val_prenotify) + ihost_obj = pecan.request.dbapi.host_update( + ihost_obj['uuid'], hostupdate.ihost_val_prenotify) + return Host.convert_with_links(ihost_obj) + + if myaction == k_host.ACTION_SUBFUNCTION_CONFIG: + self.perform_action_subfunction_config(ihost_obj) + + if myaction in delta_handle: + delta_handle.remove(myaction) + + LOG.info("%s post action_stage hostupdate " + "action=%s notify_vim=%s notify_mtc=%s " + "skip_notify_mtce=%s" % + (hostupdate.displayid, + hostupdate.action, + hostupdate.notify_vim, + hostupdate.notify_mtce, + hostupdate.skip_notify_mtce)) + + self._optimize_delta_handling(delta_handle) + + if 'administrative' in delta or 'operational' in delta: + self.stage_administrative_update(hostupdate) + + if delta_handle: + LOG.info("%s 2. delta_handle %s" % + (hostupdate.displayid, delta_handle)) + self._check_provisioning(hostupdate, patch) + if (hostupdate.ihost_orig['administrative'] == + k_host.ADMIN_UNLOCKED): + self.check_updates_while_unlocked(hostupdate, delta) + + current_ihosts = None + hostupdate.bm_type_changed_to_none = \ + self._bm_semantic_check_and_update(hostupdate.ihost_orig, + hostupdate.ihost_patch, + delta, patch_obj, + current_ihosts, + hostupdate) + LOG.info("%s post delta_handle hostupdate " + "action=%s notify_vim=%s notify_mtc=%s " + "skip_notify_mtce=%s" % + (hostupdate.displayid, + hostupdate.action, + hostupdate.notify_vim, + hostupdate.notify_mtce, + hostupdate.skip_notify_mtce)) + + if hostupdate.bm_type_changed_to_none: + hostupdate.ihost_val_update({'bm_ip': None, + 'bm_username': None, + 'bm_password': None}) + + if hostupdate.ihost_val_prenotify: + # update value in db prior to notifications + LOG.info("update ihost_val_prenotify: %s" % + hostupdate.ihost_val_prenotify) + pecan.request.dbapi.host_update(ihost_obj['uuid'], + hostupdate.ihost_val_prenotify) + + if hostupdate.ihost_val: + # apply the staged updates in preparation for update + LOG.info("%s apply ihost_val %s" % + (hostupdate.displayid, hostupdate.ihost_val)) + for k, v in hostupdate.ihost_val.iteritems(): + ihost_obj[k] = v + LOG.debug("AFTER Apply ihost_val %s to iHost %s" % + (hostupdate.ihost_val, ihost_obj.as_dict())) + + if 'personality' in delta: + self._update_subfunctions(ihost_obj) + + if hostupdate.notify_vim: + action = hostupdate.action + LOG.info("Notify VIM host action %s action=%s" % ( + ihost_obj['hostname'], action)) + try: + vim_api.vim_host_action( + pecan.request.context, + ihost_obj['uuid'], + ihost_obj['hostname'], + action, + constants.VIM_DEFAULT_TIMEOUT_IN_SECS) + except Exception as e: + LOG.warn(_("No response vim_api {} on action={} e={}").format( + ihost_obj['hostname'], action, e)) + self._api_token = None + if action == k_host.ACTION_FORCE_LOCK: + pass + else: + # reject continuation if VIM rejects action + raise wsme.exc.ClientSideError(_( + "VIM API Error or Timeout on action = %s " + "Please retry and if problem persists then " + "contact your system administrator.") % action) + + if hostupdate.configure_required: + LOG.info("%s Perform configure_host." % hostupdate.displayid) + if not ((ihost_obj['hostname']) and (ihost_obj['personality'])): + raise wsme.exc.ClientSideError( + _("Please provision 'hostname' and 'personality'.")) + + ihost_ret = pecan.request.rpcapi.configure_host( + pecan.request.context, ihost_obj) + + pecan.request.dbapi.host_update( + ihost_obj['uuid'], + {'capabilities': ihost_obj['capabilities']}) + + # Notify maintenance about updated mgmt_ip + ihost_obj['mgmt_ip'] = ihost_ret.get('mgmt_ip') + + hostupdate.notify_mtce = True + + pecan.request.dbapi.host_update( + ihost_obj['uuid'], + {'capabilities': ihost_obj['capabilities']}) + + if (k_host.TASK_REINSTALLING == ihost_obj.task and + k_host.CONFIG_STATUS_REINSTALL == ihost_obj.config_status): + # Clear reinstall flag when reinstall starts + ihost_obj.config_status = None + + mtce_response = {'status': None} + nonmtc_change_count = 0 + if hostupdate.notify_mtce and not hostupdate.skip_notify_mtce: + nonmtc_change_count = self.check_notify_mtce(myaction, hostupdate) + if nonmtc_change_count > 0: + LOG.info("%s Action %s perform notify_mtce" % + (hostupdate.displayid, myaction)) + new_ihost_mtc = ihost_obj.as_dict() + new_ihost_mtc = cutils.removekeys_nonmtce(new_ihost_mtc) + + if hostupdate.ihost_orig['invprovision'] == \ + k_host.PROVISIONED: + new_ihost_mtc.update({'operation': 'modify'}) + else: + new_ihost_mtc.update({'operation': 'add'}) + new_ihost_mtc.update({"invprovision": + ihost_obj['invprovision']}) + + if hostupdate.notify_action_lock: + new_ihost_mtc['action'] = k_host.ACTION_LOCK + elif hostupdate.notify_action_lock_force: + new_ihost_mtc['action'] = k_host.ACTION_FORCE_LOCK + elif myaction == k_host.ACTION_FORCE_UNLOCK: + new_ihost_mtc['action'] = k_host.ACTION_UNLOCK + + new_ihost_mtc.update({ + 'infra_ip': self._get_infra_ip_by_ihost(ihost_obj['uuid']) + }) + + if new_ihost_mtc['operation'] == 'add': + mtce_response = mtce_api.host_add( + self._api_token, self._mtc_address, self._mtc_port, + new_ihost_mtc, + constants.MTC_DEFAULT_TIMEOUT_IN_SECS) + elif new_ihost_mtc['operation'] == 'modify': + mtce_response = mtce_api.host_modify( + self._api_token, self._mtc_address, self._mtc_port, + new_ihost_mtc, + constants.MTC_DEFAULT_TIMEOUT_IN_SECS, + 3) + else: + LOG.warn("Unsupported Operation: %s" % new_ihost_mtc) + mtce_response = None + + if mtce_response is None: + mtce_response = {'status': 'fail', + 'reason': 'no response', + 'action': 'retry'} + + ihost_obj['action'] = k_host.ACTION_NONE + hostupdate.ihost_val_update({'action': k_host.ACTION_NONE}) + + if ((mtce_response['status'] == 'pass') or + (nonmtc_change_count == 0) or hostupdate.skip_notify_mtce): + + ihost_obj.save() + + if hostupdate.ihost_patch['operational'] == \ + k_host.OPERATIONAL_ENABLED: + self._update_add_ceph_state() + + if hostupdate.notify_availability: + if (hostupdate.notify_availability == + k_host.VIM_SERVICES_DISABLED): + imsg_dict = {'availability': + k_host.AVAILABILITY_OFFLINE} + else: + imsg_dict = {'availability': + k_host.VIM_SERVICES_ENABLED} + if (hostupdate.notify_availability != + k_host.VIM_SERVICES_ENABLED): + LOG.error( + _("Unexpected notify_availability={}").format( + hostupdate.notify_availability)) + + LOG.info(_("{} notify_availability={}").format( + hostupdate.displayid, + hostupdate.notify_availability)) + + pecan.request.rpcapi.platform_update_by_host( + pecan.request.context, ihost_obj['uuid'], imsg_dict) + + if hostupdate.bm_type_changed_to_none: + ibm_msg_dict = {} + pecan.request.rpcapi.bm_deprovision_by_host( + pecan.request.context, + ihost_obj['uuid'], + ibm_msg_dict) + + elif mtce_response['status'] is None: + raise wsme.exc.ClientSideError( + _("Timeout waiting for maintenance response. " + "Please retry and if problem persists then " + "contact your system administrator.")) + else: + if hostupdate.configure_required: + # rollback to unconfigure host as mtce has failed the request + invprovision_state = hostupdate.ihost_orig.get( + 'invprovision') or "" + if invprovision_state != k_host.PROVISIONED: + LOG.warn("unconfigure ihost %s provision=%s" % + (ihost_obj.uuid, invprovision_state)) + pecan.request.rpcapi.unconfigure_host( + pecan.request.context, + ihost_obj) + + raise wsme.exc.ClientSideError( + _("Operation Rejected: {}.{}.").format( + mtce_response['reason'], + mtce_response['action'])) + + if hostupdate.notify_vim_add_host: + # Notify the VIM that the host has been added - must be done after + # the host has been added to mtc and saved to the DB. + LOG.info("inventory notify add host add %s subfunctions=%s" % + (ihost_obj['hostname'], ihost_obj['subfunctions'])) + try: + self._vim_host_add(ihost_obj) + except Exception as e: + LOG.warn(_("No response from vim_api {} e={}").format( + ihost_obj['hostname'], e)) + self._api_token = None + pass # VIM audit will pickup + + # check if ttys_dcd is updated and notify the agent via conductor + # if necessary + if 'ttys_dcd' in hostupdate.delta: + self._handle_ttys_dcd_change(hostupdate.ihost_orig, + hostupdate.ihost_patch['ttys_dcd']) + + log_end = cutils.timestamped("host_patch_end") + if uptime_update: + LOG.debug("host %s %s patch" % (ihost_obj.hostname, + log_end)) + else: + LOG.info("host %s %s patch" % (ihost_obj.hostname, + log_end)) + + if ('administrative' in hostupdate.delta and + hostupdate.ihost_patch['administrative'] == + k_host.ADMIN_LOCKED): + LOG.info("Update host memory for (%s)" % ihost_obj['hostname']) + pecan.request.rpcapi.update_host_memory(pecan.request.context, + ihost_obj['uuid']) + return Host.convert_with_links(ihost_obj) + + def _vim_host_add(self, ihost): + LOG.info("inventory notify vim add host %s personality=%s" % ( + ihost['hostname'], ihost['personality'])) + + subfunctions = self._update_subfunctions(ihost) + try: + vim_api.vim_host_add( + pecan.request.context, + ihost['uuid'], + ihost['hostname'], + subfunctions, + ihost['administrative'], + ihost['operational'], + ihost['availability'], + ihost['subfunction_oper'], + ihost['subfunction_avail']) + except Exception as e: + LOG.warn(_("No response from vim_api {} e={}").format( + (ihost['hostname'], e))) + self._api_token = None + pass # VIM audit will pickup + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, unicode, status_code=204) + def delete(self, host_id): + """Delete a host. + """ + + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError(_( + "Deleting a host on a simplex system is not allowed.")) + + ihost = objects.Host.get_by_uuid(pecan.request.context, + host_id) + + if ihost.administrative == k_host.ADMIN_UNLOCKED: + if not ihost.hostname: + host = ihost.uuid + else: + host = ihost.hostname + + raise exception.HostLocked( + action=k_host.ACTION_DELETE, host=host) + + personality = ihost.personality + # allow delete of unprovisioned locked disabled & offline storage hosts + skip_ceph_checks = ( + (not ihost.invprovision or + ihost.invprovision == k_host.UNPROVISIONED) and + ihost.administrative == k_host.ADMIN_LOCKED and + ihost.operational == k_host.OPERATIONAL_DISABLED and + ihost.availability == k_host.AVAILABILITY_OFFLINE) + + if (personality is not None and + personality.find(k_host.STORAGE_HOSTNAME) != -1 and + not skip_ceph_checks): + # perform self.sc_op.check_delete; send to systemconfig + # to check monitors + LOG.info("TODO storage check with systemconfig for quoroum, " + "delete storage pools and tiers") + hosts = objects.Host.list(pecan.request.context) + num_monitors, required_monitors = \ + self._ceph.get_monitors_status(hosts) + if num_monitors < required_monitors: + raise wsme.exc.ClientSideError( + _("Only %d storage " + "monitor available. At least {} unlocked and " + "enabled hosts with monitors are required. Please " + "ensure hosts with monitors are unlocked and " + "enabled - candidates: {}, {}, {}").format( + (num_monitors, constants.MIN_STOR_MONITORS, + k_host.CONTROLLER_0_HOSTNAME, + k_host.CONTROLLER_1_HOSTNAME, + k_host.STORAGE_0_HOSTNAME))) + # send to systemconfig to delete storage pools and tiers + + LOG.warn("REST API delete host=%s user_agent=%s" % + (ihost['uuid'], pecan.request.user_agent)) + if not pecan.request.user_agent.startswith('vim'): + try: + vim_api.vim_host_delete( + pecan.request.context, + ihost.uuid, + ihost.hostname) + except Exception: + LOG.warn(_("No response from vim_api {} ").format( + ihost['uuid'])) + raise wsme.exc.ClientSideError( + _("System rejected delete request. " + "Please retry and if problem persists then " + "contact your system administrator.")) + + if (ihost.hostname and ihost.personality and + ihost.invprovision and + ihost.invprovision == k_host.PROVISIONED and + (k_host.COMPUTE in ihost.subfunctions)): + # wait for VIM signal + return + + idict = {'operation': k_host.ACTION_DELETE, + 'uuid': ihost.uuid, + 'invprovision': ihost.invprovision} + + mtce_response = mtce_api.host_delete( + self._api_token, self._mtc_address, self._mtc_port, + idict, constants.MTC_DELETE_TIMEOUT_IN_SECS) + + # Check mtce response prior to attempting delete + if mtce_response.get('status') != 'pass': + self._vim_host_add(ihost) + self._handle_mtce_response(k_host.ACTION_DELETE, + mtce_response) + + pecan.request.rpcapi.unconfigure_host(pecan.request.context, + ihost) + + # Delete the stor entries associated with this host + # Notify sysinv of host-delete + LOG.info("notify systemconfig of host-delete which will" + "also do stors, lvgs, pvs, ceph crush remove") + + # tell conductor to delete the keystore entry associated + # with this host (if present) + try: + pecan.request.rpcapi.unconfigure_keystore_account( + pecan.request.context, + KEYRING_BM_SERVICE, + ihost.uuid) + except exception.NotFound: + pass + + # Notify patching to drop the host + if ihost.hostname is not None: + try: + system = objects.System.get_one(pecan.request.context) + patch_api.patch_drop_host( + pecan.request.context, + hostname=ihost.hostname, + region_name=system.region_name) + except Exception as e: + LOG.warn(_("No response from drop-host patch api {}" + "e={}").format(ihost.hostname, e)) + pass + + pecan.request.dbapi.host_destroy(host_id) + + @staticmethod + def _handle_mtce_response(action, mtce_response): + LOG.info("mtce action %s response: %s" % + (action, mtce_response)) + if mtce_response is None: + mtce_response = {'status': 'fail', + 'reason': 'no response', + 'action': 'retry'} + + if mtce_response.get('reason') != 'no response': + raise wsme.exc.ClientSideError(_( + "Mtce rejected %s request." + "Please retry and if problem persists then contact your " + "system administrator.") % action) + else: + raise wsme.exc.ClientSideError(_( + "Timeout waiting for system response to %s. Please wait for a " + "few moments. If the host is not deleted,please retry. If " + "problem persists then contact your system administrator.") % + action) + + @staticmethod + def _get_infra_ip_by_ihost(ihost_uuid): + try: + # Get the list of interfaces for this ihost + iinterfaces = pecan.request.dbapi.iinterface_get_by_ihost( + ihost_uuid) + # Make a list of only the infra interfaces + infra_interfaces = [ + i for i in iinterfaces + if i['networktype'] == constants.NETWORK_TYPE_INFRA] + # Get the UUID of the infra interface (there is only one) + infra_interface_uuid = infra_interfaces[0]['uuid'] + # Return the first address for this interface (there is only one) + return pecan.request.dbapi.addresses_get_by_interface( + infra_interface_uuid)[0]['address'] + except Exception as ex: + LOG.debug("Could not find infra ip for host %s: %s" % ( + ihost_uuid, ex)) + return None + + @staticmethod + def _validate_ip_in_mgmt_network(ip): + network = pecan.request.systemconfig.network_get_by_type( + constants.NETWORK_TYPE_MGMT) + utils.validate_address_within_nework(ip, network) + + @staticmethod + def _validate_address_not_allocated(name, ip_address): + """Validate that address isn't allocated + + :param name: Address name to check isn't allocated. + :param ip_address: IP address to check isn't allocated. + """ + # When a host is added by systemconfig, this would already + # have been checked + LOG.info("TODO(sc) _validate_address_not_allocated name={} " + "ip_address={}".format(name, ip_address)) + + @staticmethod + def _dnsmasq_mac_exists(mac_addr): + """Check the dnsmasq.hosts file for an existing mac. + + :param mac_addr: mac address to check for. + """ + + dnsmasq_hosts_file = tsc.CONFIG_PATH + 'dnsmasq.hosts' + with open(dnsmasq_hosts_file, 'r') as f_in: + for line in f_in: + if mac_addr in line: + return True + return False + + @staticmethod + def _get_controllers(): + return objects.Host.list( + pecan.request.context, + filters={'personality': k_host.CONTROLLER}) + + @staticmethod + def _validate_delta(delta): + restricted_updates = ['uuid', 'id', 'created_at', 'updated_at', + 'cstatus', + 'mgmt_mac', 'mgmt_ip', 'infra_ip', + 'invprovision', 'recordtype', + 'host_action', + 'action_state'] + + if not pecan.request.user_agent.startswith('mtce'): + # Allow mtc to modify these through inventory-api. + mtce_only_updates = ['administrative', + 'availability', + 'operational', + 'subfunction_oper', + 'subfunction_avail', + 'reserved', + 'mtce_info', + 'task', + 'uptime'] + restricted_updates.extend(mtce_only_updates) + + if not pecan.request.user_agent.startswith('vim'): + vim_only_updates = ['vim_progress_status'] + restricted_updates.extend(vim_only_updates) + + intersection = set.intersection(set(delta), set(restricted_updates)) + if intersection: + raise wsme.exc.ClientSideError( + _("Change {} contains restricted {}.").format( + delta, intersection)) + else: + LOG.debug("PASS deltaset=%s restricted_updates %s" % + (delta, intersection)) + + @staticmethod + def _valid_storage_hostname(hostname): + return bool(re.match('^%s-[0-9]+$' % k_host.STORAGE_HOSTNAME, + hostname)) + + def _validate_hostname(self, hostname, personality): + + if personality and personality == k_host.COMPUTE: + # Check for invalid hostnames + err_tl = 'Name restricted to at most 255 characters.' + err_ic = 'Name may only contain letters, ' \ + 'numbers, underscores, periods and hyphens.' + myexpression = re.compile("^[\w\.\-]+$") + if not myexpression.match(hostname): + raise wsme.exc.ClientSideError(_("Error: {}").format(err_ic)) + if len(hostname) > 255: + raise wsme.exc.ClientSideError(_("Error: {}").format(err_tl)) + non_compute_hosts = ([k_host.CONTROLLER_0_HOSTNAME, + k_host.CONTROLLER_1_HOSTNAME]) + if (hostname and (hostname in non_compute_hosts) or + hostname.startswith(k_host.STORAGE_HOSTNAME)): + raise wsme.exc.ClientSideError( + _("{} Reject attempt to configure " + "invalid hostname for personality {}.").format( + (hostname, personality))) + else: + if personality and personality == k_host.CONTROLLER: + valid_hostnames = [k_host.CONTROLLER_0_HOSTNAME, + k_host.CONTROLLER_1_HOSTNAME] + if hostname not in valid_hostnames: + raise wsme.exc.ClientSideError( + _("Host with personality={} can only have a hostname " + "from {}").format(personality, valid_hostnames)) + elif personality and personality == k_host.STORAGE: + if not self._valid_storage_hostname(hostname): + raise wsme.exc.ClientSideError( + _("Host with personality={} can only have a hostname " + "starting with %s-(number)").format( + (personality, k_host.STORAGE_HOSTNAME))) + + else: + raise wsme.exc.ClientSideError( + _("{}: Reject attempt to configure with " + "invalid personality={} ").format( + (hostname, personality))) + + def _check_compute(self, patched_ihost, hostupdate=None): + # Check for valid compute node setup + hostname = patched_ihost.get('hostname') or "" + + if not hostname: + raise wsme.exc.ClientSideError( + _("Host {} of personality {}, must be provisioned " + "with a hostname.").format( + (patched_ihost.get('uuid'), + patched_ihost.get('personality')))) + + non_compute_hosts = ([k_host.CONTROLLER_0_HOSTNAME, + k_host.CONTROLLER_1_HOSTNAME]) + if (hostname in non_compute_hosts or + self._valid_storage_hostname(hostname)): + raise wsme.exc.ClientSideError( + _("Hostname {} is not allowed for personality 'compute'. " + "Please check hostname and personality.").format(hostname)) + + def _controller_storage_node_setup(self, patched_ihost, hostupdate=None): + # Initially set the subfunction of the host to it's personality + + if hostupdate: + patched_ihost = hostupdate.ihost_patch + + patched_ihost['subfunctions'] = patched_ihost['personality'] + + if patched_ihost['personality'] == k_host.CONTROLLER: + controller_0_exists = False + controller_1_exists = False + current_ihosts = objects.Host.list( + pecan.request.context, + filters={'personality': k_host.CONTROLLER}) + + for h in current_ihosts: + if h['hostname'] == k_host.CONTROLLER_0_HOSTNAME: + controller_0_exists = True + elif h['hostname'] == k_host.CONTROLLER_1_HOSTNAME: + controller_1_exists = True + if controller_0_exists and controller_1_exists: + raise wsme.exc.ClientSideError( + _("Two controller nodes have already been configured. " + "This host can not be configured as a controller.")) + + # Look up the IP address to use for this controller and set + # the hostname. + if controller_0_exists: + hostname = k_host.CONTROLLER_1_HOSTNAME + mgmt_ip = self._get_controller_address(hostname) + if hostupdate: + hostupdate.ihost_val_update({'hostname': hostname, + 'mgmt_ip': mgmt_ip}) + else: + patched_ihost['hostname'] = hostname + patched_ihost['mgmt_ip'] = mgmt_ip + elif controller_1_exists: + hostname = k_host.CONTROLLER_0_HOSTNAME + mgmt_ip = self._get_controller_address(hostname) + if hostupdate: + hostupdate.ihost_val_update({'hostname': hostname, + 'mgmt_ip': mgmt_ip}) + else: + patched_ihost['hostname'] = hostname + patched_ihost['mgmt_ip'] = mgmt_ip + else: + raise wsme.exc.ClientSideError( + _("Attempting to provision a controller when none " + "exists. This is impossible.")) + + # Subfunctions can be set directly via the config file + subfunctions = ','.join(tsc.subfunctions) + if hostupdate: + hostupdate.ihost_val_update({'subfunctions': subfunctions}) + else: + patched_ihost['subfunctions'] = subfunctions + + elif patched_ihost['personality'] == k_host.STORAGE: + # Storage nodes are only allowed if we are configured to use + # ceph for the cinder backend. + if not StorageBackendConfig.has_backend_configured( + pecan.request.dbapi, + constants.CINDER_BACKEND_CEPH + ): + raise wsme.exc.ClientSideError( + _("Storage nodes can only be configured if storage " + "cluster is configured for the cinder backend.")) + + current_storage_ihosts = objects.Host.list( + pecan.request.context, + filters={'personality': k_host.STORAGE}) + + current_storage = [] + for h in current_storage_ihosts: + if self._valid_storage_hostname(h['hostname']): + current_storage.append(h['hostname']) + + max_storage_hostnames = ["storage-%s" % x for x in + range(len(current_storage_ihosts) + 1)] + + # Look up IP address to use storage hostname + for h in reversed(max_storage_hostnames): + if h not in current_storage: + hostname = h + mgmt_ip = self._get_storage_address(hostname) + LOG.info("Found new hostname=%s mgmt_ip=%s " + "current_storage=%s" % + (hostname, mgmt_ip, current_storage)) + break + + if patched_ihost['hostname']: + if patched_ihost['hostname'] != hostname: + raise wsme.exc.ClientSideError( + _("Storage name {} not allowed. Expected {}. " + "Storage nodes can be one of: " + "storage-#.").format( + (patched_ihost['hostname'], hostname))) + + if hostupdate: + hostupdate.ihost_val_update({'hostname': hostname, + 'mgmt_ip': mgmt_ip}) + else: + patched_ihost['hostname'] = hostname + patched_ihost['mgmt_ip'] = mgmt_ip + + @staticmethod + def _optimize_delta_handling(delta_handle): + """Optimize specific patch operations. + Updates delta_handle to identify remaining patch semantics to check. + """ + optimizable = ['location', 'serialid'] + if pecan.request.user_agent.startswith('mtce'): + mtc_optimizable = ['operational', 'availability', 'task', 'uptime', + 'subfunction_oper', 'subfunction_avail'] + optimizable.extend(mtc_optimizable) + + for k in optimizable: + if k in delta_handle: + delta_handle.remove(k) + + @staticmethod + def _semantic_mtc_check_action(hostupdate, action): + """ + Perform semantic checks with patch action vs current state + + returns: notify_mtc_check_action + """ + notify_mtc_check_action = True + ihost = hostupdate.ihost_orig + patched_ihost = hostupdate.ihost_patch + + if action in [k_host.VIM_SERVICES_DISABLED, + k_host.VIM_SERVICES_DISABLE_FAILED, + k_host.VIM_SERVICES_DISABLE_EXTEND, + k_host.VIM_SERVICES_ENABLED, + k_host.VIM_SERVICES_DELETE_FAILED]: + # These are not mtce actions + return notify_mtc_check_action + + LOG.info("%s _semantic_mtc_check_action %s" % + (hostupdate.displayid, action)) + + # Semantic Check: Auto-Provision: Reset, Reboot or Power-On case + if ((cutils.host_has_function(ihost, k_host.COMPUTE)) and + (ihost['administrative'] == k_host.ADMIN_LOCKED) and + ((patched_ihost['action'] == k_host.ACTION_RESET) or + (patched_ihost['action'] == k_host.ACTION_REBOOT) or + (patched_ihost['action'] == k_host.ACTION_POWERON) or + (patched_ihost['action'] == k_host.ACTION_POWEROFF))): + notify_mtc_check_action = True + + return notify_mtc_check_action + + @staticmethod + def _bm_semantic_check_and_update(ohost, phost, delta, patch_obj, + current_ihosts=None, hostupdate=None): + """Parameters: + ohost: object original host + phost: mutable dictionary patch host + delta: default keys changed + patch_obj: all changed paths + returns bm_type_changed_to_none + """ + + # NOTE: since the bm_mac is still in the DB; + # this is just to disallow user to modify it. + if 'bm_mac' in delta: + raise wsme.exc.ClientSideError( + _("Patching Error: can't replace non-existent object " + "'bm_mac' ")) + + bm_type_changed_to_none = False + + bm_set = {'bm_type', + 'bm_ip', + 'bm_username', + 'bm_password'} + + password_exists = any(p['path'] == '/bm_password' for p in patch_obj) + if not (delta.intersection(bm_set) or password_exists): + return bm_type_changed_to_none + + if hostupdate: + hostupdate.notify_mtce = True + + patch_bm_password = None + for p in patch_obj: + if p['path'] == '/bm_password': + patch_bm_password = p['value'] + + password_exists = password_exists and patch_bm_password is not None + + bm_type_orig = ohost.get('bm_type') or "" + bm_type_patch = phost.get('bm_type') or "" + if bm_type_patch.lower() == 'none': + bm_type_patch = '' + if (not bm_type_patch) and (bm_type_orig != bm_type_patch): + LOG.info("bm_type None from %s to %s." % + (ohost['bm_type'], phost['bm_type'])) + + bm_type_changed_to_none = True + + if 'bm_ip' in delta: + obm_ip = ohost['bm_ip'] or "" + nbm_ip = phost['bm_ip'] or "" + LOG.info("bm_ip in delta=%s obm_ip=%s nbm_ip=%s" % + (delta, obm_ip, nbm_ip)) + if obm_ip != nbm_ip: + if (pecan.request.user_agent.startswith('mtce') and + not bm_type_changed_to_none): + raise wsme.exc.ClientSideError( + _("Rejected: {} Board Management " + "controller IP Address is not" + "user-modifiable.").format(phost['hostname'])) + + if phost['bm_ip'] or phost['bm_type'] or phost['bm_username']: + if (not phost['bm_type'] or + (phost['bm_type'] and phost['bm_type'].lower() == + k_host.BM_TYPE_NONE)) and not bm_type_changed_to_none: + raise wsme.exc.ClientSideError( + _("{}: Rejected: Board Management controller Type " + "is not provisioned. Provisionable values: " + "'bmc'.").format(phost['hostname'])) + elif not phost['bm_username']: + raise wsme.exc.ClientSideError( + _("{}: Rejected: Board Management controller username " + "is not configured.").format(phost['hostname'])) + + # Semantic Check: Validate BM type against supported list + # ilo, quanta is kept for backwards compatability only + valid_bm_type_list = [None, 'None', k_host.BM_TYPE_NONE, + k_host.BM_TYPE_GENERIC, + 'ilo', 'ilo3', 'ilo4', 'quanta'] + + if not phost['bm_type']: + phost['bm_type'] = None + + if not (phost['bm_type'] in valid_bm_type_list): + raise wsme.exc.ClientSideError( + _("{}: Rejected: '{}' is not a supported board management " + "type. Must be one of {}").format( + (phost['hostname'], phost['bm_type'], valid_bm_type_list))) + + bm_type_str = phost['bm_type'] + if (phost['bm_type'] and + bm_type_str.lower() != k_host.BM_TYPE_NONE): + LOG.info("Updating bm_type from %s to %s" % + (phost['bm_type'], k_host.BM_TYPE_GENERIC)) + phost['bm_type'] = k_host.BM_TYPE_GENERIC + if hostupdate: + hostupdate.ihost_val_update( + {'bm_type': k_host.BM_TYPE_GENERIC}) + else: + phost['bm_type'] = None + if hostupdate: + hostupdate.ihost_val_update({'bm_type': None}) + + if (phost['bm_type'] and phost['bm_ip'] and + (ohost['bm_ip'] != phost['bm_ip'])): + if not cutils.is_valid_ip(phost['bm_ip']): + raise wsme.exc.ClientSideError( + _("{}: Rejected: Board Management controller IP Address " + "is not valid.").format(phost['hostname'])) + + if current_ihosts and ('bm_ip' in phost): + bm_ips = [h['bm_ip'] for h in current_ihosts] + + if phost['bm_ip'] and (phost['bm_ip'] in bm_ips): + raise wsme.exc.ClientSideError( + _("Host-add Rejected: bm_ip %s already exists") % + phost['bm_ip']) + + # Update keyring with updated board management credentials, if supplied + if (ohost['bm_username'] and phost['bm_username'] and + (ohost['bm_username'] != phost['bm_username'])): + if not password_exists: + raise wsme.exc.ClientSideError( + _("{} Rejected: username change attempt from {} to {} " + "without corresponding password.").format( + (phost['hostname'], + ohost['bm_username'], + phost['bm_username']))) + + if password_exists: + # The conductor will handle creating the keystore acct + pecan.request.rpcapi.configure_keystore_account( + pecan.request.context, + KEYRING_BM_SERVICE, + phost['uuid'], + patch_bm_password) + LOG.info("%s bm semantic checks for user_agent %s passed" % + (phost['hostname'], pecan.request.user_agent)) + + return bm_type_changed_to_none + + @staticmethod + def _semantic_check_nova_local_storage(ihost_uuid, personality): + """ + Perform semantic checking for nova local storage + :param ihost_uuid: uuid of host with compute functionality + :param personality: personality of host with compute functionality + """ + + LOG.info("TODO _semantic_check_nova_local_storage nova local obsol") + # TODO(sc) configure_check (unlock_compute) + return + + @staticmethod + def _handle_ttys_dcd_change(ihost, ttys_dcd): + """ + Handle serial line carrier detection enable or disable request. + :param ihost: unpatched ihost dictionary + :param ttys_dcd: attribute supplied in patch + """ + LOG.info("%s _handle_ttys_dcd_change from %s to %s" % + (ihost['hostname'], ihost['ttys_dcd'], ttys_dcd)) + + # check if the flag is changed + if ttys_dcd is not None: + if ihost['ttys_dcd'] is None or ihost['ttys_dcd'] != ttys_dcd: + if ((ihost['administrative'] == k_host.ADMIN_LOCKED and + ihost['availability'] == k_host.AVAILABILITY_ONLINE) or + (ihost['administrative'] == k_host.ADMIN_UNLOCKED and + ihost['operational'] == k_host.OPERATIONAL_ENABLED)): + LOG.info("Notify conductor ttys_dcd change: (%s) (%s)" % + (ihost['uuid'], ttys_dcd)) + pecan.request.rpcapi.configure_ttys_dcd( + pecan.request.context, ihost['uuid'], ttys_dcd) + + def action_check(self, action, hostupdate): + """Performs semantic checks related to action""" + + if not action or (action.lower() == k_host.ACTION_NONE): + rc = False + return rc + + valid_actions = [k_host.ACTION_UNLOCK, + k_host.ACTION_FORCE_UNLOCK, + k_host.ACTION_LOCK, + k_host.ACTION_FORCE_LOCK, + k_host.ACTION_SWACT, + k_host.ACTION_FORCE_SWACT, + k_host.ACTION_RESET, + k_host.ACTION_REBOOT, + k_host.ACTION_REINSTALL, + k_host.ACTION_POWERON, + k_host.ACTION_POWEROFF, + k_host.VIM_SERVICES_ENABLED, + k_host.VIM_SERVICES_DISABLED, + k_host.VIM_SERVICES_DISABLE_FAILED, + k_host.VIM_SERVICES_DISABLE_EXTEND, + k_host.VIM_SERVICES_DELETE_FAILED, + k_host.ACTION_SUBFUNCTION_CONFIG] + + if action not in valid_actions: + raise wsme.exc.ClientSideError( + _("'%s' is not a supported maintenance action") % action) + + force_unlock = False + if action == k_host.ACTION_FORCE_UNLOCK: + # set force_unlock for semantic check and update action + # for compatability with vim and mtce + action = k_host.ACTION_UNLOCK + force_unlock = True + hostupdate.action = action + rc = True + + if action == k_host.ACTION_UNLOCK: + # Set host_action in DB as early as possible as we need + # it as a synchronization point for things like lvg/pv + # deletion which is not allowed when ihost is unlokced + # or in the process of unlocking. + rc = self.update_host_action(action, hostupdate) + if rc: + pecan.request.dbapi.host_update(hostupdate.ihost_orig['uuid'], + hostupdate.ihost_val_prenotify) + try: + self.check_unlock(hostupdate, force_unlock) + except Exception as e: + LOG.info("host unlock check didn't pass, " + "so set the host_action back to None " + "and re-raise the exception") + self.update_host_action(None, hostupdate) + pecan.request.dbapi.host_update( + hostupdate.ihost_orig['uuid'], + hostupdate.ihost_val_prenotify) + raise e + elif action == k_host.ACTION_LOCK: + if self.check_lock(hostupdate): + rc = self.update_host_action(action, hostupdate) + elif action == k_host.ACTION_FORCE_LOCK: + if self.check_force_lock(hostupdate): + rc = self.update_host_action(action, hostupdate) + elif action == k_host.ACTION_SWACT: + self.check_swact(hostupdate) + elif action == k_host.ACTION_FORCE_SWACT: + self.check_force_swact(hostupdate) + elif action == k_host.ACTION_REBOOT: + self.check_reboot(hostupdate) + elif action == k_host.ACTION_RESET: + self.check_reset(hostupdate) + elif action == k_host.ACTION_REINSTALL: + self.check_reinstall(hostupdate) + elif action == k_host.ACTION_POWERON: + self.check_poweron(hostupdate) + elif action == k_host.ACTION_POWEROFF: + self.check_poweroff(hostupdate) + elif action == k_host.VIM_SERVICES_ENABLED: + self.update_vim_progress_status(action, hostupdate) + elif action == k_host.VIM_SERVICES_DISABLED: + self.update_vim_progress_status(action, hostupdate) + elif action == k_host.VIM_SERVICES_DISABLE_FAILED: + self.update_vim_progress_status(action, hostupdate) + elif action == k_host.VIM_SERVICES_DISABLE_EXTEND: + self.update_vim_progress_status(action, hostupdate) + elif action == k_host.VIM_SERVICES_DELETE_FAILED: + self.update_vim_progress_status(action, hostupdate) + elif action == k_host.ACTION_SUBFUNCTION_CONFIG: + self._check_subfunction_config(hostupdate) + self._semantic_check_nova_local_storage( + hostupdate.ihost_patch['uuid'], + hostupdate.ihost_patch['personality']) + else: + raise wsme.exc.ClientSideError( + _("action_check unrecognized action: {}").format(action)) + + if action in k_host.ACTIONS_MTCE: + if self._semantic_mtc_check_action(hostupdate, action): + hostupdate.notify_mtce = True + task_val = hostupdate.get_task_from_action(action) + if task_val: + hostupdate.ihost_val_update({'task': task_val}) + + elif 'administrative' in hostupdate.delta: + # administrative state changed, update task, host_action in case + hostupdate.ihost_val_update({'task': "", + 'host_action': ""}) + + LOG.info("%s action=%s ihost_val_prenotify: %s ihost_val: %s" % + (hostupdate.displayid, + hostupdate.action, + hostupdate.ihost_val_prenotify, + hostupdate.ihost_val)) + + if hostupdate.ihost_val_prenotify: + LOG.info("%s host.update.ihost_val_prenotify %s" % + (hostupdate.displayid, hostupdate.ihost_val_prenotify)) + + if self.check_notify_vim(action): + hostupdate.notify_vim = True + + if self.check_notify_mtce(action, hostupdate) > 0: + hostupdate.notify_mtce = True + + LOG.info("%s action_check action=%s, notify_vim=%s " + "notify_mtce=%s rc=%s" % + (hostupdate.displayid, + action, + hostupdate.notify_vim, + hostupdate.notify_mtce, + rc)) + + return rc + + @staticmethod + def check_notify_vim(action): + if action in k_host.ACTIONS_VIM: + return True + else: + return False + + @staticmethod + def check_notify_mtce(action, hostupdate): + """Determine whether mtce should be notified of this patch request + returns: Integer (nonmtc_change_count) + """ + + nonmtc_change_count = 0 + if action in k_host.ACTIONS_VIM: + return nonmtc_change_count + elif action in k_host.ACTIONS_CONFIG: + return nonmtc_change_count + elif action in k_host.VIM_SERVICES_ENABLED: + return nonmtc_change_count + + mtc_ignore_list = ['administrative', 'availability', 'operational', + 'task', 'uptime', 'capabilities', + 'host_action', + 'subfunction_oper', 'subfunction_avail', + 'vim_progress_status' + 'location', 'serialid', 'invprovision'] + + if pecan.request.user_agent.startswith('mtce'): + mtc_ignore_list.append('bm_ip') + + nonmtc_change_count = len(set(hostupdate.delta) - set(mtc_ignore_list)) + + return nonmtc_change_count + + @staticmethod + def stage_administrative_update(hostupdate): + # Always configure when the host is unlocked - this will set the + # hostname and allow the node to boot and configure itself. + # NOTE: This is being hit the second time through this function on + # the unlock. The first time through, the "action" is set to unlock + # on the patched_iHost, but the "administrative" is still locked. + # Once maintenance processes the unlock, they do another patch and + # set the "administrative" to unlocked. + if ('administrative' in hostupdate.delta and + hostupdate.ihost_patch['administrative'] == + k_host.ADMIN_UNLOCKED): + if hostupdate.ihost_orig['invprovision'] == \ + k_host.UNPROVISIONED or \ + hostupdate.ihost_orig['invprovision'] is None: + LOG.info("stage_administrative_update: provisioning") + hostupdate.ihost_val_update({'invprovision': + k_host.PROVISIONING}) + + if ('operational' in hostupdate.delta and + hostupdate.ihost_patch['operational'] == + k_host.OPERATIONAL_ENABLED): + if hostupdate.ihost_orig['invprovision'] == k_host.PROVISIONING: + # first time unlocked successfully + LOG.info("stage_administrative_update: provisioned") + hostupdate.ihost_val_update( + {'invprovision': k_host.PROVISIONED}) + + @staticmethod + def _update_add_ceph_state(): + # notify systemconfig of the new ceph state + LOG.info("TODO(SC) _update_add_ceph_state") + + @staticmethod + def update_host_action(action, hostupdate): + if action is None: + preval = {'host_action': ''} + elif action == k_host.ACTION_FORCE_LOCK: + preval = {'host_action': k_host.ACTION_FORCE_LOCK} + elif action == k_host.ACTION_LOCK: + preval = {'host_action': k_host.ACTION_LOCK} + elif (action == k_host.ACTION_UNLOCK or + action == k_host.ACTION_FORCE_UNLOCK): + preval = {'host_action': k_host.ACTION_UNLOCK} + else: + LOG.error("update_host_action unsupported action: %s" % action) + return False + hostupdate.ihost_val_prenotify.update(preval) + hostupdate.ihost_val.update(preval) + + task_val = hostupdate.get_task_from_action(action) + if task_val: + hostupdate.ihost_val_update({'task': task_val}) + return True + + @staticmethod + def update_vim_progress_status(action, hostupdate): + LOG.info("%s Pending update_vim_progress_status %s" % + (hostupdate.displayid, action)) + return True + + def _check_provisioning(self, hostupdate, patch): + # Once the host has been provisioned lock down additional fields + + ihost = hostupdate.ihost_patch + delta = hostupdate.delta + + provision_state = [k_host.PROVISIONED, k_host.PROVISIONING] + if hostupdate.ihost_orig['invprovision'] in provision_state: + state_rel_path = ['hostname', 'personality', 'subfunctions'] + if any(p in state_rel_path for p in delta): + raise wsme.exc.ClientSideError( + _("The following fields can not be modified because " + "this host {} has been configured: " + "hostname, personality, subfunctions").format( + hostupdate.ihost_orig['hostname'])) + + # Check whether any configurable installation parameters are updated + install_parms = ['boot_device', 'rootfs_device', 'install_output', + 'console', 'tboot'] + if any(p in install_parms for p in delta): + # Disallow changes if the node is not locked + if ihost['administrative'] != k_host.ADMIN_LOCKED: + raise wsme.exc.ClientSideError( + _("Host must be locked before updating " + "installation parameters.")) + + # An update to PXE boot information is required + hostupdate.configure_required = True + + if 'personality' in delta: + LOG.info("iHost['personality']=%s" % + hostupdate.ihost_orig['personality']) + + if hostupdate.ihost_orig['personality']: + raise wsme.exc.ClientSideError( + _("Can not change personality after it has been set. " + "Host {} must be deleted and re-added in order to change" + " the personality.").format( + hostupdate.ihost_orig['hostname'])) + + if (hostupdate.ihost_patch['personality'] in + (k_host.CONTROLLER, k_host.STORAGE)): + self._controller_storage_node_setup(hostupdate.ihost_patch, + hostupdate) + # check the subfunctions are updated properly + LOG.info("hostupdate.ihost_patch.subfunctions %s" % + hostupdate.ihost_patch['subfunctions']) + elif hostupdate.ihost_patch['personality'] == k_host.COMPUTE: + self._check_compute(hostupdate.ihost_patch, hostupdate) + else: + LOG.error("Unexpected personality: %s" % + hostupdate.ihost_patch['personality']) + + # Always configure when the personality has been set - this will + # set up the PXE boot information so the software can be installed + hostupdate.configure_required = True + + # Notify VIM when the personality is set. + hostupdate.notify_vim_add_host = True + + if k_host.SUBFUNCTIONS in delta: + if hostupdate.ihost_orig[k_host.SUBFUNCTIONS]: + raise wsme.exc.ClientSideError( + _("Can not change subfunctions after it has been set. Host" + "{} must be deleted and re-added in order to change " + "the subfunctions.").format( + hostupdate.ihost_orig['hostname'])) + + if hostupdate.ihost_patch['personality'] == k_host.COMPUTE: + valid_subfunctions = (k_host.COMPUTE, + k_host.LOWLATENCY) + elif hostupdate.ihost_patch['personality'] == k_host.CONTROLLER: + valid_subfunctions = (k_host.CONTROLLER, + k_host.COMPUTE, + k_host.LOWLATENCY) + elif hostupdate.ihost_patch['personality'] == k_host.STORAGE: + # Comparison is expecting a list + valid_subfunctions = (k_host.STORAGE, k_host.STORAGE) + + subfunctions_set = \ + set(hostupdate.ihost_patch[k_host.SUBFUNCTIONS].split(',')) + + if not subfunctions_set.issubset(valid_subfunctions): + raise wsme.exc.ClientSideError( + ("%s subfunctions %s contains unsupported values. " + "Allowable: %s." % + (hostupdate.displayid, + subfunctions_set, + valid_subfunctions))) + + if hostupdate.ihost_patch['personality'] == k_host.COMPUTE: + if k_host.COMPUTE not in subfunctions_set: + # Automatically add it + subfunctions_list = list(subfunctions_set) + subfunctions_list.insert(0, k_host.COMPUTE) + subfunctions = ','.join(subfunctions_list) + + LOG.info("%s update subfunctions=%s" % + (hostupdate.displayid, subfunctions)) + hostupdate.ihost_val_prenotify.update( + {'subfunctions': subfunctions}) + hostupdate.ihost_val.update({'subfunctions': subfunctions}) + + # The hostname for a controller or storage node cannot be modified + + # Disallow hostname changes + if 'hostname' in delta: + if hostupdate.ihost_orig['hostname']: + if (hostupdate.ihost_patch['hostname'] != + hostupdate.ihost_orig['hostname']): + raise wsme.exc.ClientSideError( + _("The hostname field can not be modified because " + "the hostname {} has already been configured. " + "If changing hostname is required, please delete " + "this host, then readd.").format( + hostupdate.ihost_orig['hostname'])) + + for attribute in patch: + # check for duplicate attributes + for attribute2 in patch: + if attribute['path'] == attribute2['path']: + if attribute['value'] != attribute2['value']: + raise wsme.exc.ClientSideError( + _("Illegal duplicate parameters passed.")) + + if 'personality' in delta or 'hostname' in delta: + personality = hostupdate.ihost_patch.get('personality') or "" + hostname = hostupdate.ihost_patch.get('hostname') or "" + if personality and hostname: + self._validate_hostname(hostname, personality) + + if 'personality' in delta: + HostController._personality_license_check( + hostupdate.ihost_patch['personality']) + + @staticmethod + def _personality_license_check(personality): + if personality == k_host.CONTROLLER: + return + + if not personality: + return + + if personality == k_host.COMPUTE and utils.is_aio_duplex_system(): + if utils.get_compute_count() >= constants.AIO_DUPLEX_MAX_COMPUTES: + msg = _("All-in-one Duplex is restricted to " + "%s computes.") % constants.AIO_DUPLEX_MAX_COMPUTES + raise wsme.exc.ClientSideError(msg) + else: + return + + if (utils.SystemHelper.get_product_build() == + constants.TIS_AIO_BUILD): + msg = _("Personality [%s] for host is not compatible " + "with installed software. ") % personality + + raise wsme.exc.ClientSideError(msg) + + @staticmethod + def check_reset(hostupdate): + """Check semantics on host-reset.""" + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError( + _("Can not 'Reset' a simplex system")) + + if hostupdate.ihost_orig['administrative'] == k_host.ADMIN_UNLOCKED: + raise wsme.exc.ClientSideError( + _("Can not 'Reset' an 'unlocked' host {}; " + "Please 'Lock' first").format(hostupdate.displayid)) + + return True + + @staticmethod + def check_poweron(hostupdate): + # Semantic Check: State Dependency: Power-On case + if (hostupdate.ihost_orig['administrative'] == + k_host.ADMIN_UNLOCKED): + raise wsme.exc.ClientSideError( + _("Can not 'Power-On' an already Powered-on " + "and 'unlocked' host {}").format(hostupdate.displayid)) + + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError( + _("Can not 'Power-On' an already Powered-on " + "simplex system")) + + @staticmethod + def check_poweroff(hostupdate): + # Semantic Check: State Dependency: Power-Off case + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError( + _("Can not 'Power-Off' a simplex system via " + "system commands")) + + if (hostupdate.ihost_orig['administrative'] == + k_host.ADMIN_UNLOCKED): + raise wsme.exc.ClientSideError( + _("Can not 'Power-Off' an 'unlocked' host {}; " + "Please 'Lock' first").format(hostupdate.displayid)) + + @staticmethod + def check_reinstall(hostupdate): + """Semantic Check: State Dependency: Reinstall case""" + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError(_( + "Reinstalling a simplex system is not allowed.")) + + ihost = hostupdate.ihost_orig + if ihost['administrative'] == k_host.ADMIN_UNLOCKED: + raise wsme.exc.ClientSideError( + _("Can not 'Reinstall' an 'unlocked' host {}; " + "Please 'Lock' first").format(hostupdate.displayid)) + elif ((ihost['administrative'] == k_host.ADMIN_LOCKED) and + (ihost['availability'] != "online")): + raise wsme.exc.ClientSideError( + _("Can not 'Reinstall' {} while it is 'offline'. " + "Please wait for this host's availability state " + "to be 'online' and then re-issue the reinstall " + "command.").format(hostupdate.displayid)) + + def check_unlock(self, hostupdate, force_unlock=False): + """Check semantics on host-unlock.""" + if (hostupdate.action != k_host.ACTION_UNLOCK and + hostupdate.action != k_host.ACTION_FORCE_UNLOCK): + LOG.error("check_unlock unexpected action: %s" % hostupdate.action) + return False + + # Semantic Check: Don't unlock if installation failed + if (hostupdate.ihost_orig['install_state'] == + constants.INSTALL_STATE_FAILED): + raise wsme.exc.ClientSideError( + _("Cannot unlock host {} due to installation failure").format( + hostupdate.displayid)) + + # Semantic Check: Avoid Unlock of Unlocked Host + if hostupdate.ihost_orig['administrative'] == k_host.ADMIN_UNLOCKED: + raise wsme.exc.ClientSideError( + _("Avoiding 'unlock' action on already " + "'unlocked' host {}").format( + hostupdate.ihost_orig['hostname'])) + + # Semantic Check: Action Dependency: Power-Off / Unlock case + if (hostupdate.ihost_orig['availability'] == + k_host.ACTION_POWEROFF): + raise wsme.exc.ClientSideError( + _("Can not 'Unlock a Powered-Off' host {}; Power-on, " + "wait for 'online' status and then 'unlock'").format( + hostupdate.displayid)) + + # Semantic Check: Action Dependency: Online / Unlock case + if (not force_unlock and hostupdate.ihost_orig['availability'] != + k_host.AVAILABILITY_ONLINE): + raise wsme.exc.ClientSideError( + _("Host {} is not online. " + "Wait for 'online' availability status and " + "then 'unlock'").format(hostupdate.displayid)) + + # To unlock, we need the following additional fields + if not (hostupdate.ihost_patch['mgmt_mac'] and + hostupdate.ihost_patch['mgmt_ip'] and + hostupdate.ihost_patch['hostname'] and + hostupdate.ihost_patch['personality'] and + hostupdate.ihost_patch['subfunctions']): + raise wsme.exc.ClientSideError( + _("Can not unlock an unprovisioned host {}. " + "Please perform 'Edit Host' to provision host.").format( + hostupdate.displayid)) + + # To unlock, ensure reinstall has completed + action_state = hostupdate.ihost_orig[k_host.HOST_ACTION_STATE] + if (action_state and + action_state == k_host.HAS_REINSTALLING): + if not force_unlock: + raise wsme.exc.ClientSideError( + _("Can not unlock host {} undergoing reinstall. " + "Please ensure host has completed reinstall " + "prior to unlock.").format(hostupdate.displayid)) + else: + LOG.warn("Allowing force-unlock of host %s " + "undergoing reinstall." % hostupdate.displayid) + + personality = hostupdate.ihost_patch.get('personality') + if personality == k_host.CONTROLLER: + self.check_unlock_controller(hostupdate, force_unlock) + if cutils.host_has_function(hostupdate.ihost_patch, k_host.COMPUTE): + self.check_unlock_compute(hostupdate) + elif personality == k_host.STORAGE: + self.check_unlock_storage(hostupdate) + + # self.check_unlock_interfaces(hostupdate) + # self.unlock_update_mgmt_infra_interface(hostupdate.ihost_patch) + # TODO(storage) self.check_unlock_partitions(hostupdate) + self.check_unlock_patching(hostupdate, force_unlock) + + hostupdate.configure_required = True + hostupdate.notify_vim = True + + return True + + def check_unlock_patching(self, hostupdate, force_unlock): + """Check whether the host is patch current. + """ + + if force_unlock: + return + + try: + system = objects.System.get_one(pecan.request.context) + response = patch_api.patch_query_hosts( + pecan.request.context, + system.region_name) + phosts = response['data'] + except Exception as e: + LOG.warn(_("No response from patch api {} e={}").format( + (hostupdate.displayid, e))) + self._api_token = None + return + + for phost in phosts: + if phost.get('hostname') == hostupdate.ihost_patch.get('hostname'): + if not phost.get('patch_current'): + raise wsme.exc.ClientSideError( + _("host-unlock rejected: Not patch current. " + "'sw-patch host-install {}' is required.").format( + hostupdate.displayid)) + + def check_lock(self, hostupdate): + """Check semantics on host-lock.""" + LOG.info("%s ihost check_lock" % hostupdate.displayid) + if hostupdate.action != k_host.ACTION_LOCK: + LOG.error("%s check_lock unexpected action: %s" % + (hostupdate.displayid, hostupdate.action)) + return False + + # Semantic Check: Avoid Lock of Locked Host + if hostupdate.ihost_orig['administrative'] == k_host.ADMIN_LOCKED: + raise wsme.exc.ClientSideError( + _("Avoiding {} action on already " + "'locked' host {}").format( + hostupdate.ihost_patch['action'], + hostupdate.ihost_orig['hostname'])) + + # personality specific lock checks + personality = hostupdate.ihost_patch.get('personality') + if personality == k_host.CONTROLLER: + self.check_lock_controller(hostupdate) + elif personality == k_host.STORAGE: + self.check_lock_storage(hostupdate) + + subfunctions_set = \ + set(hostupdate.ihost_patch[k_host.SUBFUNCTIONS].split(',')) + if k_host.COMPUTE in subfunctions_set: + self.check_lock_compute(hostupdate) + + hostupdate.notify_vim = True + hostupdate.notify_mtce = True + + return True + + def check_force_lock(self, hostupdate): + # personality specific lock checks + personality = hostupdate.ihost_patch.get('personality') + if personality == k_host.CONTROLLER: + self.check_lock_controller(hostupdate, force=True) + + elif personality == k_host.STORAGE: + self.check_lock_storage(hostupdate, force=True) + return True + + @staticmethod + def check_lock_controller(hostupdate, force=False): + """Pre lock semantic checks for controller""" + + LOG.info("%s ihost check_lock_controller" % hostupdate.displayid) + + if utils.get_system_mode() != constants.SYSTEM_MODE_SIMPLEX: + if utils.is_host_active_controller(hostupdate.ihost_orig): + raise wsme.exc.ClientSideError( + _("%s : Rejected: Can not lock an active controller") % + hostupdate.ihost_orig['hostname']) + + if StorageBackendConfig.has_backend_configured( + pecan.request.dbapi, + constants.CINDER_BACKEND_CEPH): + try: + st_nodes = objects.Host.list( + pecan.request.context, + filters={'personality': k_host.STORAGE}) + + except exception.HostNotFound: + # If we don't have any storage nodes we don't need to + # check for quorum. We'll allow the node to be locked. + return + # TODO(oponcea) remove once SM supports in-service config reload + # Allow locking controllers when all storage nodes are locked. + for node in st_nodes: + if node['administrative'] == k_host.ADMIN_UNLOCKED: + break + else: + return + + if not force: + # sm-lock-pre-check + node_name = hostupdate.displayid + response = sm_api.lock_pre_check(pecan.request.context, node_name) + if response: + error_code = response.get('error_code') + if ERR_CODE_LOCK_SOLE_SERVICE_PROVIDER == error_code: + impact_svc_list = response.get('impact_service_list') + svc_list = ','.join(impact_svc_list) + if len(impact_svc_list) > 1: + msg = _("Services {svc_list} are only running on " + "{host}, locking {host} will result " + "service outage. If lock {host} is required, " + "please use \"force lock\" command.").format( + svc_list=svc_list, host=node_name) + else: + msg = _("Service {svc_list} is only running on " + "{host}, locking {host} will result " + "service outage. If lock {host} is required, " + "please use \"force lock\" command.").format( + svc_list=svc_list, host=node_name) + + raise wsme.exc.ClientSideError(msg) + elif "0" != error_code: + raise wsme.exc.ClientSideError( + _("{}").format(response['error_details'])) + + @staticmethod + def _host_configure_check(host_uuid): + # check with systemconfig host//state/configure_check + if pecan.request.systemconfig.host_configure_check(host_uuid): + LOG.info("Configuration check {} passed".format(host_uuid)) + raise wsme.exc.ClientSideError("host_configure_check Passed") + else: + LOG.info("Configuration check {} failed".format(host_uuid)) + raise wsme.exc.ClientSideError("host_configure_check Failed") + + def check_unlock_controller(self, hostupdate, force_unlock=False): + """Pre unlock semantic checks for controller""" + LOG.info("{} host check_unlock_controller".format( + hostupdate.displayid)) + self._host_configure_check(hostupdate.ihost_orig['uuid']) + + def check_unlock_compute(self, hostupdate): + """Check semantics on host-unlock of a compute.""" + LOG.info("%s ihost check_unlock_compute" % hostupdate.displayid) + ihost = hostupdate.ihost_orig + if ihost['invprovision'] is None: + raise wsme.exc.ClientSideError( + _("Can not unlock an unconfigured host {}. Please " + "configure host and wait for Availability State " + "'online' prior to unlock.").format(hostupdate.displayid)) + self._host_configure_check(ihost['uuid']) + + def check_unlock_storage(self, hostupdate): + """Storage unlock semantic checks""" + self._host_configure_check(hostupdate.ihost_orig['uuid']) + + @staticmethod + def check_updates_while_unlocked(hostupdate, delta): + """Check semantics host-update of an unlocked host.""" + + ihost = hostupdate.ihost_patch + if ihost['administrative'] == k_host.ADMIN_UNLOCKED: + deltaset = set(delta) + + restricted_updates = () + if not pecan.request.user_agent.startswith('mtce'): + # Allow mtc to modify the state throughthe REST API. + # Eventually mtc should switch to using the + # conductor API to modify hosts because this check will also + # allow users to modify these states (which is bad). + restricted_updates = ('administrative', + 'availability', + 'operational', + 'subfunction_oper', + 'subfunction_avail', + 'task', 'uptime') + + if deltaset.issubset(restricted_updates): + raise wsme.exc.ClientSideError( + ("Change set %s contains a subset of restricted %s." % + (deltaset, restricted_updates))) + else: + LOG.debug("PASS deltaset=%s restricted_updates=%s" % + (deltaset, restricted_updates)) + + if 'administrative' in delta: + # Transition to unlocked + if ihost['host_action']: + LOG.info("Host: %s Admin state change to: %s " + "Clearing host_action=%s" % + (ihost['uuid'], + ihost['administrative'], + ihost['host_action'])) + hostupdate.ihost_val_update({'host_action': ""}) + pass + + @staticmethod + def check_force_swact(hostupdate): + """Pre swact semantic checks for controller""" + # Allow force-swact to continue + return True + + @staticmethod + def check_reboot(hostupdate): + """Pre reboot semantic checks""" + # Semantic Check: State Dependency: Reboot case + if hostupdate.ihost_orig['administrative'] == k_host.ADMIN_UNLOCKED: + raise wsme.exc.ClientSideError( + _("Can not 'Reboot' an 'unlocked' host {}; " + "Please 'Lock' first").format(hostupdate.displayid)) + + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError(_( + "Rebooting a simplex system is not allowed.")) + return True + + def check_swact(self, hostupdate): + """Pre swact semantic checks for controller""" + + if hostupdate.ihost_orig['personality'] != k_host.CONTROLLER: + raise wsme.exc.ClientSideError( + _("Swact action not allowed for " + "non controller host {}.").format( + hostupdate.ihost_orig['hostname'])) + + if hostupdate.ihost_orig['administrative'] == k_host.ADMIN_LOCKED: + raise wsme.exc.ClientSideError( + _("Controller is Locked ; No services to Swact")) + + if utils.get_system_mode() == constants.SYSTEM_MODE_SIMPLEX: + raise wsme.exc.ClientSideError(_( + "Swact action not allowed for a simplex system.")) + + # check target controller + ihost_ctrs = objects.Host.list( + pecan.request.context, + filters={'personality': k_host.CONTROLLER}) + + for ihost_ctr in ihost_ctrs: + if ihost_ctr.hostname != hostupdate.ihost_orig['hostname']: + if (ihost_ctr.operational != + k_host.OPERATIONAL_ENABLED): + raise wsme.exc.ClientSideError( + _("{} is not enabled and has operational " + "state {}." + "Standby controller must be operationally " + "enabled.").format( + (ihost_ctr.hostname, ihost_ctr.operational))) + + if (ihost_ctr.availability == + k_host.AVAILABILITY_DEGRADED): + health_helper = health.Health( + pecan.request.context, + pecan.request.dbapi) + degrade_alarms = health_helper.get_alarms_degrade( + pecan.request.context, + alarm_ignore_list=[ + fm_constants.FM_ALARM_ID_HA_SERVICE_GROUP_STATE, + fm_constants.FM_ALARM_ID_HA_SERVICE_GROUP_REDUNDANCY, # noqa + fm_constants.FM_ALARM_ID_HA_NODE_LICENSE, + fm_constants.FM_ALARM_ID_HA_COMMUNICATION_FAILURE + ], + entity_instance_id_filter=ihost_ctr.hostname) + if degrade_alarms: + raise wsme.exc.ClientSideError( + _("%s has degraded availability status. Standby " + "controller must be in available status.") % + ihost_ctr.hostname) + + if k_host.COMPUTE in ihost_ctr.subfunctions: + if (ihost_ctr.subfunction_oper != + k_host.OPERATIONAL_ENABLED): + raise wsme.exc.ClientSideError( + _("{} subfunction is not enabled and has " + "operational state {}." + "Standby controller subfunctions {} " + "must all be operationally enabled.").format( + (ihost_ctr.hostname, + ihost_ctr.subfunction_oper, + ihost_ctr.subfunctions))) + + LOG.info("TODO sc_op to check swact (storage_backend, tpm_config" + ", DRBD resizing") + + # Check: Valid Swact action: Pre-Swact Check + response = sm_api.swact_pre_check( + pecan.request.context, + hostupdate.ihost_orig['hostname']) + if response and "0" != response['error_code']: + raise wsme.exc.ClientSideError( + _("%s").format(response['error_details'])) + + def check_lock_storage(self, hostupdate, force=False): + """Pre lock semantic checks for storage""" + LOG.info("%s ihost check_lock_storage" % hostupdate.displayid) + + if (hostupdate.ihost_orig['administrative'] == + k_host.ADMIN_UNLOCKED and + hostupdate.ihost_orig['operational'] == + k_host.OPERATIONAL_ENABLED): + num_monitors, required_monitors, quorum_names = \ + self._ceph.get_monitors_status(pecan.request.dbapi) + + if (hostupdate.ihost_orig['hostname'] in quorum_names and + num_monitors - 1 < required_monitors): + raise wsme.exc.ClientSideError(_( + "Only {} storage monitor available. " + "At least {} unlocked and enabled hosts with monitors " + "are required. Please ensure hosts with monitors are " + "unlocked and enabled - candidates: {}, {}, {}").format( + (num_monitors, constants.MIN_STOR_MONITORS, + k_host.CONTROLLER_0_HOSTNAME, + k_host.CONTROLLER_1_HOSTNAME, + k_host.STORAGE_0_HOSTNAME))) + + # send request to systemconfig to check disable storage + LOG.info("TODO sc_op to perform disable storage checks") + + @staticmethod + def check_lock_compute(hostupdate, force=False): + """Pre lock semantic checks for compute""" + + LOG.info("%s host check_lock_compute" % hostupdate.displayid) + if force: + return + + system = objects.System.get_one(pecan.request.context) + if system.system_mode == constants.SYSTEM_MODE_SIMPLEX: + return + + # send request to systemconfig to check disable storage + LOG.info("TODO sc_op to perform disable storage checks") + + def stage_action(self, action, hostupdate): + """Stage the action to be performed. + """ + LOG.info("%s stage_action %s" % (hostupdate.displayid, action)) + rc = True + if not action or ( + action and action.lower() == k_host.ACTION_NONE): + LOG.error("Unrecognized action perform: %s" % action) + return False + + if (action == k_host.ACTION_UNLOCK or + action == k_host.ACTION_FORCE_UNLOCK): + self._handle_unlock_action(hostupdate) + elif action == k_host.ACTION_LOCK: + self._handle_lock_action(hostupdate) + elif action == k_host.ACTION_FORCE_LOCK: + self._handle_force_lock_action(hostupdate) + elif action == k_host.ACTION_SWACT: + self._stage_swact(hostupdate) + elif action == k_host.ACTION_FORCE_SWACT: + self._stage_force_swact(hostupdate) + elif action == k_host.ACTION_REBOOT: + self._stage_reboot(hostupdate) + elif action == k_host.ACTION_RESET: + self._stage_reset(hostupdate) + elif action == k_host.ACTION_REINSTALL: + self._stage_reinstall(hostupdate) + elif action == k_host.ACTION_POWERON: + self._stage_poweron(hostupdate) + elif action == k_host.ACTION_POWEROFF: + self._stage_poweroff(hostupdate) + elif action == k_host.VIM_SERVICES_ENABLED: + self._handle_vim_services_enabled(hostupdate) + elif action == k_host.VIM_SERVICES_DISABLED: + if not self._handle_vim_services_disabled(hostupdate): + LOG.warn(_("{} exit _handle_vim_services_disabled").format( + hostupdate.ihost_patch['hostname'])) + hostupdate.nextstep = hostupdate.EXIT_RETURN_HOST + rc = False + elif action == k_host.VIM_SERVICES_DISABLE_FAILED: + if not self._handle_vim_services_disable_failed(hostupdate): + LOG.warn( + _("{} Exit _handle_vim_services_disable failed").format( + hostupdate.ihost_patch['hostname'])) + hostupdate.nextstep = hostupdate.EXIT_RETURN_HOST + rc = False + elif action == k_host.VIM_SERVICES_DISABLE_EXTEND: + self._handle_vim_services_disable_extend(hostupdate) + hostupdate.nextstep = hostupdate.EXIT_UPDATE_PREVAL + rc = False + elif action == k_host.VIM_SERVICES_DELETE_FAILED: + self._handle_vim_services_delete_failed(hostupdate) + hostupdate.nextstep = hostupdate.EXIT_UPDATE_PREVAL + rc = False + elif action == k_host.ACTION_SUBFUNCTION_CONFIG: + # Not a mtc action; disable mtc checks and config + self._stage_subfunction_config(hostupdate) + else: + LOG.error("%s Unrecognized action perform: %s" % + (hostupdate.displayid, action)) + rc = False + + if hostupdate.nextstep == hostupdate.EXIT_RETURN_HOST: + LOG.info("%s stage_action aborting request %s %s" % + (hostupdate.displayid, + hostupdate.action, + hostupdate.delta)) + + return rc + + @staticmethod + def _check_subfunction_config(hostupdate): + """Check subfunction config.""" + LOG.info("%s _check_subfunction_config" % hostupdate.displayid) + patched_ihost = hostupdate.ihost_patch + + if patched_ihost['action'] == "subfunction_config": + if (not patched_ihost['subfunctions'] or + patched_ihost['personality'] == + patched_ihost['subfunctions']): + raise wsme.exc.ClientSideError( + _("This host is not configured with a subfunction.")) + + return True + + @staticmethod + def _stage_subfunction_config(hostupdate): + """Stage subfunction config.""" + LOG.info("%s _stage_subfunction_config" % hostupdate.displayid) + + hostupdate.notify_mtce = False + hostupdate.skip_notify_mtce = True + + @staticmethod + def perform_action_subfunction_config(ihost_obj): + """Perform subfunction config via RPC to conductor.""" + LOG.info("%s perform_action_subfunction_config" % + ihost_obj['hostname']) + pecan.request.rpcapi.configure_host(pecan.request.context, + ihost_obj, + do_compute_apply=True) + + @staticmethod + def _stage_reboot(hostupdate): + """Stage reboot action.""" + LOG.info("%s stage_reboot" % hostupdate.displayid) + hostupdate.notify_mtce = True + + def _stage_reinstall(self, hostupdate): + """Stage reinstall action.""" + LOG.info("%s stage_reinstall" % hostupdate.displayid) + + # Remove manifests to enable standard install without manifests + # and enable storage allocation change + pecan.request.rpcapi.remove_host_config( + pecan.request.context, + hostupdate.ihost_orig['uuid']) + + hostupdate.notify_mtce = True + if hostupdate.ihost_orig['personality'] == k_host.STORAGE: + istors = pecan.request.dbapi.istor_get_by_ihost( + hostupdate.ihost_orig['uuid']) + for stor in istors: + istor_obj = objects.storage.get_by_uuid( + pecan.request.context, stor.uuid) + self._ceph.remove_osd_key(istor_obj['osdid']) + + hostupdate.ihost_val_update({k_host.HOST_ACTION_STATE: + k_host.HAS_REINSTALLING}) + + @staticmethod + def _stage_poweron(hostupdate): + """Stage poweron action.""" + LOG.info("%s stage_poweron" % hostupdate.displayid) + hostupdate.notify_mtce = True + + @staticmethod + def _stage_poweroff(hostupdate): + """Stage poweroff action.""" + LOG.info("%s stage_poweroff" % hostupdate.displayid) + hostupdate.notify_mtce = True + + @staticmethod + def _stage_swact(hostupdate): + """Stage swact action.""" + LOG.info("%s stage_swact" % hostupdate.displayid) + hostupdate.notify_mtce = True + + @staticmethod + def _stage_force_swact(hostupdate): + """Stage force-swact action.""" + LOG.info("%s stage_force_swact" % hostupdate.displayid) + hostupdate.notify_mtce = True + + @staticmethod + def _handle_vim_services_enabled(hostupdate): + """Handle VIM services-enabled signal.""" + vim_progress_status = hostupdate.ihost_orig.get( + 'vim_progress_status') or "" + LOG.info("%s received services-enabled task=%s vim_progress_status=%s" + % (hostupdate.displayid, + hostupdate.ihost_orig['task'], + vim_progress_status)) + + if (not vim_progress_status or + not vim_progress_status.startswith( + k_host.VIM_SERVICES_ENABLED)): + hostupdate.notify_availability = k_host.VIM_SERVICES_ENABLED + if (not vim_progress_status or + vim_progress_status == k_host.VIM_SERVICES_DISABLED): + # otherwise allow the audit to clear the error message + hostupdate.ihost_val_update({'vim_progress_status': + k_host.VIM_SERVICES_ENABLED}) + + hostupdate.skip_notify_mtce = True + + @staticmethod + def _handle_vim_services_disabled(hostupdate): + """Handle VIM services-disabled signal.""" + + LOG.info("%s _handle_vim_services_disabled'" % hostupdate.displayid) + ihost = hostupdate.ihost_orig + + hostupdate.ihost_val_update( + {'vim_progress_status': k_host.VIM_SERVICES_DISABLED}) + + ihost_task_string = ihost['host_action'] or "" + if ((ihost_task_string.startswith(k_host.ACTION_LOCK) or + ihost_task_string.startswith(k_host.ACTION_FORCE_LOCK)) and + ihost['administrative'] != k_host.ADMIN_LOCKED): + # passed - skip reset for force-lock + # iHost['host_action'] = k_host.ACTION_LOCK + hostupdate.notify_availability = k_host.VIM_SERVICES_DISABLED + hostupdate.notify_action_lock = True + hostupdate.notify_mtce = True + else: + # return False rather than failing request. + LOG.warn(_("{} Admin action task not Locking or Force Locking " + "upon receipt of 'services-disabled'.").format( + hostupdate.displayid)) + hostupdate.skip_notify_mtce = True + return False + + return True + + @staticmethod + def _handle_vim_services_disable_extend(hostupdate): + """Handle VIM services-disable-extend signal.""" + host_action = hostupdate.ihost_orig['host_action'] or "" + result_reason = hostupdate.ihost_patch.get('vim_progress_status') or "" + LOG.info("%s handle_vim_services_disable_extend " + "host_action=%s reason=%s" % + (hostupdate.displayid, host_action, result_reason)) + + hostupdate.skip_notify_mtce = True + if host_action.startswith(k_host.ACTION_LOCK): + val = {'task': k_host.LOCKING + '-', + 'host_action': k_host.ACTION_LOCK} + hostupdate.ihost_val_prenotify_update(val) + else: + LOG.warn("%s Skip vim services disable extend ihost action=%s" % + (hostupdate.displayid, host_action)) + return False + + LOG.info("services-disable-extend reason=%s" % result_reason) + return True + + @staticmethod + def _handle_vim_services_disable_failed(hostupdate): + """Handle VIM services-disable-failed signal.""" + ihost_task_string = hostupdate.ihost_orig['host_action'] or "" + LOG.info("%s handle_vim_services_disable_failed host_action=%s" % + (hostupdate.displayid, ihost_task_string)) + + result_reason = hostupdate.ihost_patch.get('vim_progress_status') or "" + + if ihost_task_string.startswith(k_host.ACTION_LOCK): + hostupdate.skip_notify_mtce = True + val = {'host_action': '', + 'task': '', + 'vim_progress_status': result_reason} + hostupdate.ihost_val_prenotify_update(val) + hostupdate.ihost_val.update(val) + hostupdate.skip_notify_mtce = True + elif ihost_task_string.startswith(k_host.ACTION_FORCE_LOCK): + # allow mtce to reset the host + hostupdate.notify_mtce = True + hostupdate.notify_action_lock_force = True + else: + hostupdate.skip_notify_mtce = True + LOG.warn("%s Skipping vim services disable notification task=%s" % + (hostupdate.displayid, ihost_task_string)) + return False + + if result_reason: + LOG.info("services-disable-failed reason=%s" % result_reason) + hostupdate.ihost_val_update({'vim_progress_status': + result_reason}) + else: + hostupdate.ihost_val_update({'vim_progress_status': + k_host.VIM_SERVICES_DISABLE_FAILED}) + + return True + + @staticmethod + def _handle_vim_services_delete_failed(hostupdate): + """Handle VIM services-delete-failed signal.""" + + ihost_admin = hostupdate.ihost_orig['administrative'] or "" + result_reason = hostupdate.ihost_patch.get('vim_progress_status') or "" + LOG.info("%s handle_vim_services_delete_failed admin=%s reason=%s" % + (hostupdate.displayid, ihost_admin, result_reason)) + + hostupdate.skip_notify_mtce = True + if ihost_admin.startswith(k_host.ADMIN_LOCKED): + val = {'host_action': '', + 'task': '', + 'vim_progress_status': result_reason} + hostupdate.ihost_val_prenotify_update(val) + # hostupdate.ihost_val.update(val) + else: + LOG.warn("%s Skip vim services delete failed notify admin=%s" % + (hostupdate.displayid, ihost_admin)) + return False + + if result_reason: + hostupdate.ihost_val_prenotify_update({'vim_progress_status': + result_reason}) + else: + hostupdate.ihost_val_prenotify_update( + {'vim_progress_status': k_host.VIM_SERVICES_DELETE_FAILED}) + + LOG.info("services-disable-failed reason=%s" % result_reason) + return True + + @staticmethod + def _stage_reset(hostupdate): + """Handle host-reset action.""" + LOG.info("%s _stage_reset" % hostupdate.displayid) + hostupdate.notify_mtce = True + + def _handle_unlock_action(self, hostupdate): + """Handle host-unlock action.""" + LOG.info("%s _handle_unlock_action" % hostupdate.displayid) + if hostupdate.ihost_patch.get('personality') == k_host.STORAGE: + self._handle_unlock_storage_host(hostupdate) + hostupdate.notify_vim_action = False + hostupdate.notify_mtce = True + val = {'host_action': k_host.ACTION_UNLOCK} + hostupdate.ihost_val_prenotify_update(val) + hostupdate.ihost_val.update(val) + + def _handle_unlock_storage_host(self, hostupdate): + self._ceph.update_crushmap(hostupdate) + + @staticmethod + def _handle_lock_action(hostupdate): + """Handle host-lock action.""" + LOG.info("%s _handle_lock_action" % hostupdate.displayid) + + hostupdate.notify_vim_action = True + hostupdate.skip_notify_mtce = True + val = {'host_action': k_host.ACTION_LOCK} + hostupdate.ihost_val_prenotify_update(val) + hostupdate.ihost_val.update(val) + + @staticmethod + def _handle_force_lock_action(hostupdate): + """Handle host-force-lock action.""" + LOG.info("%s _handle_force_lock_action" % hostupdate.displayid) + + hostupdate.notify_vim_action = True + hostupdate.skip_notify_mtce = True + val = {'host_action': k_host.ACTION_FORCE_LOCK} + hostupdate.ihost_val_prenotify_update(val) + hostupdate.ihost_val.update(val) + + +def _create_node(host, xml_node, personality, is_dynamic_ip): + host_node = et.SubElement(xml_node, 'host') + et.SubElement(host_node, 'personality').text = personality + if personality == k_host.COMPUTE: + et.SubElement(host_node, 'hostname').text = host.hostname + et.SubElement(host_node, 'subfunctions').text = host.subfunctions + + et.SubElement(host_node, 'mgmt_mac').text = host.mgmt_mac + if not is_dynamic_ip: + et.SubElement(host_node, 'mgmt_ip').text = host.mgmt_ip + if host.location is not None and 'locn' in host.location: + et.SubElement(host_node, 'location').text = host.location['locn'] + + pw_on_instruction = _('Uncomment the statement below to power on the host ' + 'automatically through board management.') + host_node.append(et.Comment(pw_on_instruction)) + host_node.append(et.Comment('')) + et.SubElement(host_node, 'bm_type').text = host.bm_type + et.SubElement(host_node, 'bm_username').text = host.bm_username + et.SubElement(host_node, 'bm_password').text = '' + + et.SubElement(host_node, 'boot_device').text = host.boot_device + et.SubElement(host_node, 'rootfs_device').text = host.rootfs_device + et.SubElement(host_node, 'install_output').text = host.install_output + et.SubElement(host_node, 'console').text = host.console + et.SubElement(host_node, 'tboot').text = host.tboot diff --git a/inventory/inventory/inventory/api/controllers/v1/link.py b/inventory/inventory/inventory/api/controllers/v1/link.py new file mode 100644 index 00000000..b4038476 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/link.py @@ -0,0 +1,58 @@ +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from wsme import types as wtypes + +from inventory.api.controllers.v1 import base + + +def build_url(resource, resource_args, bookmark=False, base_url=None): + if base_url is None: + base_url = pecan.request.public_url + + template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s' + # FIXME(lucasagomes): I'm getting a 404 when doing a GET on + # a nested resource that the URL ends with a '/'. + # https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs + template += '%(args)s' if resource_args.startswith('?') else '/%(args)s' + return template % {'url': base_url, 'res': resource, 'args': resource_args} + + +class Link(base.APIBase): + """A link representation.""" + + href = wtypes.text + """The url of a link.""" + + rel = wtypes.text + """The name of a link.""" + + type = wtypes.text + """Indicates the type of document/link.""" + + @staticmethod + def make_link(rel_name, url, resource, resource_args, + bookmark=False, type=wtypes.Unset): + href = build_url(resource, resource_args, + bookmark=bookmark, base_url=url) + return Link(href=href, rel=rel_name, type=type) + + @classmethod + def sample(cls): + sample = cls(href="http://localhost:18002" + "eeaca217-e7d8-47b4-bb41-3f99f20ead81", + rel="bookmark") + return sample diff --git a/inventory/inventory/inventory/api/controllers/v1/lldp_agent.py b/inventory/inventory/inventory/api/controllers/v1/lldp_agent.py new file mode 100644 index 00000000..ea3e81ec --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/lldp_agent.py @@ -0,0 +1,366 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2016 Wind River Systems, Inc. +# + + +import jsonpatch + +import pecan +from pecan import rest + +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import lldp_tlv +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_lldp +from inventory.common import utils as cutils +from inventory import objects + +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class LLDPAgentPatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class LLDPAgent(base.APIBase): + """API representation of an LLDP Agent + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + LLDP agent. + """ + + uuid = types.uuid + "Unique UUID for this port" + + status = wtypes.text + "Represent the status of the lldp agent" + + host_id = int + "Represent the host_id the lldp agent belongs to" + + port_id = int + "Represent the port_id the lldp agent belongs to" + + host_uuid = types.uuid + "Represent the UUID of the host the lldp agent belongs to" + + port_uuid = types.uuid + "Represent the UUID of the port the lldp agent belongs to" + + port_name = wtypes.text + "Represent the name of the port the lldp neighbour belongs to" + + port_namedisplay = wtypes.text + "Represent the display name of the port. Unique per host" + + links = [link.Link] + "Represent a list containing a self link and associated lldp agent links" + + tlvs = [link.Link] + "Links to the collection of LldpNeighbours on this ihost" + + chassis_id = wtypes.text + "Represent the status of the lldp agent" + + port_identifier = wtypes.text + "Represent the LLDP port id of the lldp agent" + + port_description = wtypes.text + "Represent the port description of the lldp agent" + + system_description = wtypes.text + "Represent the status of the lldp agent" + + system_name = wtypes.text + "Represent the status of the lldp agent" + + system_capabilities = wtypes.text + "Represent the status of the lldp agent" + + management_address = wtypes.text + "Represent the status of the lldp agent" + + ttl = wtypes.text + "Represent the time-to-live of the lldp agent" + + dot1_lag = wtypes.text + "Represent the 802.1 link aggregation status of the lldp agent" + + dot1_vlan_names = wtypes.text + "Represent the 802.1 vlan names of the lldp agent" + + dot3_mac_status = wtypes.text + "Represent the 802.3 MAC/PHY status of the lldp agent" + + dot3_max_frame = wtypes.text + "Represent the 802.3 maximum frame size of the lldp agent" + + def __init__(self, **kwargs): + self.fields = objects.LLDPAgent.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_lldp_agent, expand=True): + lldp_agent = LLDPAgent(**rpc_lldp_agent.as_dict()) + if not expand: + lldp_agent.unset_fields_except([ + 'uuid', 'host_id', 'port_id', 'status', 'host_uuid', + 'port_uuid', 'port_name', 'port_namedisplay', + 'created_at', 'updated_at', + k_lldp.LLDP_TLV_TYPE_CHASSIS_ID, + k_lldp.LLDP_TLV_TYPE_PORT_ID, + k_lldp.LLDP_TLV_TYPE_TTL, + k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME, + k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC, + k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP, + k_lldp.LLDP_TLV_TYPE_MGMT_ADDR, + k_lldp.LLDP_TLV_TYPE_PORT_DESC, + k_lldp.LLDP_TLV_TYPE_DOT1_LAG, + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES, + k_lldp.LLDP_TLV_TYPE_DOT3_MAC_STATUS, + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME]) + + # never expose the id attribute + lldp_agent.host_id = wtypes.Unset + lldp_agent.port_id = wtypes.Unset + + lldp_agent.links = [ + link.Link.make_link('self', pecan.request.host_url, + 'lldp_agents', lldp_agent.uuid), + link.Link.make_link('bookmark', pecan.request.host_url, + 'lldp_agents', lldp_agent.uuid, + bookmark=True)] + + if expand: + lldp_agent.tlvs = [ + link.Link.make_link('self', + pecan.request.host_url, + 'lldp_agents', + lldp_agent.uuid + "/tlvs"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'lldp_agents', + lldp_agent.uuid + "/tlvs", + bookmark=True)] + + return lldp_agent + + +class LLDPAgentCollection(collection.Collection): + """API representation of a collection of LldpAgent objects.""" + + lldp_agents = [LLDPAgent] + "A list containing LldpAgent objects" + + def __init__(self, **kwargs): + self._type = 'lldp_agents' + + @classmethod + def convert_with_links(cls, rpc_lldp_agents, limit, url=None, + expand=False, **kwargs): + collection = LLDPAgentCollection() + collection.lldp_agents = [LLDPAgent.convert_with_links(a, expand) + for a in rpc_lldp_agents] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'LLDPAgentController' + + +class LLDPAgentController(rest.RestController): + """REST controller for LldpAgents.""" + + tlvs = lldp_tlv.LLDPTLVController( + from_lldp_agents=True) + "Expose tlvs as a sub-element of LldpAgents" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_ports=False): + self._from_hosts = from_hosts + self._from_ports = from_ports + + def _get_lldp_agents_collection(self, uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_("Host id not specified.")) + + if self._from_ports and not uuid: + raise exception.InvalidParameterValue(_("Port id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.LLDPAgent.get_by_uuid(pecan.request.context, + marker) + + if self._from_hosts: + agents = objects.LLDPAgent.get_by_host( + pecan.request.context, + uuid, limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + + elif self._from_ports: + agents = [] + agent = objects.LLDPAgent.get_by_port(pecan.request.context, uuid) + agents.append(agent) + else: + agents = objects.LLDPAgent.list( + pecan.request.context, + limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + + return LLDPAgentCollection.convert_with_links(agents, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(LLDPAgentCollection, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of lldp agents.""" + return self._get_lldp_agents_collection(uuid, marker, limit, sort_key, + sort_dir) + + @wsme_pecan.wsexpose(LLDPAgentCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of lldp_agents with detail.""" + + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "lldp_agents": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['lldp_agents', 'detail']) + return self._get_lldp_agents_collection(uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(LLDPAgent, types.uuid) + def get_one(self, port_uuid): + """Retrieve information about the given lldp agent.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_lldp_agent = objects.LLDPAgent.get_by_uuid( + pecan.request.context, port_uuid) + return LLDPAgent.convert_with_links(rpc_lldp_agent) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(LLDPAgent, body=LLDPAgent) + def post(self, agent): + """Create a new lldp agent.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + try: + host_uuid = agent.host_uuid + port_uuid = agent.port_uuid + new_agent = objects.LLDPAgent.create( + pecan.request.context, + port_uuid, + host_uuid, + agent.as_dict()) + except exception.InventoryException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Invalid data")) + return agent.convert_with_links(new_agent) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [LLDPAgentPatchType]) + @wsme_pecan.wsexpose(LLDPAgent, types.uuid, + body=[LLDPAgentPatchType]) + def patch(self, uuid, patch): + """Update an existing lldp agent.""" + if self._from_hosts: + raise exception.OperationNotPermitted + if self._from_ports: + raise exception.OperationNotPermitted + + rpc_agent = objects.LLDPAgent.get_by_uuid( + pecan.request.context, uuid) + + # replace ihost_uuid and port_uuid with corresponding + patch_obj = jsonpatch.JsonPatch(patch) + for p in patch_obj: + if p['path'] == '/host_uuid': + p['path'] = '/host_id' + host = objects.Host.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = host.id + + if p['path'] == '/port_uuid': + p['path'] = '/port_id' + try: + port = objects.Port.get_by_uuid( + pecan.request.context, p['value']) + p['value'] = port.id + except exception.InventoryException as e: + LOG.exception(e) + p['value'] = None + + try: + agent = LLDPAgent(**jsonpatch.apply_patch(rpc_agent.as_dict(), + patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Update only the fields that have changed + for field in objects.LLDPAgent.fields: + if rpc_agent[field] != getattr(agent, field): + rpc_agent[field] = getattr(agent, field) + + rpc_agent.save() + return LLDPAgent.convert_with_links(rpc_agent) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, types.uuid, status_code=204) + def delete(self, uuid): + """Delete an lldp agent.""" + if self._from_hosts: + raise exception.OperationNotPermitted + if self._from_ports: + raise exception.OperationNotPermitted + + pecan.request.dbapi.lldp_agent_destroy(uuid) diff --git a/inventory/inventory/inventory/api/controllers/v1/lldp_neighbour.py b/inventory/inventory/inventory/api/controllers/v1/lldp_neighbour.py new file mode 100644 index 00000000..68c11414 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/lldp_neighbour.py @@ -0,0 +1,390 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2016 Wind River Systems, Inc. +# + + +import jsonpatch + +import pecan +from pecan import rest + +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import lldp_tlv +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_lldp +from inventory.common import utils as cutils +from inventory import objects +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class LLDPNeighbourPatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class LLDPNeighbour(base.APIBase): + """API representation of an LLDP Neighbour + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + LLDP neighbour. + """ + + uuid = types.uuid + "Unique UUID for this port" + + msap = wtypes.text + "Represent the MAC service access point of the lldp neighbour" + + host_id = int + "Represent the host_id the lldp neighbour belongs to" + + port_id = int + "Represent the port_id the lldp neighbour belongs to" + + host_uuid = types.uuid + "Represent the UUID of the host the lldp neighbour belongs to" + + port_uuid = types.uuid + "Represent the UUID of the port the lldp neighbour belongs to" + + port_name = wtypes.text + "Represent the name of the port the lldp neighbour belongs to" + + port_namedisplay = wtypes.text + "Represent the display name of the port. Unique per host" + + links = [link.Link] + "Represent a list containing a self link and associated lldp neighbour" + "links" + + tlvs = [link.Link] + "Links to the collection of LldpNeighbours on this ihost" + + chassis_id = wtypes.text + "Represent the status of the lldp neighbour" + + system_description = wtypes.text + "Represent the status of the lldp neighbour" + + system_name = wtypes.text + "Represent the status of the lldp neighbour" + + system_capabilities = wtypes.text + "Represent the status of the lldp neighbour" + + management_address = wtypes.text + "Represent the status of the lldp neighbour" + + port_identifier = wtypes.text + "Represent the port identifier of the lldp neighbour" + + port_description = wtypes.text + "Represent the port description of the lldp neighbour" + + dot1_lag = wtypes.text + "Represent the 802.1 link aggregation status of the lldp neighbour" + + dot1_port_vid = wtypes.text + "Represent the 802.1 port vlan id of the lldp neighbour" + + dot1_vid_digest = wtypes.text + "Represent the 802.1 vlan id digest of the lldp neighbour" + + dot1_management_vid = wtypes.text + "Represent the 802.1 management vlan id of the lldp neighbour" + + dot1_vlan_names = wtypes.text + "Represent the 802.1 vlan names of the lldp neighbour" + + dot1_proto_vids = wtypes.text + "Represent the 802.1 protocol vlan ids of the lldp neighbour" + + dot1_proto_ids = wtypes.text + "Represent the 802.1 protocol ids of the lldp neighbour" + + dot3_mac_status = wtypes.text + "Represent the 802.3 MAC/PHY status of the lldp neighbour" + + dot3_max_frame = wtypes.text + "Represent the 802.3 maximum frame size of the lldp neighbour" + + dot3_power_mdi = wtypes.text + "Represent the 802.3 power mdi status of the lldp neighbour" + + ttl = wtypes.text + "Represent the neighbour time-to-live" + + def __init__(self, **kwargs): + self.fields = objects.LLDPNeighbour.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_lldp_neighbour, expand=True): + lldp_neighbour = LLDPNeighbour(**rpc_lldp_neighbour.as_dict()) + + if not expand: + lldp_neighbour.unset_fields_except([ + 'uuid', 'host_id', 'port_id', 'msap', 'host_uuid', 'port_uuid', + 'port_name', 'port_namedisplay', 'created_at', 'updated_at', + k_lldp.LLDP_TLV_TYPE_CHASSIS_ID, + k_lldp.LLDP_TLV_TYPE_PORT_ID, + k_lldp.LLDP_TLV_TYPE_TTL, + k_lldp.LLDP_TLV_TYPE_SYSTEM_NAME, + k_lldp.LLDP_TLV_TYPE_SYSTEM_DESC, + k_lldp.LLDP_TLV_TYPE_SYSTEM_CAP, + k_lldp.LLDP_TLV_TYPE_MGMT_ADDR, + k_lldp.LLDP_TLV_TYPE_PORT_DESC, + k_lldp.LLDP_TLV_TYPE_DOT1_LAG, + k_lldp.LLDP_TLV_TYPE_DOT1_PORT_VID, + k_lldp.LLDP_TLV_TYPE_DOT1_VID_DIGEST, + k_lldp.LLDP_TLV_TYPE_DOT1_MGMT_VID, + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_VIDS, + k_lldp.LLDP_TLV_TYPE_DOT1_PROTO_IDS, + k_lldp.LLDP_TLV_TYPE_DOT1_VLAN_NAMES, + k_lldp.LLDP_TLV_TYPE_DOT1_VID_DIGEST, + k_lldp.LLDP_TLV_TYPE_DOT3_MAC_STATUS, + k_lldp.LLDP_TLV_TYPE_DOT3_MAX_FRAME, + k_lldp.LLDP_TLV_TYPE_DOT3_POWER_MDI]) + + # never expose the id attribute + lldp_neighbour.host_id = wtypes.Unset + lldp_neighbour.port_id = wtypes.Unset + + lldp_neighbour.links = [ + link.Link.make_link('self', pecan.request.host_url, + 'lldp_neighbours', lldp_neighbour.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'lldp_neighbours', lldp_neighbour.uuid, + bookmark=True)] + + if expand: + lldp_neighbour.tlvs = [ + link.Link.make_link('self', + pecan.request.host_url, + 'lldp_neighbours', + lldp_neighbour.uuid + "/tlvs"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'lldp_neighbours', + lldp_neighbour.uuid + "/tlvs", + bookmark=True)] + + return lldp_neighbour + + +class LLDPNeighbourCollection(collection.Collection): + """API representation of a collection of LldpNeighbour objects.""" + + lldp_neighbours = [LLDPNeighbour] + "A list containing LldpNeighbour objects" + + def __init__(self, **kwargs): + self._type = 'lldp_neighbours' + + @classmethod + def convert_with_links(cls, rpc_lldp_neighbours, limit, url=None, + expand=False, **kwargs): + collection = LLDPNeighbourCollection() + + collection.lldp_neighbours = [LLDPNeighbour.convert_with_links(a, + expand) + for a in rpc_lldp_neighbours] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'LLDPNeighbourController' + + +class LLDPNeighbourController(rest.RestController): + """REST controller for LldpNeighbours.""" + + tlvs = lldp_tlv.LLDPTLVController( + from_lldp_neighbours=True) + "Expose tlvs as a sub-element of LldpNeighbours" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_ports=False): + self._from_hosts = from_hosts + self._from_ports = from_ports + + def _get_lldp_neighbours_collection(self, uuid, marker, limit, sort_key, + sort_dir, expand=False, + resource_url=None): + + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_("Host id not specified.")) + + if self._from_ports and not uuid: + raise exception.InvalidParameterValue(_("Port id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.LLDPNeighbour.get_by_uuid( + pecan.request.context, marker) + + if self._from_hosts: + neighbours = pecan.request.dbapi.lldp_neighbour_get_by_host( + uuid, limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + + elif self._from_ports: + neighbours = pecan.request.dbapi.lldp_neighbour_get_by_port( + uuid, limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + else: + neighbours = pecan.request.dbapi.lldp_neighbour_get_list( + limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + + return LLDPNeighbourCollection.convert_with_links(neighbours, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(LLDPNeighbourCollection, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of lldp neighbours.""" + + return self._get_lldp_neighbours_collection(uuid, marker, limit, + sort_key, sort_dir) + + @wsme_pecan.wsexpose(LLDPNeighbourCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of lldp_neighbours with detail.""" + + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "lldp_neighbours": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['lldp_neighbours', 'detail']) + return self._get_lldp_neighbours_collection(uuid, marker, limit, + sort_key, sort_dir, expand, + resource_url) + + @wsme_pecan.wsexpose(LLDPNeighbour, types.uuid) + def get_one(self, port_uuid): + """Retrieve information about the given lldp neighbour.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_lldp_neighbour = objects.LLDPNeighbour.get_by_uuid( + pecan.request.context, port_uuid) + return LLDPNeighbour.convert_with_links(rpc_lldp_neighbour) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(LLDPNeighbour, body=LLDPNeighbour) + def post(self, neighbour): + """Create a new lldp neighbour.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + try: + host_uuid = neighbour.host_uuid + port_uuid = neighbour.port_uuid + new_neighbour = pecan.request.dbapi.lldp_neighbour_create( + port_uuid, host_uuid, neighbour.as_dict()) + except exception.InventoryException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Invalid data")) + return neighbour.convert_with_links(new_neighbour) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [LLDPNeighbourPatchType]) + @wsme_pecan.wsexpose(LLDPNeighbour, types.uuid, + body=[LLDPNeighbourPatchType]) + def patch(self, uuid, patch): + """Update an existing lldp neighbour.""" + if self._from_hosts: + raise exception.OperationNotPermitted + if self._from_ports: + raise exception.OperationNotPermitted + + rpc_neighbour = objects.LLDPNeighbour.get_by_uuid( + pecan.request.context, uuid) + + # replace host_uuid and port_uuid with corresponding + patch_obj = jsonpatch.JsonPatch(patch) + for p in patch_obj: + if p['path'] == '/host_uuid': + p['path'] = '/host_id' + host = objects.Host.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = host.id + + if p['path'] == '/port_uuid': + p['path'] = '/port_id' + try: + port = objects.Port.get_by_uuid( + pecan.request.context, p['value']) + p['value'] = port.id + except exception.InventoryException as e: + LOG.exception(e) + p['value'] = None + + try: + neighbour = LLDPNeighbour( + **jsonpatch.apply_patch(rpc_neighbour.as_dict(), patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Update only the fields that have changed + for field in objects.LLDPNeighbour.fields: + if rpc_neighbour[field] != getattr(neighbour, field): + rpc_neighbour[field] = getattr(neighbour, field) + + rpc_neighbour.save() + return LLDPNeighbour.convert_with_links(rpc_neighbour) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, types.uuid, status_code=204) + def delete(self, uuid): + """Delete an lldp neighbour.""" + if self._from_hosts: + raise exception.OperationNotPermitted + if self._from_ports: + raise exception.OperationNotPermitted + + pecan.request.dbapi.lldp_neighbour_destroy(uuid) diff --git a/inventory/inventory/inventory/api/controllers/v1/lldp_tlv.py b/inventory/inventory/inventory/api/controllers/v1/lldp_tlv.py new file mode 100644 index 00000000..eb7bd034 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/lldp_tlv.py @@ -0,0 +1,297 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2016-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import jsonpatch + +import pecan +from pecan import rest + +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import utils as cutils +from inventory import objects + +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class LLDPTLVPatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class LLDPTLV(base.APIBase): + """API representation of an LldpTlv + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + LLDP tlv. + """ + + type = wtypes.text + "Represent the type of the lldp tlv" + + value = wtypes.text + "Represent the value of the lldp tlv" + + agent_id = int + "Represent the agent_id the lldp tlv belongs to" + + neighbour_id = int + "Represent the neighbour the lldp tlv belongs to" + + agent_uuid = types.uuid + "Represent the UUID of the agent the lldp tlv belongs to" + + neighbour_uuid = types.uuid + "Represent the UUID of the neighbour the lldp tlv belongs to" + + links = [link.Link] + "Represent a list containing a self link and associated lldp tlv links" + + def __init__(self, **kwargs): + self.fields = objects.LLDPTLV.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_lldp_tlv, expand=True): + lldp_tlv = LLDPTLV(**rpc_lldp_tlv.as_dict()) + if not expand: + lldp_tlv.unset_fields_except(['type', 'value']) + + # never expose the id attribute + lldp_tlv.agent_id = wtypes.Unset + lldp_tlv.neighbour_id = wtypes.Unset + + lldp_tlv.links = [link.Link.make_link('self', pecan.request.host_url, + 'lldp_tlvs', lldp_tlv.type), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'lldp_tlvs', lldp_tlv.type, + bookmark=True)] + return lldp_tlv + + +class LLDPTLVCollection(collection.Collection): + """API representation of a collection of LldpTlv objects.""" + + lldp_tlvs = [LLDPTLV] + "A list containing LldpTlv objects" + + def __init__(self, **kwargs): + self._type = 'lldp_tlvs' + + @classmethod + def convert_with_links(cls, rpc_lldp_tlvs, limit, url=None, + expand=False, **kwargs): + collection = LLDPTLVCollection() + collection.lldp_tlvs = [LLDPTLV.convert_with_links(a, expand) + for a in rpc_lldp_tlvs] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'LLDPTLVController' + + +class LLDPTLVController(rest.RestController): + """REST controller for LldpTlvs.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_lldp_agents=False, from_lldp_neighbours=False): + self._from_lldp_agents = from_lldp_agents + self._from_lldp_neighbours = from_lldp_neighbours + + def _get_lldp_tlvs_collection(self, uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_lldp_agents and not uuid: + raise exception.InvalidParameterValue( + _("LLDP agent id not specified.")) + + if self._from_lldp_neighbours and not uuid: + raise exception.InvalidParameterValue( + _("LLDP neighbour id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.LLDPTLV.get_by_id(pecan.request.context, + marker) + + if self._from_lldp_agents: + tlvs = objects.LLDPTLV.get_by_agent(pecan.request.context, + uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + elif self._from_lldp_neighbours: + tlvs = objects.LLDPTLV.get_by_neighbour( + pecan.request.context, + uuid, limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + else: + tlvs = objects.LLDPTLV.list( + pecan.request.context, + limit, marker_obj, sort_key=sort_key, sort_dir=sort_dir) + + return LLDPTLVCollection.convert_with_links(tlvs, + limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(LLDPTLVCollection, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of lldp tlvs.""" + return self._get_lldp_tlvs_collection(uuid, marker, limit, sort_key, + sort_dir) + + @wsme_pecan.wsexpose(LLDPTLVCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of lldp_tlvs with detail.""" + + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "lldp_tlvs": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['lldp_tlvs', 'detail']) + return self._get_lldp_tlvs_collection(uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(LLDPTLV, int) + def get_one(self, id): + """Retrieve information about the given lldp tlv.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_lldp_tlv = objects.LLDPTLV.get_by_id( + pecan.request.context, id) + return LLDPTLV.convert_with_links(rpc_lldp_tlv) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(LLDPTLV, body=LLDPTLV) + def post(self, tlv): + """Create a new lldp tlv.""" + if self._from_lldp_agents: + raise exception.OperationNotPermitted + + if self._from_lldp_neighbours: + raise exception.OperationNotPermitted + + try: + agent_uuid = tlv.agent_uuid + neighbour_uuid = tlv.neighbour_uuid + new_tlv = pecan.request.dbapi.lldp_tlv_create(tlv.as_dict(), + agent_uuid, + neighbour_uuid) + except exception.InventoryException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Invalid data")) + return tlv.convert_with_links(new_tlv) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [LLDPTLVPatchType]) + @wsme_pecan.wsexpose(LLDPTLV, int, + body=[LLDPTLVPatchType]) + def patch(self, id, patch): + """Update an existing lldp tlv.""" + if self._from_lldp_agents: + raise exception.OperationNotPermitted + if self._from_lldp_neighbours: + raise exception.OperationNotPermitted + + rpc_tlv = objects.LLDPTLV.get_by_id( + pecan.request.context, id) + + # replace agent_uuid and neighbour_uuid with corresponding + patch_obj = jsonpatch.JsonPatch(patch) + for p in patch_obj: + if p['path'] == '/agent_uuid': + p['path'] = '/agent_id' + agent = objects.LLDPAgent.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = agent.id + + if p['path'] == '/neighbour_uuid': + p['path'] = '/neighbour_id' + try: + neighbour = objects.LLDPNeighbour.get_by_uuid( + pecan.request.context, p['value']) + p['value'] = neighbour.id + except exception.InventoryException as e: + LOG.exception(e) + p['value'] = None + + try: + tlv = LLDPTLV( + **jsonpatch.apply_patch(rpc_tlv.as_dict(), patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Update only the fields that have changed + for field in objects.LLDPTLV.fields: + if rpc_tlv[field] != getattr(tlv, field): + rpc_tlv[field] = getattr(tlv, field) + + rpc_tlv.save() + return LLDPTLV.convert_with_links(rpc_tlv) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, int, status_code=204) + def delete(self, id): + """Delete an lldp tlv.""" + if self._from_lldp_agents: + raise exception.OperationNotPermitted + if self._from_lldp_neighbours: + raise exception.OperationNotPermitted + + tlv = objects.LLDPTLV.get_by_id(pecan.request.context, id) + tlv.destroy() + # pecan.request.dbapi.lldp_tlv_destroy(id) diff --git a/inventory/inventory/inventory/api/controllers/v1/memory.py b/inventory/inventory/inventory/api/controllers/v1/memory.py new file mode 100644 index 00000000..20ddb4b8 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/memory.py @@ -0,0 +1,729 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import jsonpatch +import six + +import pecan +from pecan import rest + +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import utils as cutils +from inventory import objects +from oslo_log import log + + +LOG = log.getLogger(__name__) + + +class MemoryPatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class Memory(base.APIBase): + """API representation of host memory. + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of a memory. + """ + + _minimum_platform_reserved_mib = None + + def _get_minimum_platform_reserved_mib(self): + return self._minimum_platform_reserved_mib + + def _set_minimum_platform_reserved_mib(self, value): + if self._minimum_platform_reserved_mib is None: + try: + ihost = objects.Host.get_by_uuid(pecan.request.context, value) + self._minimum_platform_reserved_mib = \ + cutils.get_minimum_platform_reserved_memory(ihost, + self.numa_node) + except exception.HostNotFound as e: + # Change error code because 404 (NotFound) is inappropriate + # response for a POST request to create + e.code = 400 # BadRequest + raise e + elif value == wtypes.Unset: + self._minimum_platform_reserved_mib = wtypes.Unset + + uuid = types.uuid + "Unique UUID for this memory" + + memtotal_mib = int + "Represent the imemory total in MiB" + + memavail_mib = int + "Represent the imemory available in MiB" + + platform_reserved_mib = int + "Represent the imemory platform reserved in MiB" + + hugepages_configured = wtypes.text + "Represent whether huge pages are configured" + + vswitch_hugepages_size_mib = int + "Represent the imemory vswitch huge pages size in MiB" + + vswitch_hugepages_reqd = int + "Represent the imemory vswitch required number of hugepages" + + vswitch_hugepages_nr = int + "Represent the imemory vswitch number of hugepages" + + vswitch_hugepages_avail = int + "Represent the imemory vswitch number of hugepages available" + + vm_hugepages_nr_2M_pending = int + "Represent the imemory vm number of hugepages pending (2M pages)" + + vm_hugepages_nr_2M = int + "Represent the imemory vm number of hugepages (2M pages)" + + vm_hugepages_avail_2M = int + "Represent the imemory vm number of hugepages available (2M pages)" + + vm_hugepages_nr_1G_pending = int + "Represent the imemory vm number of hugepages pending (1G pages)" + + vm_hugepages_nr_1G = int + "Represent the imemory vm number of hugepages (1G pages)" + + vm_hugepages_nr_4K = int + "Represent the imemory vm number of hugepages (4K pages)" + + vm_hugepages_use_1G = wtypes.text + "1G hugepage is supported 'True' or not 'False' " + + vm_hugepages_avail_1G = int + "Represent the imemory vm number of hugepages available (1G pages)" + + vm_hugepages_possible_2M = int + "Represent the total possible number of vm hugepages available (2M pages)" + + vm_hugepages_possible_1G = int + "Represent the total possible number of vm hugepages available (1G pages)" + + minimum_platform_reserved_mib = wsme.wsproperty( + int, + _get_minimum_platform_reserved_mib, + _set_minimum_platform_reserved_mib, + mandatory=True) + "Represent the default platform reserved memory in MiB. API only attribute" + + numa_node = int + "The numa node or zone the imemory. API only attribute" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "This memory's meta data" + + host_id = int + "The ihostid that this imemory belongs to" + + node_id = int + "The nodeId that this imemory belongs to" + + ihost_uuid = types.uuid + "The UUID of the ihost this memory belongs to" + + node_uuid = types.uuid + "The UUID of the node this memory belongs to" + + links = [link.Link] + "A list containing a self link and associated memory links" + + def __init__(self, **kwargs): + self.fields = objects.Memory.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + # API only attributes + self.fields.append('minimum_platform_reserved_mib') + setattr(self, 'minimum_platform_reserved_mib', + kwargs.get('host_id', None)) + + @classmethod + def convert_with_links(cls, rpc_mem, expand=True): + # fields = ['uuid', 'address'] if not expand else None + # memory = imemory.from_rpc_object(rpc_mem, fields) + + memory = Memory(**rpc_mem.as_dict()) + if not expand: + memory.unset_fields_except( + ['uuid', 'memtotal_mib', 'memavail_mib', + 'platform_reserved_mib', 'hugepages_configured', + 'vswitch_hugepages_size_mib', 'vswitch_hugepages_nr', + 'vswitch_hugepages_reqd', + 'vswitch_hugepages_avail', + 'vm_hugepages_nr_2M', + 'vm_hugepages_nr_1G', 'vm_hugepages_use_1G', + 'vm_hugepages_nr_2M_pending', + 'vm_hugepages_avail_2M', + 'vm_hugepages_nr_1G_pending', + 'vm_hugepages_avail_1G', + 'vm_hugepages_nr_4K', + 'vm_hugepages_possible_2M', 'vm_hugepages_possible_1G', + 'numa_node', 'ihost_uuid', 'node_uuid', + 'host_id', 'node_id', + 'capabilities', + 'created_at', 'updated_at', + 'minimum_platform_reserved_mib']) + + # never expose the id attribute + memory.host_id = wtypes.Unset + memory.node_id = wtypes.Unset + + memory.links = [link.Link.make_link('self', pecan.request.host_url, + 'memorys', memory.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'memorys', memory.uuid, + bookmark=True) + ] + return memory + + +class MemoryCollection(collection.Collection): + """API representation of a collection of memorys.""" + + memorys = [Memory] + "A list containing memory objects" + + def __init__(self, **kwargs): + self._type = 'memorys' + + @classmethod + def convert_with_links(cls, memorys, limit, url=None, + expand=False, **kwargs): + collection = MemoryCollection() + collection.memorys = [ + Memory.convert_with_links(n, expand) for n in memorys] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'MemoryController' + + +class MemoryController(rest.RestController): + """REST controller for memorys.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_node=False): + self._from_hosts = from_hosts + self._from_node = from_node + + def _get_memorys_collection(self, i_uuid, node_uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not i_uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + if self._from_node and not i_uuid: + raise exception.InvalidParameterValue(_( + "Node id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.Memory.get_by_uuid(pecan.request.context, + marker) + + if self._from_hosts: + # memorys = pecan.request.dbapi.imemory_get_by_ihost( + memorys = objects.Memory.get_by_host( + pecan.request.context, + i_uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + elif self._from_node: + # memorys = pecan.request.dbapi.imemory_get_by_node( + memorys = objects.Memory.get_by_node( + pecan.request.context, + i_uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if i_uuid and not node_uuid: + # memorys = pecan.request.dbapi.imemory_get_by_ihost( + memorys = objects.Memory.get_by_host( + pecan.request.context, + i_uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + elif i_uuid and node_uuid: # Need ihost_uuid ? + # memorys = pecan.request.dbapi.imemory_get_by_ihost_node( + memorys = objects.Memory.get_by_host_node( + pecan.request.context, + i_uuid, + node_uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + elif node_uuid: + # memorys = pecan.request.dbapi.imemory_get_by_ihost_node( + memorys = objects.Memory.get_by_node( + pecan.request.context, + node_uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + # memorys = pecan.request.dbapi.imemory_get_list( + memorys = objects.Memory.list( + pecan.request.context, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return MemoryCollection.convert_with_links(memorys, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(MemoryCollection, types.uuid, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, ihost_uuid=None, node_uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of memorys.""" + + return self._get_memorys_collection( + ihost_uuid, node_uuid, marker, limit, sort_key, sort_dir) + + @wsme_pecan.wsexpose(MemoryCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, ihost_uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of memorys with detail.""" + # NOTE(lucasagomes): /detail should only work agaist collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "memorys": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['memorys', 'detail']) + return self._get_memorys_collection(ihost_uuid, marker, limit, + sort_key, sort_dir, + expand, resource_url) + + @wsme_pecan.wsexpose(Memory, types.uuid) + def get_one(self, memory_uuid): + """Retrieve information about the given memory.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_mem = objects.Memory.get_by_uuid(pecan.request.context, + memory_uuid) + return Memory.convert_with_links(rpc_mem) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(Memory, body=Memory) + def post(self, memory): + """Create a new memory.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + try: + ihost_uuid = memory.ihost_uuid + new_memory = pecan.request.dbapi.imemory_create(ihost_uuid, + memory.as_dict()) + + except exception.InventoryException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Invalid data")) + return Memory.convert_with_links(new_memory) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [MemoryPatchType]) + @wsme_pecan.wsexpose(Memory, types.uuid, + body=[MemoryPatchType]) + def patch(self, memory_uuid, patch): + """Update an existing memory.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_mem = objects.Memory.get_by_uuid( + pecan.request.context, memory_uuid) + + if 'host_id' in rpc_mem: + ihostId = rpc_mem['host_id'] + else: + ihostId = rpc_mem['ihost_uuid'] + + host_id = pecan.request.dbapi.ihost_get(ihostId) + + vm_hugepages_nr_2M_pending = None + vm_hugepages_nr_1G_pending = None + platform_reserved_mib = None + for p in patch: + if p['path'] == '/platform_reserved_mib': + platform_reserved_mib = p['value'] + if p['path'] == '/vm_hugepages_nr_2M_pending': + vm_hugepages_nr_2M_pending = p['value'] + + if p['path'] == '/vm_hugepages_nr_1G_pending': + vm_hugepages_nr_1G_pending = p['value'] + + # The host must be locked + if host_id: + _check_host(host_id) + else: + raise wsme.exc.ClientSideError(_( + "Hostname or uuid must be defined")) + + try: + # Semantics checks and update hugepage memory accounting + patch = _check_huge_values( + rpc_mem, patch, + vm_hugepages_nr_2M_pending, vm_hugepages_nr_1G_pending) + except wsme.exc.ClientSideError as e: + node = pecan.request.dbapi.node_get(node_id=rpc_mem.node_id) + numa_node = node.numa_node + msg = _('Processor {0}:').format(numa_node) + e.message + raise wsme.exc.ClientSideError(msg) + + # Semantics checks for platform memory + _check_memory(rpc_mem, host_id, platform_reserved_mib, + vm_hugepages_nr_2M_pending, vm_hugepages_nr_1G_pending) + + # only allow patching allocated_function and capabilities + # replace ihost_uuid and node_uuid with corresponding + patch_obj = jsonpatch.JsonPatch(patch) + + for p in patch_obj: + if p['path'] == '/ihost_uuid': + p['path'] = '/host_id' + ihost = objects.Host.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = ihost.id + + if p['path'] == '/node_uuid': + p['path'] = '/node_id' + try: + node = objects.Node.get_by_uuid( + pecan.request.context, p['value']) + p['value'] = node.id + except exception.InventoryException: + p['value'] = None + + try: + memory = Memory(**jsonpatch.apply_patch(rpc_mem.as_dict(), + patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Update only the fields that have changed + for field in objects.Memory.fields: + if rpc_mem[field] != getattr(memory, field): + rpc_mem[field] = getattr(memory, field) + + rpc_mem.save() + return Memory.convert_with_links(rpc_mem) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, types.uuid, status_code=204) + def delete(self, memory_uuid): + """Delete a memory.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + pecan.request.dbapi.imemory_destroy(memory_uuid) + +############## +# UTILS +############## + + +def _update(mem_uuid, mem_values): + + rpc_mem = objects.Memory.get_by_uuid(pecan.request.context, mem_uuid) + if 'host_id' in rpc_mem: + ihostId = rpc_mem['host_id'] + else: + ihostId = rpc_mem['ihost_uuid'] + + host_id = pecan.request.dbapi.ihost_get(ihostId) + + if 'platform_reserved_mib' in mem_values: + platform_reserved_mib = mem_values['platform_reserved_mib'] + + if 'vm_hugepages_nr_2M_pending' in mem_values: + vm_hugepages_nr_2M_pending = mem_values['vm_hugepages_nr_2M_pending'] + + if 'vm_hugepages_nr_1G_pending' in mem_values: + vm_hugepages_nr_1G_pending = mem_values['vm_hugepages_nr_1G_pending'] + + # The host must be locked + if host_id: + _check_host(host_id) + else: + raise wsme.exc.ClientSideError(( + "Hostname or uuid must be defined")) + + # Semantics checks and update hugepage memory accounting + mem_values = _check_huge_values( + rpc_mem, mem_values, + vm_hugepages_nr_2M_pending, vm_hugepages_nr_1G_pending) + + # Semantics checks for platform memory + _check_memory(rpc_mem, host_id, platform_reserved_mib, + vm_hugepages_nr_2M_pending, vm_hugepages_nr_1G_pending) + + # update memory values + pecan.request.dbapi.imemory_update(mem_uuid, mem_values) + + +def _check_host(ihost): + if utils.is_aio_simplex_host_unlocked(ihost): + raise wsme.exc.ClientSideError(_("Host must be locked.")) + elif ihost['administrative'] != 'locked': + unlocked = False + current_ihosts = pecan.request.dbapi.ihost_get_list() + for h in current_ihosts: + if (h['administrative'] != 'locked' and + h['hostname'] != ihost['hostname']): + unlocked = True + if unlocked: + raise wsme.exc.ClientSideError(_("Host must be locked.")) + + +def _check_memory(rpc_mem, ihost, + platform_reserved_mib=None, + vm_hugepages_nr_2M_pending=None, + vm_hugepages_nr_1G_pending=None): + if platform_reserved_mib: + # Check for invalid characters + try: + val = int(platform_reserved_mib) + except ValueError: + raise wsme.exc.ClientSideError(( + "Platform memory must be a number")) + if val < 0: + raise wsme.exc.ClientSideError(( + "Platform memory must be greater than zero")) + + # Check for lower limit + node_id = rpc_mem['node_id'] + node = pecan.request.dbapi.node_get(node_id) + min_platform_memory = \ + cutils.get_minimum_platform_reserved_memory(ihost, node.numa_node) + if int(platform_reserved_mib) < min_platform_memory: + raise wsme.exc.ClientSideError( + _("Platform reserved memory for numa node {} " + "must be greater than the minimum value {}").format( + (node.numa_node, min_platform_memory))) + + # Check if it is within 2/3 percent of the total memory + node_memtotal_mib = rpc_mem['node_memtotal_mib'] + max_platform_reserved = node_memtotal_mib * 2 / 3 + if int(platform_reserved_mib) > max_platform_reserved: + low_core = cutils.is_low_core_system(ihost, pecan.request.dbapi) + required_platform_reserved = \ + cutils.get_required_platform_reserved_memory( + ihost, node.numa_node, low_core) + msg_platform_over = ( + _("Platform reserved memory {} MiB on node {} " + "is not within range [{}, {}]").format( + (int(platform_reserved_mib), + node.numa_node, + required_platform_reserved, + max_platform_reserved))) + + if cutils.is_virtual() or cutils.is_virtual_compute(ihost): + LOG.warn(msg_platform_over) + else: + raise wsme.exc.ClientSideError(msg_platform_over) + + # Check if it is within the total amount of memory + mem_alloc = 0 + if vm_hugepages_nr_2M_pending: + mem_alloc += int(vm_hugepages_nr_2M_pending) * 2 + elif rpc_mem['vm_hugepages_nr_2M']: + mem_alloc += int(rpc_mem['vm_hugepages_nr_2M']) * 2 + if vm_hugepages_nr_1G_pending: + mem_alloc += int(vm_hugepages_nr_1G_pending) * 1000 + elif rpc_mem['vm_hugepages_nr_1G']: + mem_alloc += int(rpc_mem['vm_hugepages_nr_1G']) * 1000 + LOG.debug("vm total=%s" % (mem_alloc)) + + vs_hp_size = rpc_mem['vswitch_hugepages_size_mib'] + vs_hp_nr = rpc_mem['vswitch_hugepages_nr'] + mem_alloc += vs_hp_size * vs_hp_nr + LOG.debug("vs_hp_nr=%s vs_hp_size=%s" % (vs_hp_nr, vs_hp_size)) + LOG.debug("memTotal %s mem_alloc %s" % (node_memtotal_mib, mem_alloc)) + + # Initial configuration defaults mem_alloc to consume 100% of 2M pages, + # so we may marginally exceed available non-huge memory. + # Note there will be some variability in total available memory, + # so we need to allow some tolerance so we do not hit the limit. + avail = node_memtotal_mib - mem_alloc + delta = int(platform_reserved_mib) - avail + mem_thresh = 32 + if int(platform_reserved_mib) > avail + mem_thresh: + msg = (_("Platform reserved memory {} MiB exceeds {} MiB " + "available by {} MiB (2M: {} pages; 1G: {} pages). " + "total memory={} MiB, allocated={} MiB.").format( + (platform_reserved_mib, avail, + delta, delta / 2, delta / 1024, + node_memtotal_mib, mem_alloc))) + raise wsme.exc.ClientSideError(msg) + else: + msg = (_("Platform reserved memory {} MiB, {} MiB available, " + "total memory={} MiB, allocated={} MiB.").format( + platform_reserved_mib, avail, + node_memtotal_mib, mem_alloc)) + LOG.info(msg) + + +def _check_huge_values(rpc_mem, patch, vm_hugepages_nr_2M=None, + vm_hugepages_nr_1G=None): + + if rpc_mem['vm_hugepages_use_1G'] == 'False' and vm_hugepages_nr_1G: + # cannot provision 1G huge pages if the processor does not support them + raise wsme.exc.ClientSideError(_( + "Processor does not support 1G huge pages.")) + + # Check for invalid characters + if vm_hugepages_nr_2M: + try: + val = int(vm_hugepages_nr_2M) + except ValueError: + raise wsme.exc.ClientSideError(_( + "VM huge pages 2M must be a number")) + if int(vm_hugepages_nr_2M) < 0: + raise wsme.exc.ClientSideError(_( + "VM huge pages 2M must be greater than or equal to zero")) + + if vm_hugepages_nr_1G: + try: + val = int(vm_hugepages_nr_1G) + except ValueError: + raise wsme.exc.ClientSideError(_( + "VM huge pages 1G must be a number")) + if val < 0: + raise wsme.exc.ClientSideError(_( + "VM huge pages 1G must be greater than or equal to zero")) + + # Check to make sure that the huge pages aren't over committed + if rpc_mem['vm_hugepages_possible_2M'] is None and vm_hugepages_nr_2M: + raise wsme.exc.ClientSideError(_( + "No available space for 2M huge page allocation")) + + if rpc_mem['vm_hugepages_possible_1G'] is None and vm_hugepages_nr_1G: + raise wsme.exc.ClientSideError(_( + "No available space for 1G huge page allocation")) + + # Update the number of available huge pages + num_2M_for_1G = 512 + + # None == unchanged + if vm_hugepages_nr_1G is not None: + new_1G_pages = int(vm_hugepages_nr_1G) + elif rpc_mem['vm_hugepages_nr_1G_pending']: + new_1G_pages = int(rpc_mem['vm_hugepages_nr_1G_pending']) + elif rpc_mem['vm_hugepages_nr_1G']: + new_1G_pages = int(rpc_mem['vm_hugepages_nr_1G']) + else: + new_1G_pages = 0 + + # None == unchanged + if vm_hugepages_nr_2M is not None: + new_2M_pages = int(vm_hugepages_nr_2M) + elif rpc_mem['vm_hugepages_nr_2M_pending']: + new_2M_pages = int(rpc_mem['vm_hugepages_nr_2M_pending']) + elif rpc_mem['vm_hugepages_nr_2M']: + new_2M_pages = int(rpc_mem['vm_hugepages_nr_2M']) + else: + new_2M_pages = 0 + + LOG.debug('new 2M pages: %s, 1G pages: %s' % (new_2M_pages, new_1G_pages)) + vm_possible_2M = 0 + vm_possible_1G = 0 + if rpc_mem['vm_hugepages_possible_2M']: + vm_possible_2M = int(rpc_mem['vm_hugepages_possible_2M']) + + if rpc_mem['vm_hugepages_possible_1G']: + vm_possible_1G = int(rpc_mem['vm_hugepages_possible_1G']) + + LOG.debug("max possible 2M pages: %s, max possible 1G pages: %s" % + (vm_possible_2M, vm_possible_1G)) + + if vm_possible_2M < new_2M_pages: + msg = _("No available space for 2M huge page allocation, " + "max 2M pages: %d") % vm_possible_2M + raise wsme.exc.ClientSideError(msg) + + if vm_possible_1G < new_1G_pages: + msg = _("No available space for 1G huge page allocation, " + "max 1G pages: %d") % vm_possible_1G + raise wsme.exc.ClientSideError(msg) + + # always use vm_possible_2M to compare, + if vm_possible_2M < (new_2M_pages + new_1G_pages * num_2M_for_1G): + max_1G = int((vm_possible_2M - new_2M_pages) / num_2M_for_1G) + max_2M = vm_possible_2M - new_1G_pages * num_2M_for_1G + if new_2M_pages > 0 and new_1G_pages > 0: + msg = _("No available space for new settings." + "Max 1G pages is {} when 2M is {}, or " + "Max 2M pages is %s when 1G is {}.").format( + max_1G, new_2M_pages, max_2M, new_1G_pages) + elif new_1G_pages > 0: + msg = _("No available space for 1G huge page allocation, " + "max 1G pages: %d") % vm_possible_1G + else: + msg = _("No available space for 2M huge page allocation, " + "max 2M pages: %d") % vm_possible_2M + + raise wsme.exc.ClientSideError(msg) + + return patch diff --git a/inventory/inventory/inventory/api/controllers/v1/node.py b/inventory/inventory/inventory/api/controllers/v1/node.py new file mode 100644 index 00000000..cdca9d62 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/node.py @@ -0,0 +1,261 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# + + +import six + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import cpu +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import memory +from inventory.api.controllers.v1 import port +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory import objects + +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class NodePatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return ['/address', '/host_uuid'] + + +class Node(base.APIBase): + """API representation of a host node. + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of + an node. + """ + + uuid = types.uuid + "Unique UUID for this node" + + numa_node = int + "numa node zone for this node" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "This node's meta data" + + host_id = int + "The hostid that this node belongs to" + + host_uuid = types.uuid + "The UUID of the host this node belongs to" + + links = [link.Link] + "A list containing a self link and associated node links" + + icpus = [link.Link] + "Links to the collection of cpus on this node" + + imemorys = [link.Link] + "Links to the collection of memorys on this node" + + ports = [link.Link] + "Links to the collection of ports on this node" + + def __init__(self, **kwargs): + self.fields = objects.Node.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_node, expand=True): + minimum_fields = ['uuid', 'numa_node', 'capabilities', + 'host_uuid', 'host_id', + 'created_at'] if not expand else None + fields = minimum_fields if not expand else None + + node = Node.from_rpc_object(rpc_node, fields) + + # never expose the host_id attribute + node.host_id = wtypes.Unset + + node.links = [link.Link.make_link('self', pecan.request.host_url, + 'nodes', node.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'nodes', node.uuid, + bookmark=True) + ] + if expand: + node.icpus = [link.Link.make_link('self', + pecan.request.host_url, + 'nodes', + node.uuid + "/cpus"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'nodes', + node.uuid + "/cpus", + bookmark=True) + ] + + node.imemorys = [link.Link.make_link('self', + pecan.request.host_url, + 'nodes', + node.uuid + "/memorys"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'nodes', + node.uuid + "/memorys", + bookmark=True) + ] + + node.ports = [link.Link.make_link('self', + pecan.request.host_url, + 'nodes', + node.uuid + "/ports"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'nodes', + node.uuid + "/ports", + bookmark=True) + ] + + return node + + +class NodeCollection(collection.Collection): + """API representation of a collection of nodes.""" + + nodes = [Node] + "A list containing node objects" + + def __init__(self, **kwargs): + self._type = 'nodes' + + @classmethod + def convert_with_links(cls, rpc_nodes, limit, url=None, + expand=False, **kwargs): + collection = NodeCollection() + collection.nodes = [Node.convert_with_links(p, expand) + for p in rpc_nodes] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'NodeController' + + +class NodeController(rest.RestController): + """REST controller for nodes.""" + + icpus = cpu.CPUController(from_node=True) + "Expose cpus as a sub-element of nodes" + + imemorys = memory.MemoryController(from_node=True) + "Expose memorys as a sub-element of nodes" + + ports = port.PortController(from_node=True) + "Expose ports as a sub-element of nodes" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False): + self._from_hosts = from_hosts + + def _get_nodes_collection(self, host_uuid, marker, limit, sort_key, + sort_dir, expand=False, resource_url=None): + if self._from_hosts and not host_uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.Node.get_by_uuid(pecan.request.context, + marker) + + if host_uuid: + nodes = objects.Node.get_by_host(pecan.request.context, + host_uuid, + limit, + marker=marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + nodes = objects.Node.list(pecan.request.context, + limit, + marker=marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return NodeCollection.convert_with_links(nodes, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(NodeCollection, + types.uuid, types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, host_uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of nodes.""" + + return self._get_nodes_collection(host_uuid, marker, limit, + sort_key, sort_dir) + + @wsme_pecan.wsexpose(NodeCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, host_uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of nodes with detail.""" + # NOTE(lucasagomes): /detail should only work agaist collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "nodes": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['nodes', 'detail']) + return self._get_nodes_collection(host_uuid, + marker, limit, + sort_key, sort_dir, + expand, resource_url) + + @wsme_pecan.wsexpose(Node, types.uuid) + def get_one(self, node_uuid): + """Retrieve information about the given node.""" + + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_node = objects.Node.get_by_uuid(pecan.request.context, node_uuid) + return Node.convert_with_links(rpc_node) diff --git a/inventory/inventory/inventory/api/controllers/v1/pci_device.py b/inventory/inventory/inventory/api/controllers/v1/pci_device.py new file mode 100644 index 00000000..4c182778 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/pci_device.py @@ -0,0 +1,313 @@ +# Copyright (c) 2015-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import jsonpatch +import pecan +from pecan import rest +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import utils as cutils +from inventory import objects +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class PCIDevicePatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class PCIDevice(base.APIBase): + """API representation of an PCI device + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + Pci Device . + """ + + uuid = types.uuid + "Unique UUID for this device" + + type = wtypes.text + "Represent the type of device" + + name = wtypes.text + "Represent the name of the device. Unique per host" + + pciaddr = wtypes.text + "Represent the pci address of the device" + + pclass_id = wtypes.text + "Represent the numerical pci class of the device" + + pvendor_id = wtypes.text + "Represent the numerical pci vendor of the device" + + pdevice_id = wtypes.text + "Represent the numerical pci device of the device" + + pclass = wtypes.text + "Represent the pci class description of the device" + + pvendor = wtypes.text + "Represent the pci vendor description of the device" + + pdevice = wtypes.text + "Represent the pci device description of the device" + + psvendor = wtypes.text + "Represent the pci svendor of the device" + + psdevice = wtypes.text + "Represent the pci sdevice of the device" + + numa_node = int + "Represent the numa node or zone sdevice of the device" + + sriov_totalvfs = int + "The total number of available SR-IOV VFs" + + sriov_numvfs = int + "The number of configured SR-IOV VFs" + + sriov_vfs_pci_address = wtypes.text + "The PCI Addresses of the VFs" + + driver = wtypes.text + "The kernel driver for this device" + + extra_info = wtypes.text + "Extra information for this device" + + host_id = int + "Represent the host_id the device belongs to" + + host_uuid = types.uuid + "Represent the UUID of the host the device belongs to" + + enabled = types.boolean + "Represent the enabled status of the device" + + links = [link.Link] + "Represent a list containing a self link and associated device links" + + def __init__(self, **kwargs): + self.fields = objects.PCIDevice.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_device, expand=True): + device = PCIDevice(**rpc_device.as_dict()) + if not expand: + device.unset_fields_except(['uuid', 'host_id', + 'name', 'pciaddr', 'pclass_id', + 'pvendor_id', 'pdevice_id', 'pclass', + 'pvendor', 'pdevice', 'psvendor', + 'psdevice', 'numa_node', + 'sriov_totalvfs', 'sriov_numvfs', + 'sriov_vfs_pci_address', 'driver', + 'host_uuid', 'enabled', + 'created_at', 'updated_at']) + + # do not expose the id attribute + device.host_id = wtypes.Unset + device.node_id = wtypes.Unset + + device.links = [link.Link.make_link('self', pecan.request.host_url, + 'pci_devices', device.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'pci_devices', device.uuid, + bookmark=True) + ] + return device + + +class PCIDeviceCollection(collection.Collection): + """API representation of a collection of PciDevice objects.""" + + pci_devices = [PCIDevice] + "A list containing PciDevice objects" + + def __init__(self, **kwargs): + self._type = 'pci_devices' + + @classmethod + def convert_with_links(cls, rpc_devices, limit, url=None, + expand=False, **kwargs): + collection = PCIDeviceCollection() + collection.pci_devices = [PCIDevice.convert_with_links(d, expand) + for d in rpc_devices] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'PCIDeviceController' + + +class PCIDeviceController(rest.RestController): + """REST controller for PciDevices.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False): + self._from_hosts = from_hosts + + def _get_pci_devices_collection(self, uuid, marker, limit, sort_key, + sort_dir, expand=False, resource_url=None): + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + marker_obj = None + if marker: + marker_obj = objects.PCIDevice.get_by_uuid( + pecan.request.context, + marker) + if self._from_hosts: + # devices = pecan.request.dbapi.pci_device_get_by_host( + devices = objects.PCIDevice.get_by_host( + pecan.request.context, + uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if uuid: + # devices = pecan.request.dbapi.pci_device_get_by_host( + devices = objects.PCIDevice.get_by_host( + pecan.request.context, + uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + # devices = pecan.request.dbapi.pci_device_get_list( + devices = objects.PCIDevice.list( + pecan.request.context, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return PCIDeviceCollection.convert_with_links(devices, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(PCIDeviceCollection, types.uuid, types.uuid, + int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of devices.""" + return self._get_pci_devices_collection( + uuid, marker, limit, sort_key, sort_dir) + + @wsme_pecan.wsexpose(PCIDeviceCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of devices with detail.""" + + # NOTE: /detail should only work against collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "pci_devices": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['pci_devices', 'detail']) + return self._get_pci_devices_collection(uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(PCIDevice, types.uuid) + def get_one(self, device_uuid): + """Retrieve information about the given device.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_device = objects.PCIDevice.get_by_uuid( + pecan.request.context, device_uuid) + return PCIDevice.convert_with_links(rpc_device) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [PCIDevicePatchType]) + @wsme_pecan.wsexpose(PCIDevice, types.uuid, + body=[PCIDevicePatchType]) + def patch(self, device_uuid, patch): + """Update an existing device.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_device = objects.PCIDevice.get_by_uuid( + pecan.request.context, device_uuid) + + # replace host_uuid and with corresponding + patch_obj = jsonpatch.JsonPatch(patch) + for p in patch_obj: + if p['path'] == '/host_uuid': + p['path'] = '/host_id' + host = objects.Host.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = host.id + + try: + device = PCIDevice(**jsonpatch.apply_patch(rpc_device.as_dict(), + patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Semantic checks + host = objects.Host.get_by_uuid(pecan.request.context, + device.host_id) + _check_host(host) + + # Update fields that have changed + for field in objects.PCIDevice.fields: + if rpc_device[field] != getattr(device, field): + _check_field(field) + rpc_device[field] = getattr(device, field) + + rpc_device.save() + return PCIDevice.convert_with_links(rpc_device) + + +def _check_host(host): + if utils.is_aio_simplex_host_unlocked(host): + raise wsme.exc.ClientSideError(_('Host must be locked.')) + elif host.administrative != k_host.ADMIN_LOCKED and not \ + utils.is_host_simplex_controller(host): + raise wsme.exc.ClientSideError(_('Host must be locked.')) + if k_host.COMPUTE not in host.subfunctions: + raise wsme.exc.ClientSideError( + _('Can only modify compute node cores.')) + + +def _check_field(field): + if field not in ["enabled", "name"]: + raise wsme.exc.ClientSideError( + _('Modifying %s attribute restricted') % field) diff --git a/inventory/inventory/inventory/api/controllers/v1/port.py b/inventory/inventory/inventory/api/controllers/v1/port.py new file mode 100644 index 00000000..643f4f05 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/port.py @@ -0,0 +1,334 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# + + +import six + +import pecan +from pecan import rest + +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import lldp_agent +from inventory.api.controllers.v1 import lldp_neighbour +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory import objects + +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class PortPatchType(types.JsonPatchType): + + @staticmethod + def mandatory_attrs(): + return [] + + +class Port(base.APIBase): + """API representation of a host port + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + port. + """ + uuid = types.uuid + "Unique UUID for this port" + + type = wtypes.text + "Represent the type of port" + + name = wtypes.text + "Represent the name of the port. Unique per host" + + namedisplay = wtypes.text + "Represent the display name of the port. Unique per host" + + pciaddr = wtypes.text + "Represent the pci address of the port" + + dev_id = int + "The unique identifier of PCI device" + + pclass = wtypes.text + "Represent the pci class of the port" + + pvendor = wtypes.text + "Represent the pci vendor of the port" + + pdevice = wtypes.text + "Represent the pci device of the port" + + psvendor = wtypes.text + "Represent the pci svendor of the port" + + psdevice = wtypes.text + "Represent the pci sdevice of the port" + + numa_node = int + "Represent the numa node or zone sdevice of the port" + + sriov_totalvfs = int + "The total number of available SR-IOV VFs" + + sriov_numvfs = int + "The number of configured SR-IOV VFs" + + sriov_vfs_pci_address = wtypes.text + "The PCI Addresses of the VFs" + + driver = wtypes.text + "The kernel driver for this device" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "Represent meta data of the port" + + host_id = int + "Represent the host_id the port belongs to" + + interface_id = int + "Represent the interface_id the port belongs to" + + dpdksupport = bool + "Represent whether or not the port supports DPDK acceleration" + + host_uuid = types.uuid + "Represent the UUID of the host the port belongs to" + + interface_uuid = types.uuid + "Represent the UUID of the interface the port belongs to" + + node_uuid = types.uuid + "Represent the UUID of the node the port belongs to" + + links = [link.Link] + "Represent a list containing a self link and associated port links" + + lldp_agents = [link.Link] + "Links to the collection of LldpAgents on this port" + + lldp_neighbours = [link.Link] + "Links to the collection of LldpNeighbours on this port" + + def __init__(self, **kwargs): + self.fields = objects.Port.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_port, expand=True): + port = Port(**rpc_port.as_dict()) + if not expand: + port.unset_fields_except(['uuid', 'host_id', 'node_id', + 'interface_id', 'type', 'name', + 'namedisplay', 'pciaddr', 'dev_id', + 'pclass', 'pvendor', 'pdevice', + 'psvendor', 'psdevice', 'numa_node', + 'sriov_totalvfs', 'sriov_numvfs', + 'sriov_vfs_pci_address', 'driver', + 'capabilities', + 'host_uuid', 'interface_uuid', + 'node_uuid', 'dpdksupport', + 'created_at', 'updated_at']) + + # never expose the id attribute + port.host_id = wtypes.Unset + port.interface_id = wtypes.Unset + port.node_id = wtypes.Unset + + port.links = [link.Link.make_link('self', pecan.request.host_url, + 'ports', port.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'ports', port.uuid, + bookmark=True) + ] + + port.lldp_agents = [link.Link.make_link('self', + pecan.request.host_url, + 'ports', + port.uuid + "/lldp_agents"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'ports', + port.uuid + "/lldp_agents", + bookmark=True) + ] + + port.lldp_neighbours = [ + link.Link.make_link('self', + pecan.request.host_url, + 'ports', + port.uuid + "/lldp_neighbors"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'ports', + port.uuid + "/lldp_neighbors", + bookmark=True) + ] + + return port + + +class PortCollection(collection.Collection): + """API representation of a collection of Port objects.""" + + ports = [Port] + "A list containing Port objects" + + def __init__(self, **kwargs): + self._type = 'ports' + + @classmethod + def convert_with_links(cls, rpc_ports, limit, url=None, + expand=False, **kwargs): + collection = PortCollection() + collection.ports = [Port.convert_with_links(p, expand) + for p in rpc_ports] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class PortController(rest.RestController): + """REST controller for Ports.""" + + lldp_agents = lldp_agent.LLDPAgentController( + from_ports=True) + "Expose lldp_agents as a sub-element of ports" + + lldp_neighbours = lldp_neighbour.LLDPNeighbourController( + from_ports=True) + "Expose lldp_neighbours as a sub-element of ports" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_iinterface=False, + from_node=False): + self._from_hosts = from_hosts + self._from_iinterface = from_iinterface + self._from_node = from_node + + def _get_ports_collection(self, uuid, interface_uuid, node_uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + if self._from_iinterface and not uuid: + raise exception.InvalidParameterValue(_( + "Interface id not specified.")) + + if self._from_node and not uuid: + raise exception.InvalidParameterValue(_( + "node id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.Port.get_by_uuid( + pecan.request.context, + marker) + + if self._from_hosts: + ports = objects.Port.get_by_host( + pecan.request.context, + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + elif self._from_node: + ports = objects.Port.get_by_numa_node( + pecan.request.context, + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if uuid and not interface_uuid: + ports = objects.Port.get_by_host( + pecan.request.context, + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + ports = objects.Port.list( + pecan.request.context, + limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return PortCollection.convert_with_links(ports, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(PortCollection, types.uuid, types.uuid, + types.uuid, types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, interface_uuid=None, node_uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of ports.""" + + return self._get_ports_collection(uuid, + interface_uuid, + node_uuid, + marker, limit, sort_key, sort_dir) + + @wsme_pecan.wsexpose(PortCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of ports with detail.""" + + # NOTE(lucasagomes): /detail should only work against collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "ports": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['ports', 'detail']) + return self._get_ports_collection(uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(Port, types.uuid) + def get_one(self, port_uuid): + """Retrieve information about the given port.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_port = objects.Port.get_by_uuid( + pecan.request.context, port_uuid) + return Port.convert_with_links(rpc_port) diff --git a/inventory/inventory/inventory/api/controllers/v1/query.py b/inventory/inventory/inventory/api/controllers/v1/query.py new file mode 100644 index 00000000..976bfb9d --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/query.py @@ -0,0 +1,168 @@ +# coding: utf-8 +# Copyright © 2012 New Dream Network, LLC (DreamHost) +# Copyright 2013 IBM Corp. +# Copyright © 2013 eNovance +# Copyright Ericsson AB 2013. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import ast +import functools +import inspect +from inventory.common.i18n import _ +from oslo_log import log +from oslo_utils import strutils +from oslo_utils import timeutils +import six +import wsme +from wsme import types as wtypes + +LOG = log.getLogger(__name__) + +operation_kind = wtypes.Enum(str, 'lt', 'le', 'eq', 'ne', 'ge', 'gt') + + +class _Base(wtypes.Base): + + @classmethod + def from_db_model(cls, m): + return cls(**(m.as_dict())) + + @classmethod + def from_db_and_links(cls, m, links): + return cls(links=links, **(m.as_dict())) + + def as_dict(self, db_model): + valid_keys = inspect.getargspec(db_model.__init__)[0] + if 'self' in valid_keys: + valid_keys.remove('self') + return self.as_dict_from_keys(valid_keys) + + def as_dict_from_keys(self, keys): + return dict((k, getattr(self, k)) + for k in keys + if hasattr(self, k) and + getattr(self, k) != wsme.Unset) + + +class Query(_Base): + """Query filter. + """ + + # The data types supported by the query. + _supported_types = ['integer', 'float', 'string', 'boolean'] + + # Functions to convert the data field to the correct type. + _type_converters = {'integer': int, + 'float': float, + 'boolean': functools.partial( + strutils.bool_from_string, strict=True), + 'string': six.text_type, + 'datetime': timeutils.parse_isotime} + + _op = None # provide a default + + def get_op(self): + return self._op or 'eq' + + def set_op(self, value): + self._op = value + + field = wtypes.text + "The name of the field to test" + + # op = wsme.wsattr(operation_kind, default='eq') + # this ^ doesn't seem to work. + op = wsme.wsproperty(operation_kind, get_op, set_op) + "The comparison operator. Defaults to 'eq'." + + value = wtypes.text + "The value to compare against the stored data" + + type = wtypes.text + "The data type of value to compare against the stored data" + + def __repr__(self): + # for logging calls + return '' % (self.field, + self.op, + self.value, + self.type) + + @classmethod + def sample(cls): + return cls(field='resource_id', + op='eq', + value='bd9431c1-8d69-4ad3-803a-8d4a6b89fd36', + type='string' + ) + + def as_dict(self): + return self.as_dict_from_keys(['field', 'op', 'type', 'value']) + + def _get_value_as_type(self, forced_type=None): + """Convert metadata value to the specified data type. + + This method is called during metadata query to help convert the + querying metadata to the data type specified by user. If there is no + data type given, the metadata will be parsed by ast.literal_eval to + try to do a smart converting. + + NOTE (flwang) Using "_" as prefix to avoid an InvocationError raised + from wsmeext/sphinxext.py. It's OK to call it outside the Query class. + Because the "public" side of that class is actually the outside of the + API, and the "private" side is the API implementation. The method is + only used in the API implementation, so it's OK. + + :returns: metadata value converted with the specified data type. + """ + type = forced_type or self.type + try: + converted_value = self.value + if not type: + try: + converted_value = ast.literal_eval(self.value) + except (ValueError, SyntaxError): + msg = _('Failed to convert the metadata value %s' + ' automatically') % (self.value) + LOG.debug(msg) + else: + if type not in self._supported_types: + # Types must be explicitly declared so the + # correct type converter may be used. Subclasses + # of Query may define _supported_types and + # _type_converters to define their own types. + raise TypeError() + converted_value = self._type_converters[type](self.value) + except ValueError: + msg = _('Failed to convert the value %(value)s' + ' to the expected data type %(type)s.') % \ + {'value': self.value, 'type': type} + raise wsme.exc.ClientSideError(msg) + except TypeError: + msg = _('The data type %(type)s is not supported. The supported' + ' data type list is: %(supported)s') % \ + {'type': type, 'supported': self._supported_types} + raise wsme.exc.ClientSideError(msg) + except Exception: + msg = _('Unexpected exception converting %(value)s to' + ' the expected data type %(type)s.') % \ + {'value': self.value, 'type': type} + raise wsme.exc.ClientSideError(msg) + return converted_value diff --git a/inventory/inventory/inventory/api/controllers/v1/sensor.py b/inventory/inventory/inventory/api/controllers/v1/sensor.py new file mode 100644 index 00000000..05a5b47e --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/sensor.py @@ -0,0 +1,586 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import copy +import jsonpatch +import pecan +from pecan import rest +import six +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import constants +from inventory.common import exception +from inventory.common import hwmon_api +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import utils as cutils +from inventory import objects +from oslo_log import log + + +LOG = log.getLogger(__name__) + + +class SensorPatchType(types.JsonPatchType): + @staticmethod + def mandatory_attrs(): + return [] + + +class Sensor(base.APIBase): + """API representation of an Sensor + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + sensor. + """ + + uuid = types.uuid + "Unique UUID for this sensor" + + sensorname = wtypes.text + "Represent the name of the sensor. Unique with path per host" + + path = wtypes.text + "Represent the path of the sensor. Unique with sensorname per host" + + sensortype = wtypes.text + "Represent the type of sensor. e.g. Temperature, WatchDog" + + datatype = wtypes.text + "Represent the entity monitored. e.g. discrete, analog" + + status = wtypes.text + "Represent current sensor status: ok, minor, major, critical, disabled" + + state = wtypes.text + "Represent the current state of the sensor" + + state_requested = wtypes.text + "Represent the requested state of the sensor" + + audit_interval = int + "Represent the audit_interval of the sensor." + + algorithm = wtypes.text + "Represent the algorithm of the sensor." + + actions_minor = wtypes.text + "Represent the minor configured actions of the sensor. CSV." + + actions_major = wtypes.text + "Represent the major configured actions of the sensor. CSV." + + actions_critical = wtypes.text + "Represent the critical configured actions of the sensor. CSV." + + suppress = wtypes.text + "Represent supress sensor if True, otherwise not suppress sensor" + + value = wtypes.text + "Represent current value of the discrete sensor" + + unit_base = wtypes.text + "Represent the unit base of the analog sensor e.g. revolutions" + + unit_modifier = wtypes.text + "Represent the unit modifier of the analog sensor e.g. 10**2" + + unit_rate = wtypes.text + "Represent the unit rate of the sensor e.g. /minute" + + t_minor_lower = wtypes.text + "Represent the minor lower threshold of the analog sensor" + + t_minor_upper = wtypes.text + "Represent the minor upper threshold of the analog sensor" + + t_major_lower = wtypes.text + "Represent the major lower threshold of the analog sensor" + + t_major_upper = wtypes.text + "Represent the major upper threshold of the analog sensor" + + t_critical_lower = wtypes.text + "Represent the critical lower threshold of the analog sensor" + + t_critical_upper = wtypes.text + "Represent the critical upper threshold of the analog sensor" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "Represent meta data of the sensor" + + host_id = int + "Represent the host_id the sensor belongs to" + + sensorgroup_id = int + "Represent the sensorgroup_id the sensor belongs to" + + host_uuid = types.uuid + "Represent the UUID of the host the sensor belongs to" + + sensorgroup_uuid = types.uuid + "Represent the UUID of the sensorgroup the sensor belongs to" + + links = [link.Link] + "Represent a list containing a self link and associated sensor links" + + def __init__(self, **kwargs): + self.fields = objects.Sensor.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_sensor, expand=True): + + sensor = Sensor(**rpc_sensor.as_dict()) + + sensor_fields_common = ['uuid', 'host_id', 'sensorgroup_id', + 'sensortype', 'datatype', + 'sensorname', 'path', + + 'status', + 'state', 'state_requested', + 'sensor_action_requested', + 'actions_minor', + 'actions_major', + 'actions_critical', + + 'suppress', + 'audit_interval', + 'algorithm', + 'capabilities', + 'host_uuid', 'sensorgroup_uuid', + 'created_at', 'updated_at', ] + + sensor_fields_analog = ['unit_base', + 'unit_modifier', + 'unit_rate', + + 't_minor_lower', + 't_minor_upper', + 't_major_lower', + 't_major_upper', + 't_critical_lower', + 't_critical_upper', ] + + if rpc_sensor.datatype == 'discrete': + sensor_fields = sensor_fields_common + elif rpc_sensor.datatype == 'analog': + sensor_fields = sensor_fields_common + sensor_fields_analog + else: + LOG.error(_("Invalid datatype={}").format(rpc_sensor.datatype)) + + if not expand: + sensor.unset_fields_except(sensor_fields) + + # never expose the id attribute + sensor.host_id = wtypes.Unset + sensor.sensorgroup_id = wtypes.Unset + + sensor.links = [link.Link.make_link('self', pecan.request.host_url, + 'sensors', sensor.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'sensors', sensor.uuid, + bookmark=True) + ] + return sensor + + +class SensorCollection(collection.Collection): + """API representation of a collection of Sensor objects.""" + + sensors = [Sensor] + "A list containing Sensor objects" + + def __init__(self, **kwargs): + self._type = 'sensors' + + @classmethod + def convert_with_links(cls, rpc_sensors, limit, url=None, + expand=False, **kwargs): + collection = SensorCollection() + collection.sensors = [Sensor.convert_with_links(p, expand) + for p in rpc_sensors] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'SensorController' + + +class SensorController(rest.RestController): + """REST controller for Sensors.""" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self, from_hosts=False, from_sensorgroup=False): + self._from_hosts = from_hosts + self._from_sensorgroup = from_sensorgroup + self._api_token = None + self._hwmon_address = k_host.LOCALHOST_HOSTNAME + self._hwmon_port = constants.HWMON_PORT + + def _get_sensors_collection(self, uuid, sensorgroup_uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + if self._from_sensorgroup and not uuid: + raise exception.InvalidParameterValue(_( + "SensorGroup id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.Sensor.get_by_uuid( + pecan.request.context, + marker) + + if self._from_hosts: + sensors = pecan.request.dbapi.sensor_get_by_host( + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + LOG.debug("dbapi.sensor_get_by_host=%s" % sensors) + elif self._from_sensorgroup: + sensors = pecan.request.dbapi.sensor_get_by_sensorgroup( + uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + LOG.debug("dbapi.sensor_get_by_sensorgroup=%s" % sensors) + else: + if uuid and not sensorgroup_uuid: + sensors = pecan.request.dbapi.sensor_get_by_host( + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + LOG.debug("dbapi.sensor_get_by_host=%s" % sensors) + elif uuid and sensorgroup_uuid: # Need ihost_uuid ? + sensors = pecan.request.dbapi.sensor_get_by_host_sensorgroup( + uuid, + sensorgroup_uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + LOG.debug("dbapi.sensor_get_by_host_sensorgroup=%s" % + sensors) + + elif sensorgroup_uuid: # Need ihost_uuid ? + sensors = pecan.request.dbapi.sensor_get_by_host_sensorgroup( + uuid, # None + sensorgroup_uuid, + limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + else: + sensors = pecan.request.dbapi.sensor_get_list( + limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return SensorCollection.convert_with_links(sensors, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(SensorCollection, types.uuid, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, sensorgroup_uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of sensors.""" + + return self._get_sensors_collection(uuid, sensorgroup_uuid, + marker, limit, + sort_key, sort_dir) + + @wsme_pecan.wsexpose(SensorCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of sensors with detail.""" + + # NOTE(lucasagomes): /detail should only work against collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "sensors": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['sensors', 'detail']) + return self._get_sensors_collection(uuid, marker, limit, sort_key, + sort_dir, expand, resource_url) + + @wsme_pecan.wsexpose(Sensor, types.uuid) + def get_one(self, sensor_uuid): + """Retrieve information about the given sensor.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_sensor = objects.Sensor.get_by_uuid( + pecan.request.context, sensor_uuid) + + if rpc_sensor.datatype == 'discrete': + rpc_sensor = objects.SensorDiscrete.get_by_uuid( + pecan.request.context, sensor_uuid) + elif rpc_sensor.datatype == 'analog': + rpc_sensor = objects.SensorAnalog.get_by_uuid( + pecan.request.context, sensor_uuid) + else: + LOG.error(_("Invalid datatype={}").format(rpc_sensor.datatype)) + + return Sensor.convert_with_links(rpc_sensor) + + @staticmethod + def _new_sensor_semantic_checks(sensor): + datatype = sensor.as_dict().get('datatype') or "" + sensortype = sensor.as_dict().get('sensortype') or "" + if not (datatype and sensortype): + raise wsme.exc.ClientSideError(_("sensor-add Cannot " + "add a sensor " + "without a valid datatype " + "and sensortype.")) + + if datatype not in constants.SENSOR_DATATYPE_VALID_LIST: + raise wsme.exc.ClientSideError( + _("sensor datatype must be one of %s.") % + constants.SENSOR_DATATYPE_VALID_LIST) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(Sensor, body=Sensor) + def post(self, sensor): + """Create a new sensor.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + self._new_sensor_semantic_checks(sensor) + try: + ihost = pecan.request.dbapi.host_get(sensor.host_uuid) + + if hasattr(sensor, 'datatype'): + if sensor.datatype == 'discrete': + new_sensor = pecan.request.dbapi.sensor_discrete_create( + ihost.id, sensor.as_dict()) + elif sensor.datatype == 'analog': + new_sensor = pecan.request.dbapi.sensor_analog_create( + ihost.id, sensor.as_dict()) + else: + raise wsme.exc.ClientSideError( + _("Invalid datatype. {}").format(sensor.datatype)) + else: + raise wsme.exc.ClientSideError(_("Unspecified datatype.")) + + except exception.InventoryException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Invalid data")) + return sensor.convert_with_links(new_sensor) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [SensorPatchType]) + @wsme_pecan.wsexpose(Sensor, types.uuid, + body=[SensorPatchType]) + def patch(self, sensor_uuid, patch): + """Update an existing sensor.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rpc_sensor = objects.Sensor.get_by_uuid(pecan.request.context, + sensor_uuid) + if rpc_sensor.datatype == 'discrete': + rpc_sensor = objects.SensorDiscrete.get_by_uuid( + pecan.request.context, sensor_uuid) + elif rpc_sensor.datatype == 'analog': + rpc_sensor = objects.SensorAnalog.get_by_uuid( + pecan.request.context, sensor_uuid) + else: + raise wsme.exc.ClientSideError(_("Invalid datatype={}").format( + rpc_sensor.datatype)) + + rpc_sensor_orig = copy.deepcopy(rpc_sensor) + + # replace ihost_uuid and sensorgroup_uuid with corresponding + utils.validate_patch(patch) + patch_obj = jsonpatch.JsonPatch(patch) + my_host_uuid = None + for p in patch_obj: + if p['path'] == '/host_uuid': + p['path'] = '/host_id' + host = objects.Host.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = host.id + my_host_uuid = host.uuid + + if p['path'] == '/sensorgroup_uuid': + p['path'] = '/sensorgroup_id' + try: + sensorgroup = objects.sensorgroup.get_by_uuid( + pecan.request.context, p['value']) + p['value'] = sensorgroup.id + LOG.info("sensorgroup_uuid=%s id=%s" % (p['value'], + sensorgroup.id)) + except exception.InventoryException: + p['value'] = None + + try: + sensor = Sensor(**jsonpatch.apply_patch(rpc_sensor.as_dict(), + patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Update only the fields that have changed + if rpc_sensor.datatype == 'discrete': + fields = objects.SensorDiscrete.fields + else: + fields = objects.SensorAnalog.fields + + for field in fields: + if rpc_sensor[field] != getattr(sensor, field): + rpc_sensor[field] = getattr(sensor, field) + + delta = rpc_sensor.obj_what_changed() + sensor_suppress_attrs = ['suppress'] + force_action = False + if any(x in delta for x in sensor_suppress_attrs): + valid_suppress = ['True', 'False', 'true', 'false', 'force_action'] + if rpc_sensor.suppress.lower() not in valid_suppress: + raise wsme.exc.ClientSideError(_("Invalid suppress value, " + "select 'True' or 'False'")) + elif rpc_sensor.suppress.lower() == 'force_action': + LOG.info("suppress=%s" % rpc_sensor.suppress.lower()) + rpc_sensor.suppress = rpc_sensor_orig.suppress + force_action = True + + self._semantic_modifiable_fields(patch_obj, force_action) + + if not pecan.request.user_agent.startswith('hwmon'): + hwmon_sensor = cutils.removekeys_nonhwmon( + rpc_sensor.as_dict()) + + if not my_host_uuid: + host = objects.Host.get_by_uuid(pecan.request.context, + rpc_sensor.host_id) + my_host_uuid = host.uuid + LOG.warn("Missing host_uuid updated=%s" % my_host_uuid) + + hwmon_sensor.update({'host_uuid': my_host_uuid}) + + hwmon_response = hwmon_api.sensor_modify( + self._api_token, self._hwmon_address, self._hwmon_port, + hwmon_sensor, + constants.HWMON_DEFAULT_TIMEOUT_IN_SECS) + + if not hwmon_response: + hwmon_response = {'status': 'fail', + 'reason': 'no response', + 'action': 'retry'} + + if hwmon_response['status'] != 'pass': + msg = _("HWMON has returned with a status of {}, reason: {}, " + "recommended action: {}").format( + hwmon_response.get('status'), + hwmon_response.get('reason'), + hwmon_response.get('action')) + + if force_action: + LOG.error(msg) + else: + raise wsme.exc.ClientSideError(msg) + + rpc_sensor.save() + + return Sensor.convert_with_links(rpc_sensor) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, types.uuid, status_code=204) + def delete(self, sensor_uuid): + """Delete a sensor.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + pecan.request.dbapi.sensor_destroy(sensor_uuid) + + @staticmethod + def _semantic_modifiable_fields(patch_obj, force_action=False): + # Prevent auto populated fields from being updated + state_rel_path = ['/uuid', '/id', '/host_id', '/datatype', + '/sensortype'] + if any(p['path'] in state_rel_path for p in patch_obj): + raise wsme.exc.ClientSideError(_("The following fields can not be " + "modified: %s ") % state_rel_path) + + state_rel_path = ['/actions_critical', + '/actions_major', + '/actions_minor'] + if any(p['path'] in state_rel_path for p in patch_obj): + raise wsme.exc.ClientSideError( + _("The following fields can only be modified at the " + "sensorgroup level: %s") % state_rel_path) + + if not (pecan.request.user_agent.startswith('hwmon') or force_action): + state_rel_path = ['/sensorname', + '/path', + '/status', + '/state', + '/possible_states', + '/algorithm', + '/actions_critical_choices', + '/actions_major_choices', + '/actions_minor_choices', + '/unit_base', + '/unit_modifier', + '/unit_rate', + '/t_minor_lower', + '/t_minor_upper', + '/t_major_lower', + '/t_major_upper', + '/t_critical_lower', + '/t_critical_upper', + ] + + if any(p['path'] in state_rel_path for p in patch_obj): + raise wsme.exc.ClientSideError( + _("The following fields are not remote-modifiable: %s") % + state_rel_path) diff --git a/inventory/inventory/inventory/api/controllers/v1/sensorgroup.py b/inventory/inventory/inventory/api/controllers/v1/sensorgroup.py new file mode 100644 index 00000000..71e0e9e9 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/sensorgroup.py @@ -0,0 +1,751 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 UnitedStack Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import copy +import jsonpatch +import pecan +from pecan import rest +import six +import uuid +import wsme +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import sensor as sensor_api +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils +from inventory.common import constants +from inventory.common import exception +from inventory.common import hwmon_api +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import utils as cutils +from inventory import objects +from oslo_log import log +from oslo_utils import uuidutils +from six import text_type as unicode + +LOG = log.getLogger(__name__) + + +class SensorGroupPatchType(types.JsonPatchType): + @staticmethod + def mandatory_attrs(): + return ['/host_uuid', 'uuid'] + + +class SensorGroup(base.APIBase): + """API representation of an Sensor Group + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of an + sensorgroup. + """ + + uuid = types.uuid + "Unique UUID for this sensorgroup" + + sensorgroupname = wtypes.text + "Represent the name of the sensorgroup. Unique with path per host" + + path = wtypes.text + "Represent the path of the sensor. Unique with sensorname per host" + + sensortype = wtypes.text + "Represent the sensortype . e.g. Temperature, WatchDog" + + datatype = wtypes.text + "Represent the datatype e.g. discrete or analog," + + state = wtypes.text + "Represent the state of the sensorgroup" + + possible_states = wtypes.text + "Represent the possible states of the sensorgroup" + + algorithm = wtypes.text + "Represent the algorithm of the sensorgroup." + + audit_interval_group = int + "Represent the audit interval of the sensorgroup." + + actions_critical_choices = wtypes.text + "Represent the configurable critical severity actions of the sensorgroup." + + actions_major_choices = wtypes.text + "Represent the configurable major severity actions of the sensorgroup." + + actions_minor_choices = wtypes.text + "Represent the configurable minor severity actions of the sensorgroup." + + actions_minor_group = wtypes.text + "Represent the minor configured actions of the sensorgroup. CSV." + + actions_major_group = wtypes.text + "Represent the major configured actions of the sensorgroup. CSV." + + actions_critical_group = wtypes.text + "Represent the critical configured actions of the sensorgroup. CSV." + + unit_base_group = wtypes.text + "Represent the unit base of the analog sensorgroup e.g. revolutions" + + unit_modifier_group = wtypes.text + "Represent the unit modifier of the analog sensorgroup e.g. 10**2" + + unit_rate_group = wtypes.text + "Represent the unit rate of the sensorgroup e.g. /minute" + + t_minor_lower_group = wtypes.text + "Represent the minor lower threshold of the analog sensorgroup" + + t_minor_upper_group = wtypes.text + "Represent the minor upper threshold of the analog sensorgroup" + + t_major_lower_group = wtypes.text + "Represent the major lower threshold of the analog sensorgroup" + + t_major_upper_group = wtypes.text + "Represent the major upper threshold of the analog sensorgroup" + + t_critical_lower_group = wtypes.text + "Represent the critical lower threshold of the analog sensorgroup" + + t_critical_upper_group = wtypes.text + "Represent the critical upper threshold of the analog sensorgroup" + + capabilities = {wtypes.text: utils.ValidTypes(wtypes.text, + six.integer_types)} + "Represent meta data of the sensorgroup" + + suppress = wtypes.text + "Represent supress sensor if True, otherwise not suppress sensor" + + sensors = wtypes.text + "Represent the sensors of the sensorgroup" + + host_id = int + "Represent the host_id the sensorgroup belongs to" + + host_uuid = types.uuid + "Represent the UUID of the host the sensorgroup belongs to" + + links = [link.Link] + "Represent a list containing a self link and associated sensorgroup links" + + sensors = [link.Link] + "Links to the collection of sensors on this sensorgroup" + + def __init__(self, **kwargs): + self.fields = objects.SensorGroup.fields.keys() + for k in self.fields: + setattr(self, k, kwargs.get(k)) + + # 'sensors' is not part of objects.SenorGroups.fields (it's an + # API-only attribute) + self.fields.append('sensors') + setattr(self, 'sensors', kwargs.get('sensors', None)) + + @classmethod + def convert_with_links(cls, rsensorgroup, expand=True): + + sensorgroup = SensorGroup(**rsensorgroup.as_dict()) + + sensorgroup_fields_common = ['uuid', 'host_id', + 'host_uuid', + 'sensortype', 'datatype', + 'sensorgroupname', + 'path', + + 'state', + 'possible_states', + 'audit_interval_group', + 'algorithm', + 'actions_critical_choices', + 'actions_major_choices', + 'actions_minor_choices', + 'actions_minor_group', + 'actions_major_group', + 'actions_critical_group', + 'sensors', + + 'suppress', + 'capabilities', + 'created_at', 'updated_at', ] + + sensorgroup_fields_analog = ['unit_base_group', + 'unit_modifier_group', + 'unit_rate_group', + + 't_minor_lower_group', + 't_minor_upper_group', + 't_major_lower_group', + 't_major_upper_group', + 't_critical_lower_group', + 't_critical_upper_group', ] + + if rsensorgroup.datatype == 'discrete': + sensorgroup_fields = sensorgroup_fields_common + elif rsensorgroup.datatype == 'analog': + sensorgroup_fields = \ + sensorgroup_fields_common + sensorgroup_fields_analog + else: + LOG.error(_("Invalid datatype={}").format(rsensorgroup.datatype)) + + if not expand: + sensorgroup.unset_fields_except(sensorgroup_fields) + + if sensorgroup.host_id and not sensorgroup.host_uuid: + host = objects.Host.get_by_uuid(pecan.request.context, + sensorgroup.host_id) + sensorgroup.host_uuid = host.uuid + + # never expose the id attribute + sensorgroup.host_id = wtypes.Unset + sensorgroup.id = wtypes.Unset + + sensorgroup.links = [ + link.Link.make_link('self', pecan.request.host_url, + 'sensorgroups', + sensorgroup.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'sensorgroups', + sensorgroup.uuid, + bookmark=True)] + + sensorgroup.sensors = [ + link.Link.make_link('self', + pecan.request.host_url, + 'sensorgroups', + sensorgroup.uuid + "/sensors"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'sensorgroups', + sensorgroup.uuid + "/sensors", + bookmark=True)] + + return sensorgroup + + +class SensorGroupCollection(collection.Collection): + """API representation of a collection of SensorGroup objects.""" + + sensorgroups = [SensorGroup] + "A list containing SensorGroup objects" + + def __init__(self, **kwargs): + self._type = 'sensorgroups' + + @classmethod + def convert_with_links(cls, rsensorgroups, limit, url=None, + expand=False, **kwargs): + collection = SensorGroupCollection() + collection.sensorgroups = [SensorGroup.convert_with_links(p, expand) + for p in rsensorgroups] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'SensorGroupController' + + +class SensorGroupController(rest.RestController): + """REST controller for SensorGroups.""" + + sensors = sensor_api.SensorController(from_sensorgroup=True) + "Expose sensors as a sub-element of sensorgroups" + + _custom_actions = { + 'detail': ['GET'], + 'relearn': ['POST'], + } + + def __init__(self, from_hosts=False): + self._from_hosts = from_hosts + self._api_token = None + self._hwmon_address = k_host.LOCALHOST_HOSTNAME + self._hwmon_port = constants.HWMON_PORT + + def _get_sensorgroups_collection(self, uuid, + marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + + if self._from_hosts and not uuid: + raise exception.InvalidParameterValue(_( + "Host id not specified.")) + + limit = utils.validate_limit(limit) + sort_dir = utils.validate_sort_dir(sort_dir) + + marker_obj = None + if marker: + marker_obj = objects.SensorGroup.get_by_uuid( + pecan.request.context, + marker) + + if self._from_hosts: + sensorgroups = pecan.request.dbapi.sensorgroup_get_by_host( + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + if uuid: + sensorgroups = pecan.request.dbapi.sensorgroup_get_by_host( + uuid, limit, + marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + else: + sensorgroups = pecan.request.dbapi.sensorgroup_get_list( + limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + + return SensorGroupCollection.convert_with_links(sensorgroups, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(SensorGroupCollection, types.uuid, + types.uuid, int, wtypes.text, wtypes.text) + def get_all(self, uuid=None, + marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of sensorgroups.""" + + return self._get_sensorgroups_collection(uuid, + marker, limit, + sort_key, sort_dir) + + @wsme_pecan.wsexpose(SensorGroupCollection, types.uuid, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, uuid=None, marker=None, limit=None, + sort_key='id', sort_dir='asc'): + """Retrieve a list of sensorgroups with detail.""" + + # NOTE(lucasagomes): /detail should only work against collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "sensorgroups": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['sensorgroups', 'detail']) + return self._get_sensorgroups_collection(uuid, marker, limit, + sort_key, sort_dir, + expand, resource_url) + + @wsme_pecan.wsexpose(SensorGroup, types.uuid) + def get_one(self, sensorgroup_uuid): + """Retrieve information about the given sensorgroup.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rsensorgroup = objects.SensorGroup.get_by_uuid( + pecan.request.context, sensorgroup_uuid) + + if rsensorgroup.datatype == 'discrete': + rsensorgroup = objects.SensorGroupDiscrete.get_by_uuid( + pecan.request.context, sensorgroup_uuid) + elif rsensorgroup.datatype == 'analog': + rsensorgroup = objects.SensorGroupAnalog.get_by_uuid( + pecan.request.context, sensorgroup_uuid) + else: + LOG.error(_("Invalid datatype={}").format(rsensorgroup.datatype)) + + return SensorGroup.convert_with_links(rsensorgroup) + + @staticmethod + def _new_sensorgroup_semantic_checks(sensorgroup): + datatype = sensorgroup.as_dict().get('datatype') or "" + sensortype = sensorgroup.as_dict().get('sensortype') or "" + if not (datatype and sensortype): + raise wsme.exc.ClientSideError(_("sensorgroup-add: Cannot " + "add a sensorgroup " + "without a valid datatype " + "and sensortype.")) + + if datatype not in constants.SENSOR_DATATYPE_VALID_LIST: + raise wsme.exc.ClientSideError( + _("sensorgroup datatype must be one of %s.") % + constants.SENSOR_DATATYPE_VALID_LIST) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(SensorGroup, body=SensorGroup) + def post(self, sensorgroup): + """Create a new sensorgroup.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + self._new_sensorgroup_semantic_checks(sensorgroup) + try: + sensorgroup_dict = sensorgroup.as_dict() + new_sensorgroup = _create(sensorgroup_dict) + + except exception.InventoryException as e: + LOG.exception(e) + raise wsme.exc.ClientSideError(_("Invalid data")) + return sensorgroup.convert_with_links(new_sensorgroup) + + def _get_host_uuid(self, body): + host_uuid = body.get('host_uuid') or "" + try: + host = pecan.request.dbapi.host_get(host_uuid) + except exception.NotFound: + raise wsme.exc.ClientSideError("_get_host_uuid lookup failed") + return host.uuid + + @wsme_pecan.wsexpose('json', body=unicode) + def relearn(self, body): + """Handle Sensor Model Relearn Request.""" + host_uuid = self._get_host_uuid(body) + # LOG.info("Host UUID: %s - BM_TYPE: %s" % (host_uuid, bm_type )) + + # hwmon_sensorgroup = {'ihost_uuid': host_uuid} + request_body = {'host_uuid': host_uuid} + hwmon_response = hwmon_api.sensorgroup_relearn( + self._api_token, self._hwmon_address, self._hwmon_port, + request_body, + constants.HWMON_DEFAULT_TIMEOUT_IN_SECS) + + if not hwmon_response: + hwmon_response = {'status': 'fail', + 'reason': 'no response', + 'action': 'retry'} + + elif hwmon_response['status'] != 'pass': + msg = _("HWMON has returned with " + "a status of {}, reason: {}, " + "recommended action: {}").format( + hwmon_response.get('status'), + hwmon_response.get('reason'), + hwmon_response.get('action')) + + raise wsme.exc.ClientSideError(msg) + + @cutils.synchronized(LOCK_NAME) + @wsme.validate(types.uuid, [SensorGroupPatchType]) + @wsme_pecan.wsexpose(SensorGroup, types.uuid, + body=[SensorGroupPatchType]) + def patch(self, sensorgroup_uuid, patch): + """Update an existing sensorgroup.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + rsensorgroup = objects.SensorGroup.get_by_uuid( + pecan.request.context, sensorgroup_uuid) + + if rsensorgroup.datatype == 'discrete': + rsensorgroup = objects.SensorGroupDiscrete.get_by_uuid( + pecan.request.context, sensorgroup_uuid) + elif rsensorgroup.datatype == 'analog': + rsensorgroup = objects.SensorGroupAnalog.get_by_uuid( + pecan.request.context, sensorgroup_uuid) + else: + raise wsme.exc.ClientSideError(_("Invalid datatype={}").format( + rsensorgroup.datatype)) + + rsensorgroup_orig = copy.deepcopy(rsensorgroup) + + host = pecan.request.dbapi.host_get( + rsensorgroup['host_id']).as_dict() + + utils.validate_patch(patch) + patch_obj = jsonpatch.JsonPatch(patch) + my_host_uuid = None + for p in patch_obj: + # For Profile replace host_uuid with corresponding id + if p['path'] == '/host_uuid': + p['path'] = '/host_id' + host = objects.Host.get_by_uuid(pecan.request.context, + p['value']) + p['value'] = host.id + my_host_uuid = host.uuid + + # update sensors if set + sensors = None + for s in patch: + if '/sensors' in s['path']: + sensors = s['value'] + patch.remove(s) + break + + if sensors: + _update_sensors("modify", rsensorgroup, host, sensors) + + try: + sensorgroup = SensorGroup(**jsonpatch.apply_patch( + rsensorgroup.as_dict(), + patch_obj)) + + except utils.JSONPATCH_EXCEPTIONS as e: + raise exception.PatchError(patch=patch, reason=e) + + # Update only the fields that have changed + if rsensorgroup.datatype == 'discrete': + fields = objects.SensorGroupDiscrete.fields + else: + fields = objects.SensorGroupAnalog.fields + + for field in fields: + if rsensorgroup[field] != getattr(sensorgroup, field): + rsensorgroup[field] = getattr(sensorgroup, field) + + delta = rsensorgroup.obj_what_changed() + + sensorgroup_suppress_attrs = ['suppress'] + force_action = False + if any(x in delta for x in sensorgroup_suppress_attrs): + valid_suppress = ['True', 'False', 'true', 'false', 'force_action'] + if rsensorgroup.suppress.lower() not in valid_suppress: + raise wsme.exc.ClientSideError(_("Invalid suppress value, " + "select 'True' or 'False'")) + elif rsensorgroup.suppress.lower() == 'force_action': + LOG.info("suppress=%s" % rsensorgroup.suppress.lower()) + rsensorgroup.suppress = rsensorgroup_orig.suppress + force_action = True + + self._semantic_modifiable_fields(patch_obj, force_action) + + if not pecan.request.user_agent.startswith('hwmon'): + hwmon_sensorgroup = cutils.removekeys_nonhwmon( + rsensorgroup.as_dict()) + + if not my_host_uuid: + host = objects.Host.get_by_uuid(pecan.request.context, + rsensorgroup.host_id) + my_host_uuid = host.uuid + + hwmon_sensorgroup.update({'host_uuid': my_host_uuid}) + + hwmon_response = hwmon_api.sensorgroup_modify( + self._api_token, self._hwmon_address, self._hwmon_port, + hwmon_sensorgroup, + constants.HWMON_DEFAULT_TIMEOUT_IN_SECS) + + if not hwmon_response: + hwmon_response = {'status': 'fail', + 'reason': 'no response', + 'action': 'retry'} + + if hwmon_response['status'] != 'pass': + msg = _("HWMON has returned with a status of {}, reason: {}, " + "recommended action: {}").format( + hwmon_response.get('status'), + hwmon_response.get('reason'), + hwmon_response.get('action')) + + if force_action: + LOG.error(msg) + else: + raise wsme.exc.ClientSideError(msg) + + sensorgroup_prop_attrs = ['audit_interval_group', + 'actions_minor_group', + 'actions_major_group', + 'actions_critical_group', + 'suppress'] + + if any(x in delta for x in sensorgroup_prop_attrs): + # propagate to Sensors within this SensorGroup + sensor_val = {'audit_interval': rsensorgroup.audit_interval_group, + 'actions_minor': rsensorgroup.actions_minor_group, + 'actions_major': rsensorgroup.actions_major_group, + 'actions_critical': + rsensorgroup.actions_critical_group} + if 'suppress' in delta: + sensor_val.update({'suppress': rsensorgroup.suppress}) + pecan.request.dbapi.sensorgroup_propagate( + rsensorgroup.uuid, sensor_val) + + rsensorgroup.save() + + return SensorGroup.convert_with_links(rsensorgroup) + + @cutils.synchronized(LOCK_NAME) + @wsme_pecan.wsexpose(None, types.uuid, status_code=204) + def delete(self, sensorgroup_uuid): + """Delete a sensorgroup.""" + if self._from_hosts: + raise exception.OperationNotPermitted + + pecan.request.dbapi.sensorgroup_destroy(sensorgroup_uuid) + + @staticmethod + def _semantic_modifiable_fields(patch_obj, force_action=False): + # Prevent auto populated fields from being updated + state_rel_path = ['/uuid', '/id', '/host_id', '/datatype', + '/sensortype'] + + if any(p['path'] in state_rel_path for p in patch_obj): + raise wsme.exc.ClientSideError(_("The following fields can not be " + "modified: %s ") % state_rel_path) + + if not (pecan.request.user_agent.startswith('hwmon') or force_action): + state_rel_path = ['/sensorgroupname', '/path', + '/state', '/possible_states', + '/actions_critical_choices', + '/actions_major_choices', + '/actions_minor_choices', + '/unit_base_group', + '/unit_modifier_group', + '/unit_rate_group', + '/t_minor_lower_group', + '/t_minor_upper_group', + '/t_major_lower_group', + '/t_major_upper_group', + '/t_critical_lower_group', + '/t_critical_upper_group', + ] + + if any(p['path'] in state_rel_path for p in patch_obj): + raise wsme.exc.ClientSideError( + _("The following fields are not remote-modifiable: %s") % + state_rel_path) + + +def _create(sensorgroup, from_profile=False): + """Create a sensorgroup through a non-HTTP request e.g. via profile.py + while still passing through sensorgroup semantic checks. + Hence, not declared inside a class. + Param: + sensorgroup - dictionary of sensorgroup values + from_profile - Boolean whether from profile + """ + + if 'host_id' in sensorgroup and sensorgroup['host_id']: + ihostid = sensorgroup['host_id'] + else: + ihostid = sensorgroup['host_uuid'] + + ihost = pecan.request.dbapi.host_get(ihostid) + if uuidutils.is_uuid_like(ihostid): + host_id = ihost['id'] + else: + host_id = ihostid + sensorgroup.update({'host_id': host_id}) + LOG.info("sensorgroup post sensorgroups ihostid: %s" % host_id) + sensorgroup['host_uuid'] = ihost['uuid'] + + # Assign UUID if not already done. + if not sensorgroup.get('uuid'): + sensorgroup['uuid'] = str(uuid.uuid4()) + + # Get sensors + sensors = None + if 'sensors' in sensorgroup: + sensors = sensorgroup['sensors'] + + # Set defaults - before checks to allow for optional attributes + # if not from_profile: + # sensorgroup = _set_defaults(sensorgroup) + + # Semantic checks + # sensorgroup = _check("add", + # sensorgroup, + # sensors=sensors, + # ifaces=uses_if, + # from_profile=from_profile) + + if sensorgroup.get('datatype'): + if sensorgroup['datatype'] == 'discrete': + new_sensorgroup = pecan.request.dbapi.sensorgroup_discrete_create( + ihost.id, sensorgroup) + elif sensorgroup['datatype'] == 'analog': + new_sensorgroup = pecan.request.dbapi.sensorgroup_analog_create( + ihost.id, sensorgroup) + else: + raise wsme.exc.ClientSideError(_("Invalid datatype. %s") % + sensorgroup.datatype) + else: + raise wsme.exc.ClientSideError(_("Unspecified datatype.")) + + # Update sensors + if sensors: + try: + _update_sensors("modify", + new_sensorgroup.as_dict(), + ihost, + sensors) + except Exception as e: + pecan.request.dbapi.sensorgroup_destroy( + new_sensorgroup.as_dict()['uuid']) + raise e + + # Update sensors + # return new_sensorgroup + return SensorGroup.convert_with_links(new_sensorgroup) + + +def _update_sensors(op, sensorgroup, ihost, sensors): + sensors = sensors.split(',') + + this_sensorgroup_datatype = None + this_sensorgroup_sensortype = None + if op == "add": + this_sensorgroup_id = 0 + else: + this_sensorgroup_id = sensorgroup['id'] + this_sensorgroup_datatype = sensorgroup['datatype'] + this_sensorgroup_sensortype = sensorgroup['sensortype'] + + if sensors: + # Update Sensors' sensorgroup_uuid attribute + sensors_list = pecan.request.dbapi.sensor_get_all( + host_id=ihost['id']) + for p in sensors_list: + # if new sensor associated + if (p.uuid in sensors or p.sensorname in sensors) \ + and not p.sensorgroup_id: + values = {'sensorgroup_id': sensorgroup['id']} + # else if old sensor disassociated + elif ((p.uuid not in sensors and p.sensorname not in sensors) and + p.sensorgroup_id and + p.sensorgroup_id == this_sensorgroup_id): + values = {'sensorgroup_id': None} + else: + continue + + if p.datatype != this_sensorgroup_datatype: + msg = _("Invalid datatype: host {} sensor {}: Expected: {} " + "Received: {}.").format( + (ihost['hostname'], p.sensorname, + this_sensorgroup_datatype, p.datatype)) + raise wsme.exc.ClientSideError(msg) + + if p.sensortype != this_sensorgroup_sensortype: + msg = _("Invalid sensortype: host {} sensor {}: Expected: {} " + "Received: {}.").format( + ihost['hostname'], p.sensorname, + this_sensorgroup_sensortype, p.sensortype) + raise wsme.exc.ClientSideError(msg) + + try: + pecan.request.dbapi.sensor_update(p.uuid, values) + except exception.HTTPNotFound: + msg = _("Sensor update of sensorgroup_uuid failed: host {} " + "sensor {}").format(ihost['hostname'], p.sensorname) + raise wsme.exc.ClientSideError(msg) diff --git a/inventory/inventory/inventory/api/controllers/v1/state.py b/inventory/inventory/inventory/api/controllers/v1/state.py new file mode 100644 index 00000000..917b5335 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/state.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import link +from wsme import types as wtypes + + +class State(base.APIBase): + + current = wtypes.text + "The current state" + + target = wtypes.text + "The user modified desired state" + + available = [wtypes.text] + "A list of available states it is able to transition to" + + links = [link.Link] + "A list containing a self link and associated state links" diff --git a/inventory/inventory/inventory/api/controllers/v1/sysinv.py b/inventory/inventory/inventory/api/controllers/v1/sysinv.py new file mode 100644 index 00000000..95b38c92 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/sysinv.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from cgtsclient.v1 import client as cgts_client +from inventory.api import config +from keystoneauth1 import loading as ks_loading +from oslo_config import cfg +from oslo_log import log + + +CONF = cfg.CONF + +LOG = log.getLogger(__name__) + +_SESSION = None + + +def cgtsclient(context, version=1, endpoint=None): + """Constructs a cgts client object for making API requests. + + :param context: The FM request context for auth. + :param version: API endpoint version. + :param endpoint: Optional If the endpoint is not available, it will be + retrieved from session + """ + global _SESSION + + if not _SESSION: + _SESSION = ks_loading.load_session_from_conf_options( + CONF, config.sysinv_group.name) + + auth_token = context.auth_token + if endpoint is None: + auth = context.get_auth_plugin() + service_type, service_name, interface = \ + CONF.sysinv.catalog_info.split(':') + service_parameters = {'service_type': service_type, + 'service_name': service_name, + 'interface': interface, + 'region_name': CONF.sysinv.os_region_name} + endpoint = _SESSION.get_endpoint(auth, **service_parameters) + + return cgts_client.Client(version=version, + endpoint=endpoint, + token=auth_token) diff --git a/inventory/inventory/inventory/api/controllers/v1/system.py b/inventory/inventory/inventory/api/controllers/v1/system.py new file mode 100644 index 00000000..a98734f5 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/system.py @@ -0,0 +1,266 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from oslo_log import log +import pecan +from pecan import rest +import six +from wsme import types as wtypes +import wsmeext.pecan as wsme_pecan + +from inventory.api.controllers.v1 import base +from inventory.api.controllers.v1 import collection +from inventory.api.controllers.v1 import host +from inventory.api.controllers.v1 import link +from inventory.api.controllers.v1 import types +from inventory.api.controllers.v1 import utils as api_utils +from inventory.common import constants +from inventory.common import exception +from inventory.common import k_host +from inventory import objects + +LOG = log.getLogger(__name__) + +VALID_VSWITCH_TYPES = [constants.VSWITCH_TYPE_OVS_DPDK] + + +class System(base.APIBase): + """API representation of a system. + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of + a system. + """ + + uuid = types.uuid + "The UUID of the system" + + name = wtypes.text + "The name of the system" + + system_type = wtypes.text + "The type of the system" + + system_mode = wtypes.text + "The mode of the system" + + description = wtypes.text + "The name of the system" + + contact = wtypes.text + "The contact of the system" + + location = wtypes.text + "The location of the system" + + services = int + "The services of the system" + + software_version = wtypes.text + "A textual description of the entity" + + timezone = wtypes.text + "The timezone of the system" + + links = [link.Link] + "A list containing a self link and associated system links" + + hosts = [link.Link] + "Links to the collection of hosts contained in this system" + + capabilities = {wtypes.text: api_utils.ValidTypes(wtypes.text, bool, + six.integer_types)} + "System defined capabilities" + + region_name = wtypes.text + "The region name of the system" + + distributed_cloud_role = wtypes.text + "The distributed cloud role of the system" + + service_project_name = wtypes.text + "The service project name of the system" + + security_feature = wtypes.text + "Kernel arguments associated with enabled spectre/meltdown fix features" + + def __init__(self, **kwargs): + self.fields = objects.System.fields.keys() + + for k in self.fields: + # Translate any special internal representation of data to its + # customer facing form + if k == 'security_feature': + # look up which customer-facing-security-feature-string goes + # with the kernel arguments tracked in sysinv + kernel_args = kwargs.get(k) + translated_string = kernel_args + + for user_string, args_string in \ + constants.SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_OPTS.iteritems(): # noqa + if args_string == kernel_args: + translated_string = user_string + break + setattr(self, k, translated_string) + else: + # No translation required + setattr(self, k, kwargs.get(k)) + + @classmethod + def convert_with_links(cls, rpc_system, expand=True): + minimum_fields = ['id', 'uuid', 'name', 'system_type', 'system_mode', + 'description', 'capabilities', + 'contact', 'location', 'software_version', + 'created_at', 'updated_at', 'timezone', + 'region_name', 'service_project_name', + 'distributed_cloud_role', 'security_feature'] + + fields = minimum_fields if not expand else None + + iSystem = System.from_rpc_object(rpc_system, fields) + + iSystem.links = [link.Link.make_link('self', pecan.request.host_url, + 'systems', iSystem.uuid), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'systems', iSystem.uuid, + bookmark=True) + ] + + if expand: + iSystem.hosts = [ + link.Link.make_link('self', + pecan.request.host_url, + 'systems', + iSystem.uuid + "/hosts"), + link.Link.make_link('bookmark', + pecan.request.host_url, + 'systems', + iSystem.uuid + "/hosts", + bookmark=True)] + + return iSystem + + +class SystemCollection(collection.Collection): + """API representation of a collection of systems.""" + + systems = [System] + "A list containing system objects" + + def __init__(self, **kwargs): + self._type = 'systems' + + @classmethod + def convert_with_links(cls, systems, limit, url=None, + expand=False, **kwargs): + collection = SystemCollection() + collection.systems = [System.convert_with_links(ch, expand) + for ch in systems] + + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +LOCK_NAME = 'SystemController' + + +class SystemController(rest.RestController): + """REST controller for system.""" + + hosts = host.HostController(from_system=True) + "Expose hosts as a sub-element of system" + + _custom_actions = { + 'detail': ['GET'], + } + + def __init__(self): + self._bm_region = None + + def _bm_region_get(self): + # only supported region type is BM_EXTERNAL + if not self._bm_region: + self._bm_region = k_host.BM_EXTERNAL + return self._bm_region + + def _get_system_collection(self, marker, limit, sort_key, sort_dir, + expand=False, resource_url=None): + limit = api_utils.validate_limit(limit) + sort_dir = api_utils.validate_sort_dir(sort_dir) + marker_obj = None + if marker: + marker_obj = objects.System.get_by_uuid(pecan.request.context, + marker) + system = pecan.request.dbapi.system_get_list(limit, marker_obj, + sort_key=sort_key, + sort_dir=sort_dir) + for i in system: + i.capabilities['bm_region'] = self._bm_region_get() + + return SystemCollection.convert_with_links(system, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @wsme_pecan.wsexpose(SystemCollection, types.uuid, + int, wtypes.text, wtypes.text) + def get_all(self, marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of systems. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + """ + return self._get_system_collection(marker, limit, sort_key, sort_dir) + + @wsme_pecan.wsexpose(SystemCollection, types.uuid, int, + wtypes.text, wtypes.text) + def detail(self, marker=None, limit=None, sort_key='id', sort_dir='asc'): + """Retrieve a list of system with detail. + + :param marker: pagination marker for large data sets. + :param limit: maximum number of resources to return in a single result. + :param sort_key: column to sort results by. Default: id. + :param sort_dir: direction to sort. "asc" or "desc". Default: asc. + """ + # /detail should only work agaist collections + parent = pecan.request.path.split('/')[:-1][-1] + if parent != "system": + raise exception.HTTPNotFound + + expand = True + resource_url = '/'.join(['system', 'detail']) + return self._get_system_collection(marker, limit, sort_key, sort_dir, + expand, resource_url) + + @wsme_pecan.wsexpose(System, types.uuid) + def get_one(self, system_uuid): + """Retrieve information about the given system. + + :param system_uuid: UUID of a system. + """ + rpc_system = objects.System.get_by_uuid(pecan.request.context, + system_uuid) + + rpc_system.capabilities['bm_region'] = self._bm_region_get() + return System.convert_with_links(rpc_system) + + @wsme_pecan.wsexpose(System, body=System) + def post(self, system): + """Create a new system.""" + raise exception.OperationNotPermitted + + @wsme_pecan.wsexpose(None, types.uuid, status_code=204) + def delete(self, system_uuid): + """Delete a system. + + :param system_uuid: UUID of a system. + """ + raise exception.OperationNotPermitted diff --git a/inventory/inventory/inventory/api/controllers/v1/types.py b/inventory/inventory/inventory/api/controllers/v1/types.py new file mode 100644 index 00000000..2056c32f --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/types.py @@ -0,0 +1,215 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding: utf-8 +# +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from oslo_utils import strutils +import six + +import wsme +from wsme import types as wtypes + +from inventory.api.controllers.v1 import utils as apiutils +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import utils + + +class MACAddressType(wtypes.UserType): + """A simple MAC address type.""" + + basetype = wtypes.text + name = 'macaddress' + + @staticmethod + def validate(value): + return utils.validate_and_normalize_mac(value) + + @staticmethod + def frombasetype(value): + return MACAddressType.validate(value) + + +class UUIDType(wtypes.UserType): + """A simple UUID type.""" + + basetype = wtypes.text + name = 'uuid' + # FIXME(lucasagomes): When used with wsexpose decorator WSME will try + # to get the name of the type by accessing it's __name__ attribute. + # Remove this __name__ attribute once it's fixed in WSME. + # https://bugs.launchpad.net/wsme/+bug/1265590 + __name__ = name + + @staticmethod + def validate(value): + if not utils.is_uuid_like(value): + raise exception.InvalidUUID(uuid=value) + return value + + @staticmethod + def frombasetype(value): + if value is None: + return None + return UUIDType.validate(value) + + +class BooleanType(wtypes.UserType): + """A simple boolean type.""" + + basetype = wtypes.text + name = 'boolean' + + @staticmethod + def validate(value): + try: + return strutils.bool_from_string(value, strict=True) + except ValueError as e: + # raise Invalid to return 400 (BadRequest) in the API + raise exception.Invalid(six.text_type(e)) + + @staticmethod + def frombasetype(value): + if value is None: + return None + return BooleanType.validate(value) + + +class IPAddressType(wtypes.UserType): + """A generic IP address type that supports both IPv4 and IPv6.""" + + basetype = wtypes.text + name = 'ipaddress' + + @staticmethod + def validate(value): + if not utils.is_valid_ip(value): + raise exception.InvalidIPAddress(address=value) + return value + + @staticmethod + def frombasetype(value): + if value is None: + return None + return IPAddressType.validate(value) + + +macaddress = MACAddressType() +uuid = UUIDType() +boolean = BooleanType() +ipaddress = IPAddressType() + + +class ApiDictType(wtypes.UserType): + name = 'apidict' + __name__ = name + + basetype = {wtypes.text: + apiutils.ValidTypes(wtypes.text, six.integer_types)} + + +apidict = ApiDictType() + + +class JsonPatchType(wtypes.Base): + """A complex type that represents a single json-patch operation.""" + + path = wtypes.wsattr(wtypes.StringType(pattern='^(/[\w-]+)+$'), + mandatory=True) + op = wtypes.wsattr(wtypes.Enum(str, 'add', 'replace', 'remove'), + mandatory=True) + value = apiutils.ValidTypes(wtypes.text, six.integer_types, float) + + @staticmethod + def internal_attrs(): + """Returns a list of internal attributes. + + Internal attributes can't be added, replaced or removed. This + method may be overwritten by derived class. + + """ + return ['/created_at', '/id', '/links', '/updated_at', '/uuid'] + + @staticmethod + def mandatory_attrs(): + """Retruns a list of mandatory attributes. + + Mandatory attributes can't be removed from the document. This + method should be overwritten by derived class. + + """ + return [] + + @staticmethod + def validate(patch): + if patch.path in patch.internal_attrs(): + msg = _("'%s' is an internal attribute and can not be updated") + raise wsme.exc.ClientSideError(msg % patch.path) + + if patch.path in patch.mandatory_attrs() and patch.op == 'remove': + msg = _("'%s' is a mandatory attribute and can not be removed") + raise wsme.exc.ClientSideError(msg % patch.path) + + if patch.op == 'add': + if patch.path.count('/') == 1: + msg = _('Adding a new attribute (%s) to the root of ' + ' the resource is not allowed') + raise wsme.exc.ClientSideError(msg % patch.path) + + if patch.op != 'remove': + if not patch.value: + msg = _("Edit and Add operation of the field requires " + "non-empty value.") + raise wsme.exc.ClientSideError(msg) + + ret = {'path': patch.path, 'op': patch.op} + if patch.value: + ret['value'] = patch.value + return ret + + +class MultiType(wtypes.UserType): + """A complex type that represents one or more types. + + Used for validating that a value is an instance of one of the types. + + :param *types: Variable-length list of types. + + """ + def __init__(self, types): + self.types = types + + def validate(self, value): + for t in self.types: + if t is wsme.types.text and isinstance(value, wsme.types.bytes): + value = value.decode() + if isinstance(t, list): + if isinstance(value, list): + for v in value: + if not isinstance(v, t[0]): + break + else: + return value + elif isinstance(value, t): + return value + else: + raise ValueError( + _("Wrong type. Expected '%(type)s', got '%(value)s'") + % {'type': self.types, 'value': type(value)}) diff --git a/inventory/inventory/inventory/api/controllers/v1/utils.py b/inventory/inventory/inventory/api/controllers/v1/utils.py new file mode 100755 index 00000000..42a19493 --- /dev/null +++ b/inventory/inventory/inventory/api/controllers/v1/utils.py @@ -0,0 +1,567 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import contextlib +import jsonpatch +import netaddr +import os +import pecan +import re +import socket +import sys +import traceback +import tsconfig.tsconfig as tsc +import wsme + +from inventory.api.controllers.v1.sysinv import cgtsclient +from inventory.common import constants +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common.utils import memoized +from inventory import objects +from oslo_config import cfg +from oslo_log import log + +CONF = cfg.CONF +LOG = log.getLogger(__name__) +KEY_VALUE_SEP = '=' +JSONPATCH_EXCEPTIONS = (jsonpatch.JsonPatchException, + jsonpatch.JsonPointerException, + KeyError) + + +def ip_version_to_string(ip_version): + return str(constants.IP_FAMILIES[ip_version]) + + +def validate_limit(limit): + if limit and limit < 0: + raise wsme.exc.ClientSideError(_("Limit must be positive")) + + return min(CONF.api.limit_max, limit) or CONF.api.limit_max + + +def validate_sort_dir(sort_dir): + if sort_dir not in ['asc', 'desc']: + raise wsme.exc.ClientSideError(_("Invalid sort direction: %s. " + "Acceptable values are " + "'asc' or 'desc'") % sort_dir) + return sort_dir + + +def validate_patch(patch): + """Performs a basic validation on patch.""" + + if not isinstance(patch, list): + patch = [patch] + + for p in patch: + path_pattern = re.compile("^/[a-zA-Z0-9-_]+(/[a-zA-Z0-9-_]+)*$") + + if not isinstance(p, dict) or \ + any(key for key in ["path", "op"] if key not in p): + raise wsme.exc.ClientSideError( + _("Invalid patch format: %s") % str(p)) + + path = p["path"] + op = p["op"] + + if op not in ["add", "replace", "remove"]: + raise wsme.exc.ClientSideError( + _("Operation not supported: %s") % op) + + if not path_pattern.match(path): + raise wsme.exc.ClientSideError(_("Invalid path: %s") % path) + + if op == "add": + if path.count('/') == 1: + raise wsme.exc.ClientSideError( + _("Adding an additional attribute (%s) to the " + "resource is not allowed") % path) + + +def validate_mtu(mtu): + """Check if MTU is valid""" + if mtu < 576 or mtu > 9216: + raise wsme.exc.ClientSideError(_( + "MTU must be between 576 and 9216 bytes.")) + + +def validate_address_within_address_pool(ip, pool): + """Determine whether an IP address is within the specified IP address pool. + :param ip netaddr.IPAddress object + :param pool objects.AddressPool object + """ + ipset = netaddr.IPSet() + for start, end in pool.ranges: + ipset.update(netaddr.IPRange(start, end)) + + if netaddr.IPAddress(ip) not in ipset: + raise wsme.exc.ClientSideError(_( + "IP address %s is not within address pool ranges") % str(ip)) + + +def validate_address_within_nework(ip, network): + """Determine whether an IP address is within the specified IP network. + :param ip netaddr.IPAddress object + :param network objects.Network object + """ + LOG.info("TODO(sc) validate_address_within_address_pool " + "ip=%s, network=%s" % (ip, network)) + + +class ValidTypes(wsme.types.UserType): + """User type for validate that value has one of a few types.""" + + def __init__(self, *types): + self.types = types + + def validate(self, value): + for t in self.types: + if t is wsme.types.text and isinstance(value, wsme.types.bytes): + value = value.decode() + if isinstance(value, t): + return value + else: + raise ValueError("Wrong type. Expected '%s', got '%s'" % ( + self.types, type(value))) + + +def is_valid_hostname(hostname): + """Determine whether an address is valid as per RFC 1123. + """ + + # Maximum length of 255 + rc = True + length = len(hostname) + if length > 255: + raise wsme.exc.ClientSideError(_( + "Hostname {} is too long. Length {} is greater than 255." + "Please configure valid hostname.").format(hostname, length)) + + # Allow a single dot on the right hand side + if hostname[-1] == ".": + hostname = hostname[:-1] + # Create a regex to ensure: + # - hostname does not begin or end with a dash + # - each segment is 1 to 63 characters long + # - valid characters are A-Z (any case) and 0-9 + valid_re = re.compile("(?!-)[A-Z\d-]{1,63}(?' + + '\n'.join(app_iter) + + ''))] + except et.ElementTree.ParseError as err: + LOG.error('Error parsing HTTP response: %s', err) + body = ['%s' % state['status_code'] + + ''] + state['headers'].append(('Content-Type', 'application/xml')) + else: + if six.PY3: + app_iter = [i.decode('utf-8') for i in app_iter] + body = [json.dumps({'error_message': '\n'.join(app_iter)})] + if six.PY3: + body = [item.encode('utf-8') for item in body] + state['headers'].append(('Content-Type', 'application/json')) + state['headers'].append(('Content-Length', str(len(body[0])))) + else: + body = app_iter + return body diff --git a/inventory/inventory/inventory/cmd/__init__.py b/inventory/inventory/inventory/cmd/__init__.py new file mode 100644 index 00000000..b1de0395 --- /dev/null +++ b/inventory/inventory/inventory/cmd/__init__.py @@ -0,0 +1,31 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os + +os.environ['EVENTLET_NO_GREENDNS'] = 'yes' # noqa E402 + +import eventlet + +eventlet.monkey_patch(os=False) + +import oslo_i18n as i18n # noqa I202 + +i18n.install('inventory') diff --git a/inventory/inventory/inventory/cmd/agent.py b/inventory/inventory/inventory/cmd/agent.py new file mode 100644 index 00000000..8d2e6d0f --- /dev/null +++ b/inventory/inventory/inventory/cmd/agent.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +The Inventory Agent Service +""" + +import sys + +from oslo_config import cfg +from oslo_log import log +from oslo_service import service + +from inventory.common import rpc_service +from inventory.common import service as inventory_service + + +CONF = cfg.CONF + +LOG = log.getLogger(__name__) + + +def main(): + # Parse config file and command line options, then start logging + inventory_service.prepare_service(sys.argv) + + # connection is based upon host and MANAGER_TOPIC + mgr = rpc_service.RPCService(CONF.host, + 'inventory.agent.manager', + 'AgentManager') + launcher = service.launch(CONF, mgr) + launcher.wait() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/inventory/inventory/inventory/cmd/api.py b/inventory/inventory/inventory/cmd/api.py new file mode 100644 index 00000000..a601cce7 --- /dev/null +++ b/inventory/inventory/inventory/cmd/api.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import sys + +import eventlet +from oslo_config import cfg +from oslo_log import log as logging +from oslo_service import systemd +from oslo_service import wsgi + +import logging as std_logging + +from inventory.api import app +from inventory.api import config +from inventory.common.i18n import _ + +api_opts = [ + cfg.StrOpt('bind_host', + default="0.0.0.0", + help=_('IP address for inventory api to listen')), + cfg.IntOpt('bind_port', + default=6380, + help=_('listen port for inventory api')), + cfg.StrOpt('bind_host_pxe', + default="0.0.0.0", + help=_('IP address for inventory api pxe to listen')), + cfg.IntOpt('api_workers', default=2, + help=_("number of api workers")), + cfg.IntOpt('limit_max', + default=1000, + help='the maximum number of items returned in a single ' + 'response from a collection resource') +] + + +CONF = cfg.CONF + + +LOG = logging.getLogger(__name__) +eventlet.monkey_patch(os=False) + + +def main(): + + config.init(sys.argv[1:]) + config.setup_logging() + + application = app.load_paste_app() + + CONF.register_opts(api_opts, 'api') + + host = CONF.api.bind_host + port = CONF.api.bind_port + workers = CONF.api.api_workers + + if workers < 1: + LOG.warning("Wrong worker number, worker = %(workers)s", workers) + workers = 1 + + LOG.info("Serving on http://%(host)s:%(port)s with %(workers)s", + {'host': host, 'port': port, 'workers': workers}) + systemd.notify_once() + service = wsgi.Server(CONF, CONF.prog, application, host, port) + + app.serve(service, CONF, workers) + + pxe_host = CONF.api.bind_host_pxe + if pxe_host: + pxe_service = wsgi.Server(CONF, CONF.prog, application, pxe_host, port) + app.serve_pxe(pxe_service, CONF, 1) + + LOG.debug("Configuration:") + CONF.log_opt_values(LOG, std_logging.DEBUG) + + app.wait() + if pxe_host: + app.wait_pxe() + + +if __name__ == '__main__': + main() diff --git a/inventory/inventory/inventory/cmd/conductor.py b/inventory/inventory/inventory/cmd/conductor.py new file mode 100644 index 00000000..be3e14e3 --- /dev/null +++ b/inventory/inventory/inventory/cmd/conductor.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +""" +The Inventory Conductor Service +""" + +import sys + +from oslo_config import cfg +from oslo_log import log +from oslo_service import service + +from inventory.common import rpc_service +from inventory.common import service as inventory_service + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +def main(): + # Parse config file and command line options, then start logging + inventory_service.prepare_service(sys.argv) + + mgr = rpc_service.RPCService(CONF.host, + 'inventory.conductor.manager', + 'ConductorManager') + + launcher = service.launch(CONF, mgr) + launcher.wait() + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/inventory/inventory/inventory/cmd/dbsync.py b/inventory/inventory/inventory/cmd/dbsync.py new file mode 100644 index 00000000..4e1c7e9c --- /dev/null +++ b/inventory/inventory/inventory/cmd/dbsync.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from oslo_config import cfg +import sys + +from inventory.db import migration + +CONF = cfg.CONF + + +def main(): + cfg.CONF(sys.argv[1:], + project='inventory') + migration.db_sync() diff --git a/inventory/inventory/inventory/cmd/dnsmasq_lease_update.py b/inventory/inventory/inventory/cmd/dnsmasq_lease_update.py new file mode 100755 index 00000000..645e8bd8 --- /dev/null +++ b/inventory/inventory/inventory/cmd/dnsmasq_lease_update.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# + + +""" +Handle lease database updates from dnsmasq DHCP server +This file was based on dhcpbridge.py from nova +""" + +from __future__ import print_function + +import os +import sys + +from inventory.common import context +from inventory.common.i18n import _ +from inventory.common import service as inventory_service +from inventory.conductor import rpcapi as conductor_rpcapi + +from oslo_config import cfg +from oslo_log import log + +CONF = cfg.CONF + + +def add_lease(mac, ip_address): + """Called when a new lease is created.""" + + ctxt = context.get_admin_context() + rpcapi = \ + conductor_rpcapi.ConductorAPI(topic=conductor_rpcapi.MANAGER_TOPIC) + + cid = None + cid = os.getenv('DNSMASQ_CLIENT_ID') + + tags = None + tags = os.getenv('DNSMASQ_TAGS') + + if tags is not None: + # TODO(sc): Maybe this shouldn't be synchronous - if this hangs, + # we could cause dnsmasq to get stuck... + rpcapi.handle_dhcp_lease(ctxt, tags, mac, ip_address, cid) + + +def old_lease(mac, ip_address): + """Called when an old lease is recognized.""" + + # This happens when a node is rebooted, but it can also happen if the + # node was deleted and then rebooted, so we need to re-add in that case. + + ctxt = context.get_admin_context() + rpcapi = conductor_rpcapi.ConductorAPI( + topic=conductor_rpcapi.MANAGER_TOPIC) + + cid = None + cid = os.getenv('DNSMASQ_CLIENT_ID') + + tags = None + tags = os.getenv('DNSMASQ_TAGS') + + if tags is not None: + # TODO(sc): Maybe this shouldn't be synchronous - if this hangs, + # we could cause dnsmasq to get stuck... + rpcapi.handle_dhcp_lease(ctxt, tags, mac, ip_address, cid) + + +def del_lease(mac, ip_address): + """Called when a lease expires.""" + # We will only delete the ihost when it is requested by the user. + pass + + +def add_action_parsers(subparsers): + # NOTE(cfb): dnsmasq always passes mac, and ip. hostname + # is passed if known. We don't care about + # hostname, but argparse will complain if we + # do not accept it. + for action in ['add', 'del', 'old']: + parser = subparsers.add_parser(action) + parser.add_argument('mac') + parser.add_argument('ip') + parser.add_argument('hostname', nargs='?', default='') + parser.set_defaults(func=globals()[action + '_lease']) + + +CONF.register_cli_opt( + cfg.SubCommandOpt('action', + title='Action options', + help='Available dnsmasq_lease_update options', + handler=add_action_parsers)) + + +def main(): + # Parse config file and command line options, then start logging + # The mac is to be truncated to 17 characters, which is the proper + # length of a mac address, in order to handle IPv6 where a DUID + # is provided instead of a mac address. The truncated DUID is + # then equivalent to the mac address. + inventory_service.prepare_service(sys.argv) + + LOG = log.getLogger(__name__) + + if CONF.action.name in ['add', 'del', 'old']: + msg = (_("Called '%(action)s' for mac '%(mac)s' with ip '%(ip)s'") % + {"action": CONF.action.name, + "mac": CONF.action.mac[-17:], + "ip": CONF.action.ip}) + LOG.info(msg) + CONF.action.func(CONF.action.mac[-17:], CONF.action.ip) + else: + LOG.error(_("Unknown action: %(action)") % {"action": + CONF.action.name}) diff --git a/inventory/inventory/inventory/common/__init__.py b/inventory/inventory/inventory/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/common/base.py b/inventory/inventory/inventory/common/base.py new file mode 100644 index 00000000..429d2167 --- /dev/null +++ b/inventory/inventory/inventory/common/base.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from oslo_log import log +LOG = log.getLogger(__name__) + + +class APIResourceWrapper(object): + """Simple wrapper for api objects. + + Define _attrs on the child class and pass in the + api object as the only argument to the constructor + """ + _attrs = [] + _apiresource = None # Make sure _apiresource is there even in __init__. + + def __init__(self, apiresource): + self._apiresource = apiresource + + def __getattribute__(self, attr): + try: + return object.__getattribute__(self, attr) + except AttributeError: + if attr not in self._attrs: + raise + # __getattr__ won't find properties + return getattr(self._apiresource, attr) + + def __repr__(self): + return "<%s: %s>" % (self.__class__.__name__, + dict((attr, getattr(self, attr)) + for attr in self._attrs + if hasattr(self, attr))) + + def as_dict(self): + obj = {} + for key in self._attrs: + obj[key] = getattr(self._apiresource, key, None) + return obj diff --git a/inventory/inventory/inventory/common/ceph.py b/inventory/inventory/inventory/common/ceph.py new file mode 100644 index 00000000..2f625629 --- /dev/null +++ b/inventory/inventory/inventory/common/ceph.py @@ -0,0 +1,211 @@ + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Copyright (c) 2016, 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# All Rights Reserved. +# + +""" Inventory Ceph Utilities and helper functions.""" + +from __future__ import absolute_import + +from cephclient import wrapper as ceph +from inventory.common import constants +from inventory.common import k_host +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class CephApiOperator(object): + """Class to encapsulate Ceph operations for Inventory API + Methods on object-based storage devices (OSDs). + """ + + def __init__(self): + self._ceph_api = ceph.CephWrapper( + endpoint='http://localhost:5001/api/v0.1/') + + def ceph_status_ok(self, timeout=10): + """ + returns rc bool. True if ceph ok, False otherwise + :param timeout: ceph api timeout + """ + rc = True + + try: + response, body = self._ceph_api.status(body='json', + timeout=timeout) + ceph_status = body['output']['health']['overall_status'] + if ceph_status != constants.CEPH_HEALTH_OK: + LOG.warn("ceph status=%s " % ceph_status) + rc = False + except Exception as e: + rc = False + LOG.warn("ceph status exception: %s " % e) + + return rc + + def _osd_quorum_names(self, timeout=10): + quorum_names = [] + try: + response, body = self._ceph_api.quorum_status(body='json', + timeout=timeout) + quorum_names = body['output']['quorum_names'] + except Exception as ex: + LOG.exception(ex) + return quorum_names + + return quorum_names + + def remove_osd_key(self, osdid): + osdid_str = "osd." + str(osdid) + # Remove the OSD authentication key + response, body = self._ceph_api.auth_del( + osdid_str, body='json') + if not response.ok: + LOG.error("Auth delete failed for OSD %s: %s", + osdid_str, response.reason) + + def osd_host_lookup(self, osd_id): + response, body = self._ceph_api.osd_crush_tree(body='json') + for i in range(0, len(body)): + # there are 2 chassis lists - cache-tier and root-tier + # that can be seen in the output of 'ceph osd crush tree': + # [{"id": -2,"name": "cache-tier", "type": "root", + # "type_id": 10, "items": [...]}, + # {"id": -1,"name": "storage-tier","type": "root", + # "type_id": 10, "items": [...]}] + chassis_list = body['output'][i]['items'] + for chassis in chassis_list: + # extract storage list/per chassis + storage_list = chassis['items'] + for storage in storage_list: + # extract osd list/per storage + storage_osd_list = storage['items'] + for osd in storage_osd_list: + if osd['id'] == osd_id: + # return storage name where osd is located + return storage['name'] + return None + + def check_osds_down_up(self, hostname, upgrade): + # check if osds from a storage are down/up + response, body = self._ceph_api.osd_tree(body='json') + osd_tree = body['output']['nodes'] + size = len(osd_tree) + for i in range(1, size): + if osd_tree[i]['type'] != "host": + continue + children_list = osd_tree[i]['children'] + children_num = len(children_list) + # when we do a storage upgrade, storage node must be locked + # and all the osds of that storage node must be down + if (osd_tree[i]['name'] == hostname): + for j in range(1, children_num + 1): + if (osd_tree[i + j]['type'] == + constants.STOR_FUNCTION_OSD and + osd_tree[i + j]['status'] == "up"): + # at least one osd is not down + return False + # all osds are up + return True + + def host_crush_remove(self, hostname): + # remove host from crushmap when system host-delete is executed + response, body = self._ceph_api.osd_crush_remove( + hostname, body='json') + + def host_osd_status(self, hostname): + # should prevent locking of a host if HEALTH_BLOCK + host_health = None + try: + response, body = self._ceph_api.pg_dump_stuck(body='json') + pg_detail = len(body['output']) + except Exception as e: + LOG.exception(e) + return host_health + + # osd_list is a list where I add + # each osd from pg_detail whose hostname + # is not equal with hostnamge given as parameter + osd_list = [] + for x in range(pg_detail): + # extract the osd and return the storage node + osd = body['output'][x]['acting'] + # osd is a list with osd where a stuck/degraded PG + # was replicated. If osd is empty, it means + # PG is not replicated to any osd + if not osd: + continue + osd_id = int(osd[0]) + if osd_id in osd_list: + continue + # potential future optimization to cache all the + # osd to host lookups for the single call to host_osd_status(). + host_name = self.osd_host_lookup(osd_id) + if (host_name is not None and + host_name == hostname): + # mark the selected storage node with HEALTH_BLOCK + # we can't lock any storage node marked with HEALTH_BLOCK + return constants.CEPH_HEALTH_BLOCK + osd_list.append(osd_id) + return constants.CEPH_HEALTH_OK + + def get_monitors_status(self, ihosts): + # first check that the monitors are available in inventory + num_active_monitors = 0 + num_inv_monitors = 0 + required_monitors = constants.MIN_STOR_MONITORS + quorum_names = [] + inventory_monitor_names = [] + for ihost in ihosts: + if ihost['personality'] == k_host.COMPUTE: + continue + capabilities = ihost['capabilities'] + if 'stor_function' in capabilities: + host_action = ihost['host_action'] or "" + locking = (host_action.startswith(k_host.ACTION_LOCK) or + host_action.startswith(k_host.ACTION_FORCE_LOCK)) + if (capabilities['stor_function'] == + constants.STOR_FUNCTION_MONITOR and + ihost['administrative'] == k_host.ADMIN_UNLOCKED and + ihost['operational'] == k_host.OPERATIONAL_ENABLED and + not locking): + num_inv_monitors += 1 + inventory_monitor_names.append(ihost['hostname']) + + LOG.info("Active ceph monitors in inventory = %s" % + str(inventory_monitor_names)) + + # check that the cluster is actually operational. + # if we can get the monitor quorum from ceph, then + # the cluster is truly operational + if num_inv_monitors >= required_monitors: + try: + quorum_names = self._osd_quorum_names() + except Exception: + # if the cluster is not responding to requests + # set quorum_names to an empty list, indicating a problem + quorum_names = [] + LOG.error("Ceph cluster not responding to requests.") + + LOG.info("Active ceph monitors in ceph cluster = %s" % + str(quorum_names)) + + # There may be cases where a host is in an unlocked-available state, + # but the monitor is down due to crashes or manual removal. + # For such cases, we determine the list of active ceph monitors to be + # the intersection of the inventory reported unlocked-available monitor + # hosts and the monitors reported in the quorum via the ceph API. + active_monitors = list(set(inventory_monitor_names) & + set(quorum_names)) + LOG.info("Active ceph monitors = %s" % str(active_monitors)) + + num_active_monitors = len(active_monitors) + + return num_active_monitors, required_monitors, active_monitors diff --git a/inventory/inventory/inventory/common/config.py b/inventory/inventory/inventory/common/config.py new file mode 100644 index 00000000..b3086df8 --- /dev/null +++ b/inventory/inventory/inventory/common/config.py @@ -0,0 +1,127 @@ +# Copyright 2016 Ericsson AB +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +File to store configurations +""" +from inventory.common import rpc +from inventory import version +from oslo_config import cfg + +global_opts = [ + cfg.BoolOpt('use_default_quota_class', + default=True, + help='Enables or disables use of default quota class ' + 'with default quota.'), + cfg.IntOpt('report_interval', + default=60, + help='Seconds between running periodic reporting tasks.'), +] + +# Pecan_opts +pecan_opts = [ + cfg.StrOpt( + 'root', + default='inventory.api.controllers.root.RootController', + help='Pecan root controller' + ), + cfg.ListOpt( + 'modules', + default=["inventory.api"], + help='A list of modules where pecan will search for applications.' + ), + cfg.BoolOpt( + 'debug', + default=False, + help='Enables the ability to display tracebacks in the browser and' + 'interactively debug during development.' + ), + cfg.BoolOpt( + 'auth_enable', + default=True, + help='Enables user authentication in pecan.' + ) +] + + +# OpenStack credentials used for Endpoint Cache +cache_opts = [ + cfg.StrOpt('auth_uri', + help='Keystone authorization url'), + cfg.StrOpt('identity_uri', + help='Keystone service url'), + cfg.StrOpt('admin_username', + help='Username of admin account, needed when' + ' auto_refresh_endpoint set to True'), + cfg.StrOpt('admin_password', + help='Password of admin account, needed when' + ' auto_refresh_endpoint set to True'), + cfg.StrOpt('admin_tenant', + help='Tenant name of admin account, needed when' + ' auto_refresh_endpoint set to True'), + cfg.StrOpt('admin_user_domain_name', + default='Default', + help='User domain name of admin account, needed when' + ' auto_refresh_endpoint set to True'), + cfg.StrOpt('admin_project_domain_name', + default='Default', + help='Project domain name of admin account, needed when' + ' auto_refresh_endpoint set to True') +] + +scheduler_opts = [ + cfg.BoolOpt('periodic_enable', + default=True, + help='boolean value for enable/disenable periodic tasks'), + cfg.IntOpt('periodic_interval', + default=600, + help='periodic time interval for automatic quota sync job' + ' and resource sync audit') +] + +common_opts = [ + cfg.IntOpt('workers', default=1, + help='number of workers'), +] + +scheduler_opt_group = cfg.OptGroup('scheduler', + title='Scheduler options for periodic job') + +# The group stores the pecan configurations. +pecan_group = cfg.OptGroup(name='pecan', + title='Pecan options') + +cache_opt_group = cfg.OptGroup(name='cache', + title='OpenStack Credentials') + + +def list_opts(): + yield cache_opt_group.name, cache_opts + yield scheduler_opt_group.name, scheduler_opts + yield pecan_group.name, pecan_opts + yield None, global_opts + yield None, common_opts + + +def register_options(): + for group, opts in list_opts(): + cfg.CONF.register_opts(opts, group=group) + + +def parse_args(argv, default_config_files=None): + rpc.set_defaults(control_exchange='inventory') + cfg.CONF(argv[1:], + project='inventory', + version=version.version_info.release_string(), + default_config_files=default_config_files) + rpc.init(cfg.CONF) diff --git a/inventory/inventory/inventory/common/constants.py b/inventory/inventory/inventory/common/constants.py new file mode 100644 index 00000000..d0b26f7e --- /dev/null +++ b/inventory/inventory/inventory/common/constants.py @@ -0,0 +1,612 @@ +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.common import k_host +import os +import tsconfig.tsconfig as tsc + +INVENTORY_RUNNING_IN_LAB = '/etc/inventory/.running_in_lab' +INVENTORY_CONFIG_PATH = \ + os.path.join(tsc.PLATFORM_PATH, "inventory", tsc.SW_VERSION) + +VIM_DEFAULT_TIMEOUT_IN_SECS = 5 +VIM_DELETE_TIMEOUT_IN_SECS = 10 +MTC_ADD_TIMEOUT_IN_SECS = 6 +MTC_DELETE_TIMEOUT_IN_SECS = 10 +MTC_DEFAULT_TIMEOUT_IN_SECS = 6 +HWMON_DEFAULT_TIMEOUT_IN_SECS = 6 +PATCH_DEFAULT_TIMEOUT_IN_SECS = 6 + +DB_SUPPRESS_STATUS = 1 +DB_MGMT_AFFECTING = 2 +DB_DEGRADE_AFFECTING = 3 + +# CPU functions +PLATFORM_FUNCTION = "Platform" +VSWITCH_FUNCTION = "Vswitch" +SHARED_FUNCTION = "Shared" +VM_FUNCTION = "VMs" +NO_FUNCTION = "None" + +# Hugepage sizes in MiB +MIB_2M = 2 +MIB_1G = 1024 +Ki = 1024 +NUM_4K_PER_MiB = 256 + +# Dynamic IO Resident Set Size(RSS) in MiB per socket +DISK_IO_RESIDENT_SET_SIZE_MIB = 2000 +DISK_IO_RESIDENT_SET_SIZE_MIB_VBOX = 500 + +# Memory reserved for platform core in MiB per host +PLATFORM_CORE_MEMORY_RESERVED_MIB = 2000 +PLATFORM_CORE_MEMORY_RESERVED_MIB_VBOX = 1100 + +# For combined node, memory reserved for controller in MiB +COMBINED_NODE_CONTROLLER_MEMORY_RESERVED_MIB = 10500 +COMBINED_NODE_CONTROLLER_MEMORY_RESERVED_MIB_VBOX = 6000 +COMBINED_NODE_CONTROLLER_MEMORY_RESERVED_MIB_XEOND = 7000 + +# Max number of physical cores in a xeon-d cpu +NUMBER_CORES_XEOND = 8 + +# Max number of computes that can be added to an AIO duplex system +AIO_DUPLEX_MAX_COMPUTES = 4 + +# Network overhead for DHCP or vrouter, assume 100 networks * 40 MB each +NETWORK_METADATA_OVERHEAD_MIB = 4000 +NETWORK_METADATA_OVERHEAD_MIB_VBOX = 0 + +# Sensors +SENSOR_DATATYPE_VALID_LIST = ['discrete', 'analog'] +HWMON_PORT = 2212 + +# Supported compute node vswitch types +VSWITCH_TYPE_OVS_DPDK = "ovs-dpdk" + +# Partition default sizes +DEFAULT_IMAGE_STOR_SIZE = 10 +DEFAULT_DOCKER_STOR_SIZE = 1 +DEFAULT_DOCKER_DISTRIBUTION_STOR_SIZE = 1 +DEFAULT_DATABASE_STOR_SIZE = 20 +DEFAULT_IMG_CONVERSION_STOR_SIZE = 20 +DEFAULT_SMALL_IMAGE_STOR_SIZE = 10 +DEFAULT_SMALL_DATABASE_STOR_SIZE = 10 +DEFAULT_SMALL_IMG_CONVERSION_STOR_SIZE = 10 +DEFAULT_SMALL_BACKUP_STOR_SIZE = 30 +DEFAULT_VIRTUAL_IMAGE_STOR_SIZE = 8 +DEFAULT_VIRTUAL_DATABASE_STOR_SIZE = 5 +DEFAULT_VIRTUAL_IMG_CONVERSION_STOR_SIZE = 8 +DEFAULT_VIRTUAL_BACKUP_STOR_SIZE = 5 +DEFAULT_EXTENSION_STOR_SIZE = 1 +DEFAULT_PATCH_VAULT_STOR_SIZE = 8 +DEFAULT_ETCD_STORE_SIZE = 1 +DEFAULT_GNOCCHI_STOR_SIZE = 5 + +# Openstack Interface names +OS_INTERFACE_PUBLIC = 'public' +OS_INTERFACE_INTERNAL = 'internal' +OS_INTERFACE_ADMIN = 'admin' + +# Default region one name +REGION_ONE_NAME = 'RegionOne' +# DC Region Must match VIRTUAL_MASTER_CLOUD in dcorch +SYSTEM_CONTROLLER_REGION = 'SystemController' + +# Valid major numbers for disks: +# https://www.kernel.org/doc/Documentation/admin-guide/devices.txt +# +# 3 block First MFM, RLL and IDE hard disk/CD-ROM interface +# 8 block SCSI disk devices (0-15) +# 65 block SCSI disk devices (16-31) +# 66 block SCSI disk devices (32-47) +# 67 block SCSI disk devices (48-63) +# 68 block SCSI disk devices (64-79) +# 69 block SCSI disk devices (80-95) +# 70 block SCSI disk devices (96-111) +# 71 block SCSI disk devices (112-127) +# 128 block SCSI disk devices (128-143) +# 129 block SCSI disk devices (144-159) +# 130 block SCSI disk devices (160-175) +# 131 block SCSI disk devices (176-191) +# 132 block SCSI disk devices (192-207) +# 133 block SCSI disk devices (208-223) +# 134 block SCSI disk devices (224-239) +# 135 block SCSI disk devices (240-255) +# 240-254 block LOCAL/EXPERIMENTAL USE (253 == /dev/vdX) +# 259 block Block Extended Major (NVMe - /dev/nvmeXn1) +VALID_MAJOR_LIST = ['3', '8', '65', '66', '67', '68', '69', '70', '71', + '128', '129', '130', '131', '132', '133', '134', + '135', '253', '259'] +VENDOR_ID_LIO = 'LIO-ORG' + +# Storage backends supported +SB_TYPE_FILE = 'file' +SB_TYPE_LVM = 'lvm' +SB_TYPE_CEPH = 'ceph' +SB_TYPE_CEPH_EXTERNAL = 'ceph-external' +SB_TYPE_EXTERNAL = 'external' + +SB_SUPPORTED = [SB_TYPE_FILE, + SB_TYPE_LVM, + SB_TYPE_CEPH, + SB_TYPE_CEPH_EXTERNAL, + SB_TYPE_EXTERNAL] + +# Storage backend default names +SB_DEFAULT_NAME_SUFFIX = "-store" +SB_DEFAULT_NAMES = { + SB_TYPE_FILE: SB_TYPE_FILE + SB_DEFAULT_NAME_SUFFIX, + SB_TYPE_LVM: SB_TYPE_LVM + SB_DEFAULT_NAME_SUFFIX, + SB_TYPE_CEPH: SB_TYPE_CEPH + SB_DEFAULT_NAME_SUFFIX, + SB_TYPE_CEPH_EXTERNAL: SB_TYPE_CEPH_EXTERNAL + SB_DEFAULT_NAME_SUFFIX, + SB_TYPE_EXTERNAL: 'shared_services' +} + +# Storage backends services +SB_SVC_CINDER = 'cinder' +SB_SVC_GLANCE = 'glance' +SB_SVC_NOVA = 'nova' +SB_SVC_SWIFT = 'swift' + +SB_FILE_SVCS_SUPPORTED = [SB_SVC_GLANCE] +SB_LVM_SVCS_SUPPORTED = [SB_SVC_CINDER] +SB_CEPH_SVCS_SUPPORTED = [ + SB_SVC_GLANCE, SB_SVC_CINDER, + SB_SVC_SWIFT, SB_SVC_NOVA] # supported primary tier svc +SB_CEPH_EXTERNAL_SVCS_SUPPORTED = [SB_SVC_CINDER, SB_SVC_GLANCE, SB_SVC_NOVA] +SB_EXTERNAL_SVCS_SUPPORTED = [SB_SVC_CINDER, SB_SVC_GLANCE] + +# Storage backend: Service specific backend nomenclature +CINDER_BACKEND_CEPH = SB_TYPE_CEPH +CINDER_BACKEND_CEPH_EXTERNAL = SB_TYPE_CEPH_EXTERNAL +CINDER_BACKEND_LVM = SB_TYPE_LVM +GLANCE_BACKEND_FILE = SB_TYPE_FILE +GLANCE_BACKEND_RBD = 'rbd' +GLANCE_BACKEND_HTTP = 'http' +GLANCE_BACKEND_GLANCE = 'glance' + +# Storage Tiers: types (aligns with polymorphic backends) +SB_TIER_TYPE_CEPH = SB_TYPE_CEPH +SB_TIER_SUPPORTED = [SB_TIER_TYPE_CEPH] +SB_TIER_DEFAULT_NAMES = { + SB_TIER_TYPE_CEPH: 'storage' # maps to crushmap 'storage-tier' root +} +SB_TIER_CEPH_SECONDARY_SVCS = [SB_SVC_CINDER] # supported secondary tier svcs + +SB_TIER_STATUS_DEFINED = 'defined' +SB_TIER_STATUS_IN_USE = 'in-use' + +# File name reserved for internal ceph cluster. +SB_TYPE_CEPH_CONF_FILENAME = "ceph.conf" + +# Glance images path when it is file backended +GLANCE_IMAGE_PATH = tsc.CGCS_PATH + "/" + SB_SVC_GLANCE + "/images" + +# Path for Ceph (internal and external) config files +CEPH_CONF_PATH = "/etc/ceph/" + +# Requested storage backend API operations +SB_API_OP_CREATE = "create" +SB_API_OP_MODIFY = "modify" +SB_API_OP_DELETE = "delete" + +# Storage backend state +SB_STATE_CONFIGURED = 'configured' +SB_STATE_CONFIGURING = 'configuring' +SB_STATE_CONFIG_ERR = 'configuration-failed' + +# Storage backend tasks +SB_TASK_NONE = None +SB_TASK_APPLY_MANIFESTS = 'applying-manifests' +SB_TASK_APPLY_CONFIG_FILE = 'applying-config-file' +SB_TASK_RECONFIG_CONTROLLER = 'reconfig-controller' +SB_TASK_PROVISION_STORAGE = 'provision-storage' +SB_TASK_PROVISION_SERVICES = 'provision-services' +SB_TASK_RECONFIG_COMPUTE = 'reconfig-compute' +SB_TASK_RESIZE_CEPH_MON_LV = 'resize-ceph-mon-lv' +SB_TASK_ADD_OBJECT_GATEWAY = 'add-object-gateway' +SB_TASK_RESTORE = 'restore' + +# Storage backend ceph-mon-lv size +SB_CEPH_MON_GIB = 20 +SB_CEPH_MON_GIB_MIN = 20 +SB_CEPH_MON_GIB_MAX = 40 + +SB_CONFIGURATION_TIMEOUT = 1200 + +# Storage: Minimum number of monitors +MIN_STOR_MONITORS = 2 + +# Suffix used in LVM volume name to indicate that the +# volume is actually a thin pool. (And thin volumes will +# be created in the thin pool.) +LVM_POOL_SUFFIX = '-pool' + +# File system names +FILESYSTEM_NAME_BACKUP = 'backup' +FILESYSTEM_NAME_CGCS = 'cgcs' +FILESYSTEM_DISPLAY_NAME_CGCS = 'glance' +FILESYSTEM_NAME_CINDER = 'cinder' +FILESYSTEM_NAME_DATABASE = 'database' +FILESYSTEM_NAME_IMG_CONVERSIONS = 'img-conversions' +FILESYSTEM_NAME_SCRATCH = 'scratch' +FILESYSTEM_NAME_DOCKER = 'docker' +FILESYSTEM_NAME_DOCKER_DISTRIBUTION = 'docker-distribution' +FILESYSTEM_NAME_EXTENSION = 'extension' +FILESYSTEM_NAME_ETCD = 'etcd' +FILESYSTEM_NAME_PATCH_VAULT = 'patch-vault' +FILESYSTEM_NAME_GNOCCHI = 'gnocchi' + +FILESYSTEM_LV_DICT = { + FILESYSTEM_NAME_CGCS: 'cgcs-lv', + FILESYSTEM_NAME_BACKUP: 'backup-lv', + FILESYSTEM_NAME_SCRATCH: 'scratch-lv', + FILESYSTEM_NAME_DOCKER: 'docker-lv', + FILESYSTEM_NAME_DOCKER_DISTRIBUTION: 'dockerdistribution-lv', + FILESYSTEM_NAME_IMG_CONVERSIONS: 'img-conversions-lv', + FILESYSTEM_NAME_DATABASE: 'pgsql-lv', + FILESYSTEM_NAME_EXTENSION: 'extension-lv', + FILESYSTEM_NAME_ETCD: 'etcd-lv', + FILESYSTEM_NAME_PATCH_VAULT: 'patch-vault-lv', + FILESYSTEM_NAME_GNOCCHI: 'gnocchi-lv' +} + +SUPPORTED_LOGICAL_VOLUME_LIST = FILESYSTEM_LV_DICT.values() + +SUPPORTED_FILEYSTEM_LIST = [ + FILESYSTEM_NAME_BACKUP, + FILESYSTEM_NAME_CGCS, + FILESYSTEM_NAME_CINDER, + FILESYSTEM_NAME_DATABASE, + FILESYSTEM_NAME_EXTENSION, + FILESYSTEM_NAME_IMG_CONVERSIONS, + FILESYSTEM_NAME_SCRATCH, + FILESYSTEM_NAME_DOCKER, + FILESYSTEM_NAME_DOCKER_DISTRIBUTION, + FILESYSTEM_NAME_PATCH_VAULT, + FILESYSTEM_NAME_ETCD, + FILESYSTEM_NAME_GNOCCHI +] + +SUPPORTED_REPLICATED_FILEYSTEM_LIST = [ + FILESYSTEM_NAME_CGCS, + FILESYSTEM_NAME_DATABASE, + FILESYSTEM_NAME_EXTENSION, + FILESYSTEM_NAME_PATCH_VAULT, + FILESYSTEM_NAME_ETCD, + FILESYSTEM_NAME_DOCKER_DISTRIBUTION, +] + +# Storage: Volume Group Types +LVG_NOVA_LOCAL = 'nova-local' +LVG_CGTS_VG = 'cgts-vg' +LVG_CINDER_VOLUMES = 'cinder-volumes' +LVG_ALLOWED_VGS = [LVG_NOVA_LOCAL, LVG_CGTS_VG, LVG_CINDER_VOLUMES] + +# Cinder LVM Parameters +CINDER_LVM_MINIMUM_DEVICE_SIZE_GIB = 5 # GiB +CINDER_LVM_DRBD_RESOURCE = 'drbd-cinder' +CINDER_LVM_DRBD_WAIT_PEER_RETRY = 5 +CINDER_LVM_DRBD_WAIT_PEER_SLEEP = 2 +CINDER_LVM_POOL_LV = LVG_CINDER_VOLUMES + LVM_POOL_SUFFIX +CINDER_LVM_POOL_META_LV = CINDER_LVM_POOL_LV + "_tmeta" +CINDER_RESIZE_FAILURE = "cinder-resize-failure" +CINDER_DRBD_DEVICE = '/dev/drbd4' + +CINDER_LVM_TYPE_THIN = 'thin' +CINDER_LVM_TYPE_THICK = 'thick' + +# Storage: Volume Group Parameter Types +LVG_NOVA_PARAM_BACKING = 'instance_backing' +LVG_NOVA_PARAM_INST_LV_SZ = 'instances_lv_size_mib' +LVG_NOVA_PARAM_INST_LV_SZ_GIB = 'instances_lv_size_gib' +LVG_NOVA_PARAM_DISK_OPS = 'concurrent_disk_operations' +LVG_CINDER_PARAM_LVM_TYPE = 'lvm_type' + +# Storage: Volume Group Parameter: Nova: Backing types +LVG_NOVA_BACKING_LVM = 'lvm' +LVG_NOVA_BACKING_IMAGE = 'image' +LVG_NOVA_BACKING_REMOTE = 'remote' + +# Storage: Volume Group Parameter: Cinder: LVM provisioing +LVG_CINDER_LVM_TYPE_THIN = 'thin' +LVG_CINDER_LVM_TYPE_THICK = 'thick' + +# Storage: Volume Group Parameter: Nova: Instances LV +LVG_NOVA_PARAM_INST_LV_SZ_DEFAULT = 0 + +# Storage: Volume Group Parameter: Nova: Concurrent Disk Ops +LVG_NOVA_PARAM_DISK_OPS_DEFAULT = 2 + +# Controller audit requests (force updates from agents) +DISK_AUDIT_REQUEST = "audit_disk" +LVG_AUDIT_REQUEST = "audit_lvg" +PV_AUDIT_REQUEST = "audit_pv" +PARTITION_AUDIT_REQUEST = "audit_partition" +CONTROLLER_AUDIT_REQUESTS = [DISK_AUDIT_REQUEST, + LVG_AUDIT_REQUEST, + PV_AUDIT_REQUEST, + PARTITION_AUDIT_REQUEST] + +# IP families +IPV4_FAMILY = 4 +IPV6_FAMILY = 6 +IP_FAMILIES = {IPV4_FAMILY: "IPv4", + IPV6_FAMILY: "IPv6"} + +# Interface definitions +NETWORK_TYPE_NONE = 'none' +NETWORK_TYPE_INFRA = 'infra' +NETWORK_TYPE_MGMT = 'mgmt' +NETWORK_TYPE_OAM = 'oam' +NETWORK_TYPE_BM = 'bm' +NETWORK_TYPE_MULTICAST = 'multicast' +NETWORK_TYPE_DATA = 'data' +NETWORK_TYPE_SYSTEM_CONTROLLER = 'system-controller' + +NETWORK_TYPE_PCI_PASSTHROUGH = 'pci-passthrough' +NETWORK_TYPE_PCI_SRIOV = 'pci-sriov' +NETWORK_TYPE_PXEBOOT = 'pxeboot' + +PLATFORM_NETWORK_TYPES = [NETWORK_TYPE_PXEBOOT, + NETWORK_TYPE_MGMT, + NETWORK_TYPE_INFRA, + NETWORK_TYPE_OAM] + +PCI_NETWORK_TYPES = [NETWORK_TYPE_PCI_PASSTHROUGH, + NETWORK_TYPE_PCI_SRIOV] + +INTERFACE_TYPE_ETHERNET = 'ethernet' +INTERFACE_TYPE_VLAN = 'vlan' +INTERFACE_TYPE_AE = 'ae' +INTERFACE_TYPE_VIRTUAL = 'virtual' + +INTERFACE_CLASS_NONE = 'none' +INTERFACE_CLASS_PLATFORM = 'platform' +INTERFACE_CLASS_DATA = 'data' +INTERFACE_CLASS_PCI_PASSTHROUGH = 'pci-passthrough' +INTERFACE_CLASS_PCI_SRIOV = 'pci-sriov' + +SM_MULTICAST_MGMT_IP_NAME = "sm-mgmt-ip" +MTCE_MULTICAST_MGMT_IP_NAME = "mtce-mgmt-ip" +PATCH_CONTROLLER_MULTICAST_MGMT_IP_NAME = "patch-controller-mgmt-ip" +PATCH_AGENT_MULTICAST_MGMT_IP_NAME = "patch-agent-mgmt-ip" +SYSTEM_CONTROLLER_GATEWAY_IP_NAME = "system-controller-gateway-ip" + +ADDRESS_FORMAT_ARGS = (k_host.CONTROLLER_HOSTNAME, + NETWORK_TYPE_MGMT) +MGMT_CINDER_IP_NAME = "%s-cinder-%s" % ADDRESS_FORMAT_ARGS + +ETHERNET_NULL_MAC = '00:00:00:00:00:00' + +DEFAULT_MTU = 1500 + +# Stor function types +STOR_FUNCTION_CINDER = 'cinder' +STOR_FUNCTION_OSD = 'osd' +STOR_FUNCTION_MONITOR = 'monitor' +STOR_FUNCTION_JOURNAL = 'journal' + +# Disk types and names. +DEVICE_TYPE_HDD = 'HDD' +DEVICE_TYPE_SSD = 'SSD' +DEVICE_TYPE_NVME = 'NVME' +DEVICE_TYPE_UNDETERMINED = 'Undetermined' +DEVICE_TYPE_NA = 'N/A' +DEVICE_NAME_NVME = 'nvme' + +# Disk model types. +DEVICE_MODEL_UNKNOWN = 'Unknown' + +# Journal operations. +ACTION_CREATE_JOURNAL = "create" +ACTION_UPDATE_JOURNAL = "update" + +# Load constants +MNT_DIR = '/tmp/mnt' + +ACTIVE_LOAD_STATE = 'active' +IMPORTING_LOAD_STATE = 'importing' +IMPORTED_LOAD_STATE = 'imported' +ERROR_LOAD_STATE = 'error' +DELETING_LOAD_STATE = 'deleting' + +DELETE_LOAD_SCRIPT = '/etc/inventory/upgrades/delete_load.sh' + +# Ceph +CEPH_HEALTH_OK = 'HEALTH_OK' +CEPH_HEALTH_BLOCK = 'HEALTH_BLOCK' + +# See http://ceph.com/pgcalc/. We set it to more than 100 because pool usage +# varies greatly in Titanium Cloud and we want to avoid running too low on PGs +CEPH_TARGET_PGS_PER_OSD = 200 +CEPH_REPLICATION_FACTOR_DEFAULT = 2 +CEPH_REPLICATION_FACTOR_SUPPORTED = [2, 3] +CEPH_MIN_REPLICATION_FACTOR_SUPPORTED = [1, 2] +CEPH_REPLICATION_MAP_DEFAULT = { + # replication: min_replication + 2: 1, + 3: 2 +} +# ceph osd pool size +CEPH_BACKEND_REPLICATION_CAP = 'replication' +# ceph osd pool min size +CEPH_BACKEND_MIN_REPLICATION_CAP = 'min_replication' +CEPH_BACKEND_CAP_DEFAULT = { + CEPH_BACKEND_REPLICATION_CAP: + str(CEPH_REPLICATION_FACTOR_DEFAULT), + CEPH_BACKEND_MIN_REPLICATION_CAP: + str(CEPH_REPLICATION_MAP_DEFAULT[CEPH_REPLICATION_FACTOR_DEFAULT]) +} + +# Service Parameter +SERVICE_TYPE_IDENTITY = 'identity' +SERVICE_TYPE_KEYSTONE = 'keystone' +SERVICE_TYPE_IMAGE = 'image' +SERVICE_TYPE_VOLUME = 'volume' +SERVICE_TYPE_NETWORK = 'network' +SERVICE_TYPE_HORIZON = "horizon" +SERVICE_TYPE_CEPH = 'ceph' +SERVICE_TYPE_CINDER = 'cinder' +SERVICE_TYPE_MURANO = 'murano' +SERVICE_TYPE_MAGNUM = 'magnum' +SERVICE_TYPE_PLATFORM = 'configuration' +SERVICE_TYPE_NOVA = 'nova' +SERVICE_TYPE_SWIFT = 'swift' +SERVICE_TYPE_IRONIC = 'ironic' +SERVICE_TYPE_PANKO = 'panko' +SERVICE_TYPE_AODH = 'aodh' +SERVICE_TYPE_GLANCE = 'glance' +SERVICE_TYPE_BARBICAN = 'barbican' + +# TIS part number, CPE = combined load, STD = standard load +TIS_STD_BUILD = 'Standard' +TIS_AIO_BUILD = 'All-in-one' + +# wrsroot password aging. +# Setting aging to max defined value qualifies +# as "never" on certain Linux distros including WRL +WRSROOT_PASSWORD_NO_AGING = 99999 + +# Partition table size in bytes. +PARTITION_TABLE_SIZE = 2097152 + +# States that describe the states of a partition. + +# Partition is ready for being used. +PARTITION_READY_STATUS = 0 +# Partition is used by a PV. +PARTITION_IN_USE_STATUS = 1 +# An in-service request to create the partition has been sent. +PARTITION_CREATE_IN_SVC_STATUS = 2 +# An unlock request to create the partition has been sent. +PARTITION_CREATE_ON_UNLOCK_STATUS = 3 +# A request to delete the partition has been sent. +PARTITION_DELETING_STATUS = 4 +# A request to modify the partition has been sent. +PARTITION_MODIFYING_STATUS = 5 +# The partition has been deleted. +PARTITION_DELETED_STATUS = 6 +# The creation of the partition has encountered a known error. +PARTITION_ERROR_STATUS = 10 +# Partition creation failed due to an internal error, check packstack logs. +PARTITION_ERROR_STATUS_INTERNAL = 11 +# Partition was not created because disk does not have a GPT. +PARTITION_ERROR_STATUS_GPT = 12 + +PARTITION_STATUS_MSG = { + PARTITION_IN_USE_STATUS: "In-Use", + PARTITION_CREATE_IN_SVC_STATUS: "Creating", + PARTITION_CREATE_ON_UNLOCK_STATUS: "Creating (on unlock)", + PARTITION_DELETING_STATUS: "Deleting", + PARTITION_MODIFYING_STATUS: "Modifying", + PARTITION_READY_STATUS: "Ready", + PARTITION_DELETED_STATUS: "Deleted", + PARTITION_ERROR_STATUS: "Error", + PARTITION_ERROR_STATUS_INTERNAL: "Error: Internal script error.", + PARTITION_ERROR_STATUS_GPT: "Error:Missing GPT Table."} + +PARTITION_STATUS_OK_TO_DELETE = [ + PARTITION_READY_STATUS, + PARTITION_CREATE_ON_UNLOCK_STATUS, + PARTITION_ERROR_STATUS, + PARTITION_ERROR_STATUS_INTERNAL, + PARTITION_ERROR_STATUS_GPT] + +PARTITION_STATUS_SEND_DELETE_RPC = [ + PARTITION_READY_STATUS, + PARTITION_ERROR_STATUS, + PARTITION_ERROR_STATUS_INTERNAL] + +PARTITION_CMD_CREATE = "create" +PARTITION_CMD_DELETE = "delete" +PARTITION_CMD_MODIFY = "modify" + +# User creatable, system managed, GUID partitions types. +PARTITION_USER_MANAGED_GUID_PREFIX = "ba5eba11-0000-1111-2222-" +USER_PARTITION_PHYSICAL_VOLUME = \ + PARTITION_USER_MANAGED_GUID_PREFIX + "000000000001" +LINUX_LVM_PARTITION = "e6d6d379-f507-44c2-a23c-238f2a3df928" + +# Partition name for those partitions deignated for PV use. +PARTITION_NAME_PV = "LVM Physical Volume" + +# Partition table types. +PARTITION_TABLE_GPT = "gpt" +PARTITION_TABLE_MSDOS = "msdos" + +# Optional services +ALL_OPTIONAL_SERVICES = [SERVICE_TYPE_CINDER, SERVICE_TYPE_MURANO, + SERVICE_TYPE_MAGNUM, SERVICE_TYPE_SWIFT, + SERVICE_TYPE_IRONIC] + +# System mode +SYSTEM_MODE_DUPLEX = "duplex" +SYSTEM_MODE_SIMPLEX = "simplex" +SYSTEM_MODE_DUPLEX_DIRECT = "duplex-direct" + +# System Security Profiles +SYSTEM_SECURITY_PROFILE_STANDARD = "standard" +SYSTEM_SECURITY_PROFILE_EXTENDED = "extended" + +# Install states +INSTALL_STATE_PRE_INSTALL = "preinstall" +INSTALL_STATE_INSTALLING = "installing" +INSTALL_STATE_POST_INSTALL = "postinstall" +INSTALL_STATE_FAILED = "failed" +INSTALL_STATE_INSTALLED = "installed" +INSTALL_STATE_BOOTING = "booting" +INSTALL_STATE_COMPLETED = "completed" + +tox_work_dir = os.environ.get("TOX_WORK_DIR") +if tox_work_dir: + INVENTORY_LOCK_PATH = tox_work_dir +else: + INVENTORY_LOCK_PATH = os.path.join(tsc.VOLATILE_PATH, "inventory") + +NETWORK_CONFIG_LOCK_FILE = os.path.join( + tsc.VOLATILE_PATH, "apply_network_config.lock") + +INVENTORY_USERNAME = "inventory" +INVENTORY_GRPNAME = "inventory" + +# License file +LICENSE_FILE = ".license" + +# Cinder lvm config complete file. +NODE_CINDER_LVM_CONFIG_COMPLETE_FILE = \ + os.path.join(tsc.PLATFORM_CONF_PATH, '.node_cinder_lvm_config_complete') +INITIAL_CINDER_LVM_CONFIG_COMPLETE_FILE = \ + os.path.join(tsc.CONFIG_PATH, '.initial_cinder_lvm_config_complete') + +DISK_WIPE_IN_PROGRESS_FLAG = \ + os.path.join(tsc.PLATFORM_CONF_PATH, '.disk_wipe_in_progress') +DISK_WIPE_COMPLETE_TIMEOUT = 5 # wait for a disk to finish wiping. + +# Clone label set in DB +CLONE_ISO_MAC = 'CLONEISOMAC_' +CLONE_ISO_DISK_SID = 'CLONEISODISKSID_' + +# kernel options for various security feature selections +SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_V1 = 'spectre_meltdown_v1' +SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_V1_OPTS = 'nopti nospectre_v2' +SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_ALL = 'spectre_meltdown_all' +SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_ALL_OPTS = '' +SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_OPTS = { + SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_V1: + SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_V1_OPTS, + SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_ALL: + SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_ALL_OPTS +} + + +SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_DEFAULT_OPTS = \ + SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_V1_OPTS diff --git a/inventory/inventory/inventory/common/context.py b/inventory/inventory/inventory/common/context.py new file mode 100644 index 00000000..10008f62 --- /dev/null +++ b/inventory/inventory/inventory/common/context.py @@ -0,0 +1,153 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from keystoneauth1.access import service_catalog as k_service_catalog +from keystoneauth1 import plugin +from oslo_config import cfg +from oslo_context import context + +from inventory.common import policy + +REQUIRED_SERVICE_TYPES = ('faultmanagement', + 'nfv', + 'patching', + 'platform', + 'smapi', + ) + + +CONF = cfg.CONF + + +class _ContextAuthPlugin(plugin.BaseAuthPlugin): + """A keystoneauth auth plugin that uses the values from the Context. + + Ideally we would use the plugin provided by auth_token middleware however + this plugin isn't serialized yet so we construct one from the serialized + auth data. + """ + + def __init__(self, auth_token, sc): + super(_ContextAuthPlugin, self).__init__() + + self.auth_token = auth_token + self.service_catalog = k_service_catalog.ServiceCatalogV2(sc) + + def get_token(self, *args, **kwargs): + return self.auth_token + + def get_endpoint(self, session, service_type=None, interface=None, + region_name=None, service_name=None, **kwargs): + return self.service_catalog.url_for(service_type=service_type, + service_name=service_name, + interface=interface, + region_name=region_name) + + +class RequestContext(context.RequestContext): + """Extends security contexts from the OpenStack common library.""" + + def __init__(self, auth_token=None, auth_url=None, domain_id=None, + domain_name=None, user_name=None, user_id=None, + user_domain_name=None, user_domain_id=None, + project_name=None, project_id=None, roles=None, + is_admin=None, read_only=False, show_deleted=False, + request_id=None, trust_id=None, auth_token_info=None, + all_tenants=False, password=None, service_catalog=None, + user_auth_plugin=None, + **kwargs): + """Stores several additional request parameters: + + :param domain_id: The ID of the domain. + :param domain_name: The name of the domain. + :param user_domain_id: The ID of the domain to + authenticate a user against. + :param user_domain_name: The name of the domain to + authenticate a user against. + :param service_catalog: Specifies the service_catalog + """ + super(RequestContext, self).__init__(auth_token=auth_token, + user=user_name, + tenant=project_name, + is_admin=is_admin, + read_only=read_only, + show_deleted=show_deleted, + request_id=request_id, + roles=roles) + + self.user_name = user_name + self.user_id = user_id + self.project_name = project_name + self.project_id = project_id + self.domain_id = domain_id + self.domain_name = domain_name + self.user_domain_id = user_domain_id + self.user_domain_name = user_domain_name + self.auth_url = auth_url + self.auth_token_info = auth_token_info + self.trust_id = trust_id + self.all_tenants = all_tenants + self.password = password + + if service_catalog: + # Only include required parts of service_catalog + self.service_catalog = [s for s in service_catalog + if s.get('type') in + REQUIRED_SERVICE_TYPES] + else: + # if list is empty or none + self.service_catalog = [] + + self.user_auth_plugin = user_auth_plugin + if is_admin is None: + self.is_admin = policy.check_is_admin(self) + else: + self.is_admin = is_admin + + def to_dict(self): + value = super(RequestContext, self).to_dict() + value.update({'auth_token': self.auth_token, + 'auth_url': self.auth_url, + 'domain_id': self.domain_id, + 'domain_name': self.domain_name, + 'user_domain_id': self.user_domain_id, + 'user_domain_name': self.user_domain_name, + 'user_name': self.user_name, + 'user_id': self.user_id, + 'project_name': self.project_name, + 'project_id': self.project_id, + 'is_admin': self.is_admin, + 'read_only': self.read_only, + 'roles': self.roles, + 'show_deleted': self.show_deleted, + 'request_id': self.request_id, + 'trust_id': self.trust_id, + 'auth_token_info': self.auth_token_info, + 'password': self.password, + 'all_tenants': self.all_tenants, + 'service_catalog': self.service_catalog}) + return value + + @classmethod + def from_dict(cls, values): + return cls(**values) + + def get_auth_plugin(self): + if self.user_auth_plugin: + return self.user_auth_plugin + else: + return _ContextAuthPlugin(self.auth_token, self.service_catalog) + + +def make_context(*args, **kwargs): + return RequestContext(*args, **kwargs) + + +def get_admin_context(show_deleted="no"): + context = make_context(tenant=None, + is_admin=True, + show_deleted=show_deleted) + return context diff --git a/inventory/inventory/inventory/common/exception.py b/inventory/inventory/inventory/common/exception.py new file mode 100644 index 00000000..a7e931d2 --- /dev/null +++ b/inventory/inventory/inventory/common/exception.py @@ -0,0 +1,738 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Inventory base exception handling. +""" + +import six +import webob.exc + +from inventory.common.i18n import _ +from inventory.conf import CONF +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class ProcessExecutionError(IOError): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = _('Unexpected error while running command.') + if exit_code is None: + exit_code = '-' + message = (_('%(description)s\nCommand: %(cmd)s\n' + 'Exit code: %(exit_code)s\nStdout: %(stdout)r\n' + 'Stderr: %(stderr)r') % + {'description': description, 'cmd': cmd, + 'exit_code': exit_code, 'stdout': stdout, + 'stderr': stderr}) + IOError.__init__(self, message) + + +def _cleanse_dict(original): + """Strip all admin_password, new_pass, rescue_pass keys from a dict.""" + return dict((k, v) for k, v in original.iteritems() if "_pass" not in k) + + +class InventoryException(Exception): + """Base Inventory Exception + + To correctly use this class, inherit from it and define + a 'message' property. That message will get printf'd + with the keyword arguments provided to the constructor. + + """ + message = _("An unknown exception occurred.") + code = 500 + headers = {} + safe = False + + def __init__(self, message=None, **kwargs): + self.kwargs = kwargs + + if 'code' not in self.kwargs: + try: + self.kwargs['code'] = self.code + except AttributeError: + pass + + if not message: + try: + message = self.message % kwargs + + except Exception as e: + # kwargs doesn't match a variable in the message + # log the issue and the kwargs + LOG.exception(_('Exception in string format operation')) + for name, value in kwargs.iteritems(): + LOG.error("%s: %s" % (name, value)) + + if CONF.fatal_exception_format_errors: + raise e + else: + # at least get the core message out if something happened + message = self.message + + super(InventoryException, self).__init__(message) + + def format_message(self): + if self.__class__.__name__.endswith('_Remote'): + return self.args[0] + else: + return six.text_type(self) + + +class NotAuthorized(InventoryException): + message = _("Not authorized.") + code = 403 + + +class AdminRequired(NotAuthorized): + message = _("User does not have admin privileges") + + +class PolicyNotAuthorized(NotAuthorized): + message = _("Policy doesn't allow %(action)s to be performed.") + + +class OperationNotPermitted(NotAuthorized): + message = _("Operation not permitted.") + + +class Invalid(InventoryException): + message = _("Unacceptable parameters.") + code = 400 + + +class Conflict(InventoryException): + message = _('Conflict.') + code = 409 + + +class InvalidCPUInfo(Invalid): + message = _("Unacceptable CPU info") + ": %(reason)s" + + +class InvalidIpAddressError(Invalid): + message = _("%(address)s is not a valid IP v4/6 address.") + + +class IpAddressOutOfRange(Invalid): + message = _("%(address)s is not in the range: %(low)s to %(high)s") + + +class InfrastructureNetworkNotConfigured(Invalid): + message = _("An infrastructure network has not been configured") + + +class InvalidDiskFormat(Invalid): + message = _("Disk format %(disk_format)s is not acceptable") + + +class InvalidUUID(Invalid): + message = _("Expected a uuid but received %(uuid)s.") + + +class InvalidIPAddress(Invalid): + message = _("Expected an IPv4 or IPv6 address but received %(address)s.") + + +class InvalidIdentity(Invalid): + message = _("Expected an uuid or int but received %(identity)s.") + + +class PatchError(Invalid): + message = _("Couldn't apply patch '%(patch)s'. Reason: %(reason)s") + + +class InvalidMAC(Invalid): + message = _("Expected a MAC address but received %(mac)s.") + + +class ManagedIPAddress(Invalid): + message = _("The infrastructure IP address for this nodetype is " + "specified by the system configuration and cannot be " + "modified.") + + +class IncorrectPrefix(Invalid): + message = _("A prefix length of %(length)s must be used for " + "addresses on the infrastructure network, as is specified in " + "the system configuration.") + + +class InterfaceNameAlreadyExists(Conflict): + message = _("Interface with name %(name)s already exists.") + + +class InterfaceNetworkTypeNotSet(Conflict): + message = _("The Interface must have a networktype configured to " + "support addresses. (data or infra)") + + +class AddressInUseByRouteGateway(Conflict): + message = _("Address %(address)s is in use by a route to " + "%(network)s/%(prefix)s via %(gateway)s") + + +class DuplicateAddressDetectionNotSupportedOnIpv4(Conflict): + message = _("Duplicate Address Detection (DAD) not supported on " + "IPv4 Addresses") + + +class DuplicateAddressDetectionRequiredOnIpv6(Conflict): + message = _("Duplicate Address Detection (DAD) required on " + "IPv6 Addresses") + + +class RouteAlreadyExists(Conflict): + message = _("Route %(network)s/%(prefix)s via %(gateway)s already " + "exists on this host.") + + +class RouteMaxPathsForSubnet(Conflict): + message = _("Maximum number of paths (%(count)s) already reached for " + "%(network)s/%(prefix)s already reached.") + + +class RouteGatewayNotReachable(Conflict): + message = _("Route gateway %(gateway)s is not reachable by any address " + " on this interface") + + +class RouteGatewayCannotBeLocal(Conflict): + message = _("Route gateway %(gateway)s cannot be another local interface") + + +class RoutesNotSupportedOnInterfaces(Conflict): + message = _("Routes may not be configured against interfaces with network " + "type '%(iftype)s'") + + +class DefaultRouteNotAllowedOnVRSInterface(Conflict): + message = _("Default route not permitted on 'data-vrs' interfaces") + + +class CannotDeterminePrimaryNetworkType(Conflict): + message = _("Cannot determine primary network type of interface " + "%(iface)s from %(types)s") + + +class AlarmAlreadyExists(Conflict): + message = _("An Alarm with UUID %(uuid)s already exists.") + + +class CPUAlreadyExists(Conflict): + message = _("A CPU with cpu ID %(cpu)s already exists.") + + +class MACAlreadyExists(Conflict): + message = _("A Port with MAC address %(mac)s already exists " + "on host %(host)s.") + + +class PCIAddrAlreadyExists(Conflict): + message = _("A Device with PCI address %(pciaddr)s " + "for %(host)s already exists.") + + +class DiskAlreadyExists(Conflict): + message = _("A Disk with UUID %(uuid)s already exists.") + + +class PortAlreadyExists(Conflict): + message = _("A Port with UUID %(uuid)s already exists.") + + +class SystemAlreadyExists(Conflict): + message = _("A System with UUID %(uuid)s already exists.") + + +class SensorAlreadyExists(Conflict): + message = _("A Sensor with UUID %(uuid)s already exists.") + + +class SensorGroupAlreadyExists(Conflict): + message = _("A SensorGroup with UUID %(uuid)s already exists.") + + +class TrapDestAlreadyExists(Conflict): + message = _("A TrapDest with UUID %(uuid)s already exists.") + + +class UserAlreadyExists(Conflict): + message = _("A User with UUID %(uuid)s already exists.") + + +class CommunityAlreadyExists(Conflict): + message = _("A Community with UUID %(uuid)s already exists.") + + +class ServiceAlreadyExists(Conflict): + message = _("A Service with UUID %(uuid)s already exists.") + + +class ServiceGroupAlreadyExists(Conflict): + message = _("A ServiceGroup with UUID %(uuid)s already exists.") + + +class NodeAlreadyExists(Conflict): + message = _("A Node with UUID %(uuid)s already exists.") + + +class MemoryAlreadyExists(Conflict): + message = _("A Memeory with UUID %(uuid)s already exists.") + + +class LLDPAgentExists(Conflict): + message = _("An LLDP agent with uuid %(uuid)s already exists.") + + +class LLDPNeighbourExists(Conflict): + message = _("An LLDP neighbour with uuid %(uuid)s already exists.") + + +class LLDPTlvExists(Conflict): + message = _("An LLDP TLV with type %(type) already exists.") + + +class LLDPDriverError(Conflict): + message = _("An LLDP driver error has occurred. method=%(method)") + + +class SystemConfigDriverError(Conflict): + message = _("A SystemConfig driver error has occurred. method=%(method)") + + +# Cannot be templated as the error syntax varies. +# msg needs to be constructed when raised. +class InvalidParameterValue(Invalid): + message = _("%(err)s") + + +class ApiError(Exception): + + message = _("An unknown exception occurred.") + + code = webob.exc.HTTPInternalServerError + + def __init__(self, message=None, **kwargs): + + self.kwargs = kwargs + + if 'code' not in self.kwargs and hasattr(self, 'code'): + self.kwargs['code'] = self.code + + if message: + self.message = message + + try: + super(ApiError, self).__init__(self.message % kwargs) + self.message = self.message % kwargs + except Exception: + LOG.exception('Exception in string format operation, ' + 'kwargs: %s', kwargs) + raise + + def __str__(self): + return repr(self.value) + + def __unicode__(self): + return self.message + + def format_message(self): + if self.__class__.__name__.endswith('_Remote'): + return self.args[0] + else: + return six.text_type(self) + + +class NotFound(InventoryException): + message = _("Resource could not be found.") + code = 404 + + +class MultipleResults(InventoryException): + message = _("More than one result found.") + + +class SystemNotFound(NotFound): + message = _("No System %(system)s found.") + + +class CPUNotFound(NotFound): + message = _("No CPU %(cpu)s found.") + + +class NTPNotFound(NotFound): + message = _("No NTP with id %(uuid)s found.") + + +class PTPNotFound(NotFound): + message = _("No PTP with id %(uuid)s found.") + + +class DiskNotFound(NotFound): + message = _("No disk with id %(disk_id)s") + + +class DiskPartitionNotFound(NotFound): + message = _("No disk partition with id %(partition_id)s") + + +class PartitionAlreadyExists(Conflict): + message = _("Disk partition %(device_path)s already exists.") + + +class LvmLvgNotFound(NotFound): + message = _("No LVM Local Volume Group with id %(lvg_id)s") + + +class LvmPvNotFound(NotFound): + message = _("No LVM Physical Volume with id %(pv_id)s") + + +class DriverNotFound(NotFound): + message = _("Failed to load driver %(driver_name)s.") + + +class PCIDeviceNotFound(NotFound): + message = _("Failed to load pci device %(pcidevice_id)s.") + + +class ImageNotFound(NotFound): + message = _("Image %(image_id)s could not be found.") + + +class HostNotFound(NotFound): + message = _("Host %(host)s could not be found.") + + +class HostAlreadyExists(Conflict): + message = _("Host %(uuid)s already exists.") + + +class ClonedInterfaceNotFound(NotFound): + message = _("Cloned Interface %(intf)s could not be found.") + + +class StaticAddressNotConfigured(Invalid): + message = _("The IP address for this interface is assigned " + "dynamically as specified during system configuration.") + + +class HostLocked(InventoryException): + message = _("Unable to complete the action %(action)s because " + "Host %(host)s is in administrative state = unlocked.") + + +class HostMustBeLocked(InventoryException): + message = _("Unable to complete the action because " + "Host %(host)s is in administrative state = unlocked.") + + +class ConsoleNotFound(NotFound): + message = _("Console %(console_id)s could not be found.") + + +class FileNotFound(NotFound): + message = _("File %(file_path)s could not be found.") + + +class NoValidHost(NotFound): + message = _("No valid host was found. %(reason)s") + + +class NodeNotFound(NotFound): + message = _("Node %(node)s could not be found.") + + +class MemoryNotFound(NotFound): + message = _("Memory %(memory)s could not be found.") + + +class PortNotFound(NotFound): + message = _("Port %(port)s could not be found.") + + +class SensorNotFound(NotFound): + message = _("Sensor %(sensor)s could not be found.") + + +class ServerNotFound(NotFound): + message = _("Server %(server)s could not be found.") + + +class ServiceNotFound(NotFound): + message = _("Service %(service)s could not be found.") + + +class AlarmNotFound(NotFound): + message = _("Alarm %(alarm)s could not be found.") + + +class EventLogNotFound(NotFound): + message = _("Event Log %(eventLog)s could not be found.") + + +class ExclusiveLockRequired(NotAuthorized): + message = _("An exclusive lock is required, " + "but the current context has a shared lock.") + + +class SSHConnectFailed(InventoryException): + message = _("Failed to establish SSH connection to host %(host)s.") + + +class UnsupportedObjectError(InventoryException): + message = _('Unsupported object type %(objtype)s') + + +class OrphanedObjectError(InventoryException): + message = _('Cannot call %(method)s on orphaned %(objtype)s object') + + +class IncompatibleObjectVersion(InventoryException): + message = _('Version %(objver)s of %(objname)s is not supported') + + +class GlanceConnectionFailed(InventoryException): + message = "Connection to glance host %(host)s:%(port)s failed: %(reason)s" + + +class ImageNotAuthorized(InventoryException): + message = "Not authorized for image %(image_id)s." + + +class LoadNotFound(NotFound): + message = _("Load %(load)s could not be found.") + + +class LldpAgentNotFound(NotFound): + message = _("LLDP agent %(agent)s could not be found") + + +class LldpAgentNotFoundForPort(NotFound): + message = _("LLDP agent for port %(port)s could not be found") + + +class LldpNeighbourNotFound(NotFound): + message = _("LLDP neighbour %(neighbour)s could not be found") + + +class LldpNeighbourNotFoundForMsap(NotFound): + message = _("LLDP neighbour could not be found for msap %(msap)s") + + +class LldpTlvNotFound(NotFound): + message = _("LLDP TLV %(type)s could not be found") + + +class InvalidImageRef(InventoryException): + message = "Invalid image href %(image_href)s." + code = 400 + + +class ServiceUnavailable(InventoryException): + message = "Connection failed" + + +class Forbidden(InventoryException): + message = "Requested OpenStack Images API is forbidden" + + +class BadRequest(InventoryException): + pass + + +class HTTPException(InventoryException): + message = "Requested version of OpenStack Images API is not available." + + +class InventorySignalTimeout(InventoryException): + message = "Inventory Timeout." + + +class InvalidEndpoint(InventoryException): + message = "The provided endpoint is invalid" + + +class CommunicationError(InventoryException): + message = "Unable to communicate with the server." + + +class HTTPForbidden(Forbidden): + pass + + +class Unauthorized(InventoryException): + pass + + +class HTTPNotFound(NotFound): + pass + + +class ConfigNotFound(InventoryException): + pass + + +class ConfigInvalid(InventoryException): + message = _("Invalid configuration file. %(error_msg)s") + + +class NotSupported(InventoryException): + message = "Action %(action)s is not supported." + + +class PeerAlreadyExists(Conflict): + message = _("Peer %(uuid)s already exists") + + +class PeerAlreadyContainsThisHost(Conflict): + message = _("Host %(host)s is already present in peer group %(peer_name)s") + + +class PeerNotFound(NotFound): + message = _("Peer %(peer_uuid)s not found") + + +class PeerContainsDuplicates(Conflict): + message = _("Peer with name % already exists") + + +class StoragePeerGroupUnexpected(InventoryException): + message = _("Host %(host)s cannot be assigned to group %(peer_name)s. " + "group-0 is reserved for storage-0 and storage-1") + + +class StorageBackendNotFoundByName(NotFound): + message = _("StorageBackend %(name)s not found") + + +class PickleableException(Exception): + """ + Pickleable Exception + Used to mark custom exception classes that can be pickled. + """ + pass + + +class OpenStackException(PickleableException): + """ + OpenStack Exception + """ + def __init__(self, message, reason): + """ + Create an OpenStack exception + """ + super(OpenStackException, self).__init__(message, reason) + self._reason = reason # a message string or another exception + self._message = message + + def __str__(self): + """ + Return a string representing the exception + """ + return "[OpenStack Exception:reason=%s]" % self._reason + + def __repr__(self): + """ + Provide a representation of the exception + """ + return str(self) + + def __reduce__(self): + """ + Return a tuple so that we can properly pickle the exception + """ + return OpenStackException, (self.message, self._reason) + + @property + def message(self): + """ + Returns the message for the exception + """ + return self._message + + @property + def reason(self): + """ + Returns the reason for the exception + """ + return self._reason + + +class OpenStackRestAPIException(PickleableException): + """ + OpenStack Rest-API Exception + """ + def __init__(self, message, http_status_code, reason): + """ + Create an OpenStack Rest-API exception + """ + super(OpenStackRestAPIException, self).__init__(message) + self._http_status_code = http_status_code # as defined in RFC 2616 + self._reason = reason # a message string or another exception + + def __str__(self): + """ + Return a string representing the exception + """ + return ("[OpenStack Rest-API Exception: code=%s, reason=%s]" + % (self._http_status_code, self._reason)) + + def __repr__(self): + """ + Provide a representation of the exception + """ + return str(self) + + def __reduce__(self): + """ + Return a tuple so that we can properly pickle the exception + """ + return OpenStackRestAPIException, (self.message, + self._http_status_code, + self._reason) + + @property + def http_status_code(self): + """ + Returns the HTTP status code + """ + return self._http_status_code + + @property + def reason(self): + """ + Returns the reason for the exception + """ + return self._reason diff --git a/inventory/inventory/inventory/common/fm.py b/inventory/inventory/inventory/common/fm.py new file mode 100644 index 00000000..901ad6b5 --- /dev/null +++ b/inventory/inventory/inventory/common/fm.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2016-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# FM Fault Management Handling + +from fm_api import constants as fm_constants +from fm_api import fm_api +import fmclient as fm_client +from keystoneauth1.access import service_catalog as k_service_catalog +from oslo_config import cfg +from oslo_log import log + +CONF = cfg.CONF + +LOG = log.getLogger(__name__) + + +fm_group = cfg.OptGroup( + 'fm', + title='FM Options', + help="Configuration options for the fault management service") + +fm_opts = [ + cfg.StrOpt('catalog_info', + default='faultmanagement:fm:internalURL', + help="Service catalog Look up info."), + cfg.StrOpt('os_region_name', + default='RegionOne', + help="Region name of this node. It is used for catalog lookup") +] + +CONF.register_group(fm_group) +CONF.register_opts(fm_opts, group=fm_group) + + +class FmCustomerLog(object): + """ + Fault Management Customer Log + """ + + _fm_api = None + + def __init__(self): + self._fm_api = fm_api.FaultAPIs() + + def customer_log(self, log_data): + LOG.info("Generating FM Customer Log %s" % log_data) + fm_event_id = log_data.get('event_id', None) + if fm_event_id is not None: + fm_event_state = fm_constants.FM_ALARM_STATE_MSG + entity_type = log_data.get('entity_type', None) + entity = log_data.get('entity', None) + fm_severity = log_data.get('fm_severity', None) + reason_text = log_data.get('reason_text', None) + fm_event_type = log_data.get('fm_event_type', None) + fm_probable_cause = fm_constants.ALARM_PROBABLE_CAUSE_UNKNOWN + fm_uuid = None + fault = fm_api.Fault(fm_event_id, + fm_event_state, + entity_type, + entity, + fm_severity, + reason_text, + fm_event_type, + fm_probable_cause, "", + False, True) + + response = self._fm_api.set_fault(fault) + if response is None: + LOG.error("Failed to generate customer log, fm_uuid=%s." % + fm_uuid) + else: + fm_uuid = response + LOG.info("Generated customer log, fm_uuid=%s." % fm_uuid) + else: + LOG.error("Unknown event id (%s) given." % fm_event_id) + + +def fmclient(context, version=1, endpoint=None): + """Constructs a fm client object for making API requests. + + :param context: The request context for auth. + :param version: API endpoint version. + :param endpoint: Optional If the endpoint is not available, it will be + retrieved from context + """ + auth_token = context.auth_token + if endpoint is None: + sc = k_service_catalog.ServiceCatalogV2(context.service_catalog) + service_type, service_name, interface = \ + CONF.fm.catalog_info.split(':') + service_parameters = {'service_type': service_type, + 'service_name': service_name, + 'interface': interface, + 'region_name': CONF.fm.os_region_name} + endpoint = sc.url_for(**service_parameters) + + return fm_client.Client(version=version, + endpoint=endpoint, + auth_token=auth_token) diff --git a/inventory/inventory/inventory/common/health.py b/inventory/inventory/inventory/common/health.py new file mode 100755 index 00000000..8ab49e64 --- /dev/null +++ b/inventory/inventory/inventory/common/health.py @@ -0,0 +1,289 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import os +import subprocess + +from controllerconfig import backup_restore +from fm_api import fm_api + +from inventory.common import ceph +from inventory.common import constants +from inventory.common.fm import fmclient +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import patch_api +from inventory.common import vim_api +from oslo_log import log + +import cgcs_patch.constants as patch_constants + +LOG = log.getLogger(__name__) + + +class Health(object): + + SUCCESS_MSG = _('OK') + FAIL_MSG = _('Fail') + + def __init__(self, context, dbapi): + self._context = context + self._dbapi = dbapi + self._ceph = ceph.CephApiOperator() + + def _check_hosts_provisioned(self, hosts): + """Checks that each host is provisioned""" + provisioned_hosts = [] + unprovisioned_hosts = 0 + for host in hosts: + if host['invprovision'] != k_host.PROVISIONED or \ + host['hostname'] is None: + unprovisioned_hosts = unprovisioned_hosts + 1 + else: + provisioned_hosts.append(host) + + return unprovisioned_hosts, provisioned_hosts + + def _check_hosts_enabled(self, hosts): + """Checks that each host is enabled and unlocked""" + offline_host_list = [] + for host in hosts: + if host['administrative'] != k_host.ADMIN_UNLOCKED or \ + host['operational'] != k_host.OPERATIONAL_ENABLED: + offline_host_list.append(host.hostname) + + success = not offline_host_list + return success, offline_host_list + + def _check_hosts_config(self, hosts): + """Checks that the applied and target config match for each host""" + config_host_list = [] + for host in hosts: + if (host.config_target and + host.config_applied != host.config_target): + config_host_list.append(host.hostname) + + success = not config_host_list + return success, config_host_list + + def _check_patch_current(self, hosts): + """Checks that each host is patch current""" + system = self._dbapi.isystem_get_one() + response = patch_api.patch_query_hosts(context=self._context, + region_name=system.region_name) + patch_hosts = response['data'] + not_patch_current_hosts = [] + hostnames = [] + for host in hosts: + hostnames.append(host['hostname']) + + for host in patch_hosts: + # There may be instances where the patching db returns + # hosts that have been recently deleted. We will continue if a host + # is the patching db but not inventory + try: + hostnames.remove(host['hostname']) + except ValueError: + LOG.info('Host %s found in patching but not in inventory. ' + 'Continuing' % host['hostname']) + else: + if not host['patch_current']: + not_patch_current_hosts.append(host['hostname']) + + success = not not_patch_current_hosts and not hostnames + return success, not_patch_current_hosts, hostnames + + def _check_alarms(self, context, force=False): + """Checks that no alarms are active""" + db_alarms = fmclient(context).alarm.list(include_suppress=True) + + success = True + allowed = 0 + affecting = 0 + # Only fail if we find alarms past their affecting threshold + for db_alarm in db_alarms: + if isinstance(db_alarm, tuple): + alarm = db_alarm[0] + mgmt_affecting = db_alarm[constants.DB_MGMT_AFFECTING] + else: + alarm = db_alarm + mgmt_affecting = db_alarm.mgmt_affecting + if fm_api.FaultAPIs.alarm_allowed(alarm.severity, mgmt_affecting): + allowed += 1 + if not force: + success = False + else: + affecting += 1 + success = False + + return success, allowed, affecting + + def get_alarms_degrade(self, context, + alarm_ignore_list=[], entity_instance_id_filter=""): + """Return all the alarms that cause the degrade""" + db_alarms = fmclient(context).alarm.list(include_suppress=True) + degrade_alarms = [] + + for db_alarm in db_alarms: + if isinstance(db_alarm, tuple): + alarm = db_alarm[0] + degrade_affecting = db_alarm[constants.DB_DEGRADE_AFFECTING] + else: + alarm = db_alarm + degrade_affecting = db_alarm.degrade_affecting + # Ignore alarms that are part of the ignore list sent as parameter + # and also filter the alarms bases on entity instance id. + # If multiple alarms with the same ID exist, we only return the ID + # one time. + if not fm_api.FaultAPIs.alarm_allowed( + alarm.severity, degrade_affecting): + if (entity_instance_id_filter in alarm.entity_instance_id and + alarm.alarm_id not in alarm_ignore_list and + alarm.alarm_id not in degrade_alarms): + degrade_alarms.append(alarm.alarm_id) + return degrade_alarms + + def _check_ceph(self): + """Checks the ceph health status""" + return self._ceph.ceph_status_ok() + + def _check_license(self, version): + """Validates the current license is valid for the specified version""" + check_binary = "/usr/bin/sm-license-check" + license_file = '/etc/platform/.license' + system = self._dbapi.isystem_get_one() + system_type = system.system_type + system_mode = system.system_mode + + with open(os.devnull, "w") as fnull: + try: + subprocess.check_call([check_binary, license_file, version, + system_type, system_mode], + stdout=fnull, stderr=fnull) + except subprocess.CalledProcessError: + return False + + return True + + def _check_required_patches(self, patch_list): + """Validates that each patch provided is applied on the system""" + system = self._dbapi.isystem_get_one() + response = patch_api.patch_query(context=self._context, + region_name=system.region_name, + timeout=60) + query_patches = response['pd'] + applied_patches = [] + for patch_key in query_patches: + patch = query_patches[patch_key] + patchstate = patch.get('patchstate', None) + if patchstate == patch_constants.APPLIED or \ + patchstate == patch_constants.COMMITTED: + applied_patches.append(patch_key) + + missing_patches = [] + for required_patch in patch_list: + if required_patch not in applied_patches: + missing_patches.append(required_patch) + + success = not missing_patches + return success, missing_patches + + def _check_running_instances(self, host): + """Checks that no instances are running on the host""" + + vim_resp = vim_api.vim_host_get_instances( + self._context, + host['uuid'], + host['hostname']) + running_instances = vim_resp['instances'] + + success = running_instances == 0 + return success, running_instances + + def _check_simplex_available_space(self): + """Ensures there is free space for the backup""" + try: + backup_restore.check_size("/opt/backups", True) + except backup_restore.BackupFail: + return False + + return True + + def get_system_health(self, context, force=False): + """Returns the general health of the system""" + # Checks the following: + # All hosts are provisioned + # All hosts are patch current + # All hosts are unlocked/enabled + # All hosts having matching configs + # No management affecting alarms + # For ceph systems: The storage cluster is healthy + + hosts = self._dbapi.ihost_get_list() + output = _('System Health:\n') + health_ok = True + + unprovisioned_hosts, provisioned_hosts = \ + self._check_hosts_provisioned(hosts) + success = unprovisioned_hosts == 0 + output += (_('All hosts are provisioned: [%s]\n') + % (Health.SUCCESS_MSG if success else Health.FAIL_MSG)) + if not success: + output += _('%s Unprovisioned hosts\n') % unprovisioned_hosts + # Set the hosts to the provisioned_hosts. This will allow the other + # checks to continue + hosts = provisioned_hosts + + health_ok = health_ok and success + + success, error_hosts = self._check_hosts_enabled(hosts) + output += _('All hosts are unlocked/enabled: [%s]\n') \ + % (Health.SUCCESS_MSG if success else Health.FAIL_MSG) + if not success: + output += _('Locked or disabled hosts: %s\n') \ + % ', '.join(error_hosts) + + health_ok = health_ok and success + + success, error_hosts = self._check_hosts_config(hosts) + output += _('All hosts have current configurations: [%s]\n') \ + % (Health.SUCCESS_MSG if success else Health.FAIL_MSG) + if not success: + output += _('Hosts with out of date configurations: %s\n') \ + % ', '.join(error_hosts) + + health_ok = health_ok and success + + success, error_hosts, missing_hosts = self._check_patch_current(hosts) + output += _('All hosts are patch current: [%s]\n') \ + % (Health.SUCCESS_MSG if success else Health.FAIL_MSG) + if not success: + if error_hosts: + output += _('Hosts not patch current: %s\n') \ + % ', '.join(error_hosts) + if missing_hosts: + output += _('Hosts without patch data: %s\n') \ + % ', '.join(missing_hosts) + + health_ok = health_ok and success + + # if StorageBackendConfig.has_backend( + # self._dbapi, + # constants.CINDER_BACKEND_CEPH): + # success = self._check_ceph() + # output += _('Ceph Storage Healthy: [%s]\n') \ + # % (Health.SUCCESS_MSG if success else Health.FAIL_MSG) + # health_ok = health_ok and success + + success, allowed, affecting = self._check_alarms(context, force) + output += _('No alarms: [%s]\n') \ + % (Health.SUCCESS_MSG if success else Health.FAIL_MSG) + if not success: + output += _('[{}] alarms found, [{}] of which are management ' + 'affecting\n').format(allowed + affecting, affecting) + + health_ok = health_ok and success + + return health_ok, output diff --git a/inventory/inventory/inventory/common/hwmon_api.py b/inventory/inventory/inventory/common/hwmon_api.py new file mode 100755 index 00000000..a531063d --- /dev/null +++ b/inventory/inventory/inventory/common/hwmon_api.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import json +from oslo_log import log +from rest_api import rest_api_request +LOG = log.getLogger(__name__) + + +def sensorgroup_add(token, address, port, isensorgroup_hwmon, timeout): + """ + Sends a SensorGroup Add command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensorgroups/" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = isensorgroup_hwmon + + LOG.info("sensorgroup_add for %s cmd=%s hdr=%s payload=%s" % + (isensorgroup_hwmon['sensorgroupname'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "POST", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def sensorgroup_modify(token, address, port, isensorgroup_hwmon, timeout): + """ + Sends a SensorGroup Modify command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensorgroups/%s" % isensorgroup_hwmon['uuid'] + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = isensorgroup_hwmon + + LOG.info("sensorgroup_modify for %s cmd=%s hdr=%s payload=%s" % + (isensorgroup_hwmon['sensorgroupname'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "PATCH", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + LOG.debug("sensorgroup modify response=%s" % response) + + return response + + +def sensorgroup_delete(token, address, port, isensorgroup_hwmon, timeout): + """ + Sends a SensorGroup Delete command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensorgroups/%s" % isensorgroup_hwmon['uuid'] + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = None + + LOG.info("sensorgroup_delete for %s cmd=%s hdr=%s payload=%s" % + (isensorgroup_hwmon['uuid'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "DELETE", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def sensorgroup_relearn(token, address, port, payload, timeout): + """ + Sends a SensorGroup Relearn command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensorgroups/relearn" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = payload + + LOG.info("sensorgroup_relearn for %s cmd=%s hdr=%s payload=%s" % + (payload['host_uuid'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "POST", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def sensor_add(token, address, port, isensor_hwmon, timeout): + """ + Sends a Sensor Add command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensors/" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = isensor_hwmon + + LOG.info("sensor_add for %s cmd=%s hdr=%s payload=%s" % + (isensor_hwmon['sensorname'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "POST", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def sensor_modify(token, address, port, isensor_hwmon, timeout): + """ + Sends a Sensor Modify command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensors/%s" % isensor_hwmon['uuid'] + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = isensor_hwmon + + LOG.info("sensor_modify for %s cmd=%s hdr=%s payload=%s" % + (isensor_hwmon['sensorname'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "PATCH", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def sensor_delete(token, address, port, isensor_hwmon, timeout): + """ + Sends a Sensor Delete command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/isensors/%s" % isensor_hwmon['uuid'] + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = None + + LOG.info("sensor_delete for %s cmd=%s hdr=%s payload=%s" % + (isensor_hwmon['uuid'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "DELETE", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response diff --git a/inventory/inventory/inventory/common/i18n.py b/inventory/inventory/inventory/common/i18n.py new file mode 100644 index 00000000..7f813263 --- /dev/null +++ b/inventory/inventory/inventory/common/i18n.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import oslo_i18n + +_translators = oslo_i18n.TranslatorFactory(domain='inventory') + +# The primary translation function using the well-known name "_" +_ = _translators.primary diff --git a/inventory/inventory/inventory/common/k_host.py b/inventory/inventory/inventory/common/k_host.py new file mode 100644 index 00000000..fc78e12d --- /dev/null +++ b/inventory/inventory/inventory/common/k_host.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Inventory Host Management Constants + +# Administrative States +ADMIN_UNLOCKED = 'unlocked' +ADMIN_LOCKED = 'locked' + +# Operational States +OPERATIONAL_ENABLED = 'enabled' +OPERATIONAL_DISABLED = 'disabled' + +# Availability Status +AVAILABILITY_AVAILABLE = 'available' +AVAILABILITY_OFFLINE = 'offline' +AVAILABILITY_ONLINE = 'online' +AVAILABILITY_DEGRADED = 'degraded' + +# Host Actions: +ACTION_UNLOCK = 'unlock' +ACTION_FORCE_UNLOCK = 'force-unlock' +ACTION_LOCK = 'lock' +ACTION_FORCE_LOCK = 'force-lock' +ACTION_REBOOT = 'reboot' +ACTION_RESET = 'reset' +ACTION_REINSTALL = 'reinstall' +ACTION_POWERON = 'power-on' +ACTION_POWEROFF = 'power-off' +ACTION_SWACT = 'swact' +ACTION_FORCE_SWACT = 'force-swact' +ACTION_SUBFUNCTION_CONFIG = 'subfunction_config' +ACTION_DELETE = 'delete' +ACTION_NONE = 'none' + + +ACTIONS_VIM = [ACTION_LOCK, + ACTION_FORCE_LOCK] + +# VIM services +VIM_SERVICES_ENABLED = 'services-enabled' +VIM_SERVICES_DISABLED = 'services-disabled' +VIM_SERVICES_DISABLE_EXTEND = 'services-disable-extend' +VIM_SERVICES_DISABLE_FAILED = 'services-disable-failed' +VIM_SERVICES_DELETE_FAILED = 'services-delete-failed' + +ACTIONS_MTCE = [ + ACTION_REBOOT, + ACTION_REINSTALL, + ACTION_RESET, + ACTION_POWERON, + ACTION_POWEROFF, + ACTION_SWACT, + ACTION_UNLOCK, + VIM_SERVICES_DISABLED, + VIM_SERVICES_DISABLE_FAILED, + ACTION_FORCE_SWACT] + +ACTIONS_CONFIG = [ACTION_SUBFUNCTION_CONFIG] + +# Personalities +CONTROLLER = 'controller' +STORAGE = 'storage' +COMPUTE = 'compute' + +PERSONALITIES = [CONTROLLER, STORAGE, COMPUTE] + +# Host names +LOCALHOST_HOSTNAME = 'localhost' + +CONTROLLER_HOSTNAME = 'controller' +CONTROLLER_0_HOSTNAME = '%s-0' % CONTROLLER_HOSTNAME +CONTROLLER_1_HOSTNAME = '%s-1' % CONTROLLER_HOSTNAME + +STORAGE_HOSTNAME = 'storage' +STORAGE_0_HOSTNAME = '%s-0' % STORAGE_HOSTNAME +STORAGE_1_HOSTNAME = '%s-1' % STORAGE_HOSTNAME +STORAGE_2_HOSTNAME = '%s-2' % STORAGE_HOSTNAME +# Other Storage Hostnames are built dynamically. + +# SUBFUNCTION FEATURES +SUBFUNCTIONS = 'subfunctions' +LOWLATENCY = 'lowlatency' + +LOCKING = 'Locking' +FORCE_LOCKING = "Force Locking" + +# invprovision status +PROVISIONED = 'provisioned' +PROVISIONING = 'provisioning' +UNPROVISIONED = 'unprovisioned' + +# Board Management Controller +BM_EXTERNAL = "External" +BM_TYPE_GENERIC = 'bmc' +BM_TYPE_NONE = 'none' + +HOST_STOR_FUNCTION = 'stor_function' + +# ihost config_status field values +CONFIG_STATUS_REINSTALL = "Reinstall required" + +# when reinstall starts, mtc updates the db with task = 'Reinstalling' +TASK_REINSTALLING = "Reinstalling" +HOST_ACTION_STATE = "action_state" +HAS_REINSTALLING = "reinstalling" +HAS_REINSTALLED = "reinstalled" diff --git a/inventory/inventory/inventory/common/k_host_agg.py b/inventory/inventory/inventory/common/k_host_agg.py new file mode 100644 index 00000000..cc019ebf --- /dev/null +++ b/inventory/inventory/inventory/common/k_host_agg.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Host Aggregate Constants + +# Storage: Host Aggregates Groups +HOST_AGG_NAME_REMOTE = 'remote_storage_hosts' +HOST_AGG_META_REMOTE = 'remote' +HOST_AGG_NAME_LOCAL_LVM = 'local_storage_lvm_hosts' +HOST_AGG_META_LOCAL_LVM = 'local_lvm' +HOST_AGG_NAME_LOCAL_IMAGE = 'local_storage_image_hosts' +HOST_AGG_META_LOCAL_IMAGE = 'local_image' diff --git a/inventory/inventory/inventory/common/k_lldp.py b/inventory/inventory/inventory/common/k_lldp.py new file mode 100644 index 00000000..d1c5e87c --- /dev/null +++ b/inventory/inventory/inventory/common/k_lldp.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# LLDP constants + +LLDP_TLV_TYPE_CHASSIS_ID = 'chassis_id' +LLDP_TLV_TYPE_PORT_ID = 'port_identifier' +LLDP_TLV_TYPE_TTL = 'ttl' +LLDP_TLV_TYPE_SYSTEM_NAME = 'system_name' +LLDP_TLV_TYPE_SYSTEM_DESC = 'system_description' +LLDP_TLV_TYPE_SYSTEM_CAP = 'system_capabilities' +LLDP_TLV_TYPE_MGMT_ADDR = 'management_address' +LLDP_TLV_TYPE_PORT_DESC = 'port_description' +LLDP_TLV_TYPE_DOT1_LAG = 'dot1_lag' +LLDP_TLV_TYPE_DOT1_PORT_VID = 'dot1_port_vid' +LLDP_TLV_TYPE_DOT1_MGMT_VID = 'dot1_management_vid' +LLDP_TLV_TYPE_DOT1_PROTO_VIDS = 'dot1_proto_vids' +LLDP_TLV_TYPE_DOT1_PROTO_IDS = 'dot1_proto_ids' +LLDP_TLV_TYPE_DOT1_VLAN_NAMES = 'dot1_vlan_names' +LLDP_TLV_TYPE_DOT1_VID_DIGEST = 'dot1_vid_digest' +LLDP_TLV_TYPE_DOT3_MAC_STATUS = 'dot3_mac_status' +LLDP_TLV_TYPE_DOT3_MAX_FRAME = 'dot3_max_frame' +LLDP_TLV_TYPE_DOT3_POWER_MDI = 'dot3_power_mdi' +LLDP_TLV_VALID_LIST = [LLDP_TLV_TYPE_CHASSIS_ID, LLDP_TLV_TYPE_PORT_ID, + LLDP_TLV_TYPE_TTL, LLDP_TLV_TYPE_SYSTEM_NAME, + LLDP_TLV_TYPE_SYSTEM_DESC, LLDP_TLV_TYPE_SYSTEM_CAP, + LLDP_TLV_TYPE_MGMT_ADDR, LLDP_TLV_TYPE_PORT_DESC, + LLDP_TLV_TYPE_DOT1_LAG, LLDP_TLV_TYPE_DOT1_PORT_VID, + LLDP_TLV_TYPE_DOT1_VID_DIGEST, + LLDP_TLV_TYPE_DOT1_MGMT_VID, + LLDP_TLV_TYPE_DOT1_PROTO_VIDS, + LLDP_TLV_TYPE_DOT1_PROTO_IDS, + LLDP_TLV_TYPE_DOT1_VLAN_NAMES, + LLDP_TLV_TYPE_DOT1_VID_DIGEST, + LLDP_TLV_TYPE_DOT3_MAC_STATUS, + LLDP_TLV_TYPE_DOT3_MAX_FRAME, + LLDP_TLV_TYPE_DOT3_POWER_MDI] + +LLDP_AGENT_STATE_REMOVED = 'removed' +LLDP_NEIGHBOUR_STATE_REMOVED = LLDP_AGENT_STATE_REMOVED +# LLDP_FULL_AUDIT_COUNT based on frequency of host_lldp_get_and_report() +LLDP_FULL_AUDIT_COUNT = 6 diff --git a/inventory/inventory/inventory/common/k_pci.py b/inventory/inventory/inventory/common/k_pci.py new file mode 100644 index 00000000..a23aec26 --- /dev/null +++ b/inventory/inventory/inventory/common/k_pci.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# PCI device constants + +# PCI Alias types and names +NOVA_PCI_ALIAS_GPU_NAME = "gpu" +NOVA_PCI_ALIAS_GPU_CLASS = "030000" +NOVA_PCI_ALIAS_GPU_PF_NAME = "gpu-pf" +NOVA_PCI_ALIAS_GPU_VF_NAME = "gpu-vf" +NOVA_PCI_ALIAS_QAT_CLASS = "0x0b4000" +NOVA_PCI_ALIAS_QAT_DH895XCC_PF_NAME = "qat-dh895xcc-pf" +NOVA_PCI_ALIAS_QAT_C62X_PF_NAME = "qat-c62x-pf" +NOVA_PCI_ALIAS_QAT_PF_VENDOR = "8086" +NOVA_PCI_ALIAS_QAT_DH895XCC_PF_DEVICE = "0435" +NOVA_PCI_ALIAS_QAT_C62X_PF_DEVICE = "37c8" +NOVA_PCI_ALIAS_QAT_DH895XCC_VF_NAME = "qat-dh895xcc-vf" +NOVA_PCI_ALIAS_QAT_C62X_VF_NAME = "qat-c62x-vf" +NOVA_PCI_ALIAS_QAT_VF_VENDOR = "8086" +NOVA_PCI_ALIAS_QAT_DH895XCC_VF_DEVICE = "0443" +NOVA_PCI_ALIAS_QAT_C62X_VF_DEVICE = "37c9" +NOVA_PCI_ALIAS_USER_NAME = "user" diff --git a/inventory/inventory/inventory/common/keystone.py b/inventory/inventory/inventory/common/keystone.py new file mode 100644 index 00000000..459d539a --- /dev/null +++ b/inventory/inventory/inventory/common/keystone.py @@ -0,0 +1,121 @@ +# coding=utf-8 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Central place for handling Keystone authorization and service lookup.""" + +from keystoneauth1 import exceptions as kaexception +from keystoneauth1 import loading as kaloading +from keystoneauth1 import service_token +from keystoneauth1 import token_endpoint +from oslo_config import cfg +from oslo_log import log +import six + +from inventory.common import exception + +CONF = cfg.CONF + + +LOG = log.getLogger(__name__) + + +def ks_exceptions(f): + """Wraps keystoneclient functions and centralizes exception handling.""" + @six.wraps(f) + def wrapper(*args, **kwargs): + try: + return f(*args, **kwargs) + except kaexception.EndpointNotFound: + service_type = kwargs.get('service_type', 'inventory') + endpoint_type = kwargs.get('endpoint_type', 'internal') + raise exception.CatalogNotFound( + service_type=service_type, endpoint_type=endpoint_type) + except (kaexception.Unauthorized, kaexception.AuthorizationFailure): + raise exception.KeystoneUnauthorized() + except (kaexception.NoMatchingPlugin, + kaexception.MissingRequiredOptions) as e: + raise exception.ConfigInvalid(six.text_type(e)) + except Exception as e: + LOG.exception('Keystone request failed: %(msg)s', + {'msg': six.text_type(e)}) + raise exception.KeystoneFailure(six.text_type(e)) + return wrapper + + +@ks_exceptions +def get_session(group, **session_kwargs): + """Loads session object from options in a configuration file section. + + The session_kwargs will be passed directly to keystoneauth1 Session + and will override the values loaded from config. + Consult keystoneauth1 docs for available options. + + :param group: name of the config section to load session options from + + """ + return kaloading.load_session_from_conf_options( + CONF, group, **session_kwargs) + + +@ks_exceptions +def get_auth(group, **auth_kwargs): + """Loads auth plugin from options in a configuration file section. + + The auth_kwargs will be passed directly to keystoneauth1 auth plugin + and will override the values loaded from config. + Note that the accepted kwargs will depend on auth plugin type as defined + by [group]auth_type option. + Consult keystoneauth1 docs for available auth plugins and their options. + + :param group: name of the config section to load auth plugin options from + + """ + try: + auth = kaloading.load_auth_from_conf_options(CONF, group, + **auth_kwargs) + except kaexception.MissingRequiredOptions: + LOG.error('Failed to load auth plugin from group %s', group) + raise + return auth + + +@ks_exceptions +def get_adapter(group, **adapter_kwargs): + """Loads adapter from options in a configuration file section. + + The adapter_kwargs will be passed directly to keystoneauth1 Adapter + and will override the values loaded from config. + Consult keystoneauth1 docs for available adapter options. + + :param group: name of the config section to load adapter options from + + """ + return kaloading.load_adapter_from_conf_options(CONF, group, + **adapter_kwargs) + + +def get_service_auth(context, endpoint, service_auth): + """Create auth plugin wrapping both user and service auth. + + When properly configured and using auth_token middleware, + requests with valid service auth will not fail + if the user token is expired. + + Ideally we would use the plugin provided by auth_token middleware + however this plugin isn't serialized yet. + """ + # TODO(pas-ha) use auth plugin from context when it is available + user_auth = token_endpoint.Token(endpoint, context.auth_token) + return service_token.ServiceTokenAuthWrapper(user_auth=user_auth, + service_auth=service_auth) diff --git a/inventory/inventory/inventory/common/mtce_api.py b/inventory/inventory/inventory/common/mtce_api.py new file mode 100644 index 00000000..9441f20d --- /dev/null +++ b/inventory/inventory/inventory/common/mtce_api.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2015-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from inventory.common import exception as si_exception +import json +from oslo_log import log +from rest_api import rest_api_request +import time + +LOG = log.getLogger(__name__) + + +def host_add(token, address, port, ihost_mtce, timeout): + """ + Sends a Host Add command to maintenance. + """ + + # api_cmd = "http://localhost:2112" + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/hosts/" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = ihost_mtce + + LOG.info("host_add for %s cmd=%s hdr=%s payload=%s" % + (ihost_mtce['hostname'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "POST", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def host_modify(token, address, port, ihost_mtce, timeout, max_retries=1): + """ + Sends a Host Modify command to maintenance. + """ + + # api_cmd = "http://localhost:2112" + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/hosts/%s" % ihost_mtce['uuid'] + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = dict() + api_cmd_payload = ihost_mtce + + LOG.debug("host_modify for %s cmd=%s hdr=%s payload=%s" % + (ihost_mtce['hostname'], + api_cmd, api_cmd_headers, api_cmd_payload)) + + num_of_try = 0 + response = None + while num_of_try < max_retries and response is None: + try: + num_of_try = num_of_try + 1 + LOG.info("number of calls to rest_api_request=%d (max_retry=%d)" % + (num_of_try, max_retries)) + response = rest_api_request( + token, "PATCH", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + if response is None: + time.sleep(3) + except si_exception.SysInvSignalTimeout as e: + LOG.warn("WARNING rest_api_request Timeout Error e=%s" % (e)) + raise si_exception.SysInvSignalTimeout + except si_exception.InventoryException as e: + LOG.warn("WARNING rest_api_request Unexpected Error e=%s" % (e)) + + return response + + +def host_delete(token, address, port, ihost_mtce, timeout): + """ + Sends a Host Delete command to maintenance. + """ + + api_cmd = "http://%s:%s" % (address, port) + api_cmd += "/v1/hosts/%s" % ihost_mtce['uuid'] + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "sysinv/1.0" + + api_cmd_payload = None + + LOG.info("host_delete for %s cmd=%s hdr=%s payload=%s" % + (ihost_mtce['uuid'], api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(token, "DELETE", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response diff --git a/inventory/inventory/inventory/common/patch_api.py b/inventory/inventory/inventory/common/patch_api.py new file mode 100644 index 00000000..b7677225 --- /dev/null +++ b/inventory/inventory/inventory/common/patch_api.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventory.common import constants +from keystoneauth1.access import service_catalog as k_service_catalog +from oslo_log import log +from rest_api import rest_api_request + +LOG = log.getLogger(__name__) + + +def _get_endpoint(context, region_name): + # service_type, service_name, interface = \ + # CONF.patching.catalog_info.split(':') + sc = k_service_catalog.ServiceCatalogV2(context.service_catalog) + service_parameters = {'service_type': 'patching', + 'service_name': 'patching', + 'interface': 'internalURL', + 'region_name': region_name} + endpoint = sc.url_for(**service_parameters) + return endpoint + + +def patch_query(context, region_name, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS): + """ + Request the list of patches known to the patch service + """ + + api_cmd = _get_endpoint(context, region_name) + api_cmd += "/v1/query/" + + return rest_api_request(context, "GET", api_cmd, timeout=timeout) + + +def patch_query_hosts(context, region_name, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS): + """ + Request the patch state for all hosts known to the patch service + """ + + api_cmd = _get_endpoint(context, region_name) + api_cmd += "/v1/query_hosts/" + + return rest_api_request(context, "GET", api_cmd, timeout=timeout) + + +def patch_drop_host(context, hostname, region_name, + timeout=constants.PATCH_DEFAULT_TIMEOUT_IN_SECS): + """ + Notify the patch service to drop the specified host + """ + + api_cmd = _get_endpoint(context, region_name) + api_cmd += "/v1/drop_host/%s" % hostname + + return rest_api_request(context, "POST", api_cmd, timeout=timeout) diff --git a/inventory/inventory/inventory/common/policy.py b/inventory/inventory/inventory/common/policy.py new file mode 100644 index 00000000..e2a67589 --- /dev/null +++ b/inventory/inventory/inventory/common/policy.py @@ -0,0 +1,96 @@ +# Copyright (c) 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +"""Policy Engine For Inventory.""" + +from oslo_concurrency import lockutils +from oslo_config import cfg +from oslo_log import log +from oslo_policy import policy + + +base_rules = [ + policy.RuleDefault('admin_required', 'role:admin or is_admin:1', + description='Who is considered an admin'), + policy.RuleDefault('admin_api', 'is_admin_required:True', + description='admin API requirement'), + policy.RuleDefault('default', 'rule:admin_api', + description='default rule'), +] + +_ENFORCER = None +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +# we can get a policy enforcer by this init. +# oslo policy support change policy rule dynamically. +# at present, policy.enforce will reload the policy rules when it checks +# the policy files have been touched. +@lockutils.synchronized('policy_enforcer') +def init_enforcer(policy_file=None, rules=None, + default_rule=None, use_conf=True, overwrite=True): + """Init an Enforcer class. + + :param policy_file: Custom policy file to use, if none is + specified, ``conf.policy_file`` will be + used. + :param rules: Default dictionary / Rules to use. It will be + considered just in the first instantiation. If + :meth:`load_rules` with ``force_reload=True``, + :meth:`clear` or :meth:`set_rules` with + ``overwrite=True`` is called this will be overwritten. + :param default_rule: Default rule to use, conf.default_rule will + be used if none is specified. + :param use_conf: Whether to load rules from cache or config file. + :param overwrite: Whether to overwrite existing rules when reload rules + from config file. + """ + global _ENFORCER + if not _ENFORCER: + # http://docs.openstack.org/developer/oslo.policy/usage.html + _ENFORCER = policy.Enforcer(CONF, + policy_file=policy_file, + rules=rules, + default_rule=default_rule, + use_conf=use_conf, + overwrite=overwrite) + _ENFORCER.register_defaults(base_rules) + return _ENFORCER + + +def get_enforcer(): + """Provides access to the single instance of Policy enforcer.""" + + if not _ENFORCER: + init_enforcer() + + return _ENFORCER + + +def check_is_admin(context): + """Whether or not role contains 'admin' role according to policy setting. + + """ + init_enforcer() + + target = {} + credentials = context.to_dict() + + return _ENFORCER.enforce('context_is_admin', target, credentials) diff --git a/inventory/inventory/inventory/common/rest_api.py b/inventory/inventory/inventory/common/rest_api.py new file mode 100644 index 00000000..c33edb17 --- /dev/null +++ b/inventory/inventory/inventory/common/rest_api.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2015-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +import json +import signal +import urllib2 + +from inventory.common.exception import OpenStackException +from inventory.common.exception import OpenStackRestAPIException + +from oslo_log import log +LOG = log.getLogger(__name__) + + +def rest_api_request(context, method, api_cmd, api_cmd_headers=None, + api_cmd_payload=None, timeout=10): + """ + Make a rest-api request + Returns: response as a dictionary + """ + + LOG.info("%s cmd:%s hdr:%s payload:%s" % (method, + api_cmd, api_cmd_headers, api_cmd_payload)) + + if hasattr(context, 'auth_token'): + token = context.auth_token + else: + token = None + + response = None + try: + request_info = urllib2.Request(api_cmd) + request_info.get_method = lambda: method + if token: + request_info.add_header("X-Auth-Token", token) + request_info.add_header("Accept", "application/json") + + if api_cmd_headers is not None: + for header_type, header_value in api_cmd_headers.items(): + request_info.add_header(header_type, header_value) + + if api_cmd_payload is not None: + request_info.add_data(api_cmd_payload) + + request = urllib2.urlopen(request_info, timeout=timeout) + response = request.read() + + if response == "": + response = json.loads("{}") + else: + response = json.loads(response) + request.close() + + LOG.info("Response=%s" % response) + + except urllib2.HTTPError as e: + LOG.warn("HTTP Error e.code=%s e=%s" % (e.code, e)) + if hasattr(e, 'msg') and e.msg: + response = json.loads(e.msg) + else: + response = json.loads("{}") + + LOG.info("HTTPError response=%s" % (response)) + raise OpenStackRestAPIException(e.message, e.code, "%s" % e) + except urllib2.URLError as e: + LOG.warn("URLError Error e=%s" % (e)) + raise OpenStackException(e.message, "%s" % e) + + finally: + signal.alarm(0) + return response diff --git a/inventory/inventory/inventory/common/rpc.py b/inventory/inventory/inventory/common/rpc.py new file mode 100644 index 00000000..5c426bfb --- /dev/null +++ b/inventory/inventory/inventory/common/rpc.py @@ -0,0 +1,153 @@ +# Copyright 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +import oslo_messaging as messaging +from oslo_messaging.rpc import dispatcher +from osprofiler import profiler + +from inventory.common import context as inventory_context +from inventory.common import exception + + +CONF = cfg.CONF + +TRANSPORT = None +NOTIFICATION_TRANSPORT = None +SENSORS_NOTIFIER = None +VERSIONED_NOTIFIER = None + +ALLOWED_EXMODS = [ + exception.__name__, +] +EXTRA_EXMODS = [] + + +def init(conf): + global TRANSPORT, NOTIFICATION_TRANSPORT + global SENSORS_NOTIFIER, VERSIONED_NOTIFIER + exmods = get_allowed_exmods() + TRANSPORT = messaging.get_rpc_transport(conf, + allowed_remote_exmods=exmods) + + NOTIFICATION_TRANSPORT = messaging.get_notification_transport( + conf, + allowed_remote_exmods=exmods) + + serializer = RequestContextSerializer(messaging.JsonPayloadSerializer()) + SENSORS_NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT, + serializer=serializer) + if conf.notification_level is None: + VERSIONED_NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT, + serializer=serializer, + driver='noop') + else: + VERSIONED_NOTIFIER = messaging.Notifier(NOTIFICATION_TRANSPORT, + serializer=serializer, + topics=['inventory_versioned_' + 'notifications']) + + +def cleanup(): + global TRANSPORT, NOTIFICATION_TRANSPORT + global SENSORS_NOTIFIER, VERSIONED_NOTIFIER + assert TRANSPORT is not None + assert NOTIFICATION_TRANSPORT is not None + assert SENSORS_NOTIFIER is not None + assert VERSIONED_NOTIFIER is not None + TRANSPORT.cleanup() + NOTIFICATION_TRANSPORT.cleanup() + TRANSPORT = NOTIFICATION_TRANSPORT = None + SENSORS_NOTIFIER = VERSIONED_NOTIFIER = None + + +def set_defaults(control_exchange): + messaging.set_transport_defaults(control_exchange) + + +def get_allowed_exmods(): + return ALLOWED_EXMODS + EXTRA_EXMODS + + +class RequestContextSerializer(messaging.Serializer): + + def __init__(self, base): + self._base = base + + def serialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.serialize_entity(context, entity) + + def deserialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.deserialize_entity(context, entity) + + def serialize_context(self, context): + _context = context.to_dict() + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + return _context + + def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info: + profiler.init(**trace_info) + return inventory_context.RequestContext.from_dict(context) + + +def get_transport_url(url_str=None): + return messaging.TransportURL.parse(CONF, url_str) + + +def get_client(target, version_cap=None, serializer=None): + assert TRANSPORT is not None + serializer = RequestContextSerializer(serializer) + return messaging.RPCClient(TRANSPORT, + target, + version_cap=version_cap, + serializer=serializer) + + +def get_server(target, endpoints, serializer=None): + assert TRANSPORT is not None + serializer = RequestContextSerializer(serializer) + access_policy = dispatcher.DefaultRPCAccessPolicy + return messaging.get_rpc_server(TRANSPORT, + target, + endpoints, + executor='eventlet', + serializer=serializer, + access_policy=access_policy) + + +def get_sensors_notifier(service=None, host=None, publisher_id=None): + assert SENSORS_NOTIFIER is not None + if not publisher_id: + publisher_id = "%s.%s" % (service, host or CONF.host) + return SENSORS_NOTIFIER.prepare(publisher_id=publisher_id) + + +def get_versioned_notifier(publisher_id=None): + assert VERSIONED_NOTIFIER is not None + assert publisher_id is not None + return VERSIONED_NOTIFIER.prepare(publisher_id=publisher_id) diff --git a/inventory/inventory/inventory/common/rpc_service.py b/inventory/inventory/inventory/common/rpc_service.py new file mode 100644 index 00000000..5a89f62f --- /dev/null +++ b/inventory/inventory/inventory/common/rpc_service.py @@ -0,0 +1,95 @@ +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2017-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import signal + +from oslo_log import log +import oslo_messaging as messaging +from oslo_service import service +from oslo_utils import importutils + +from inventory.common import context +from inventory.common import rpc +from inventory.objects import base as objects_base + +LOG = log.getLogger(__name__) + + +class RPCService(service.Service): + + def __init__(self, host, manager_module, manager_class): + super(RPCService, self).__init__() + self.host = host + manager_module = importutils.try_import(manager_module) + manager_class = getattr(manager_module, manager_class) + self.manager = manager_class(host, manager_module.MANAGER_TOPIC) + self.topic = self.manager.topic + self.rpcserver = None + self.deregister = True + + def start(self): + super(RPCService, self).start() + admin_context = context.get_admin_context() + + target = messaging.Target(topic=self.topic, server=self.host) + endpoints = [self.manager] + serializer = objects_base.InventoryObjectSerializer() + self.rpcserver = rpc.get_server(target, endpoints, serializer) + self.rpcserver.start() + + self.handle_signal() + self.manager.init_host(admin_context) + + LOG.info('Created RPC server for service %(service)s on host ' + '%(host)s.', + {'service': self.topic, 'host': self.host}) + + def stop(self): + try: + self.rpcserver.stop() + self.rpcserver.wait() + except Exception as e: + LOG.exception('Service error occurred when stopping the ' + 'RPC server. Error: %s', e) + try: + self.manager.del_host(deregister=self.deregister) + except Exception as e: + LOG.exception('Service error occurred when cleaning up ' + 'the RPC manager. Error: %s', e) + + super(RPCService, self).stop(graceful=True) + LOG.info('Stopped RPC server for service %(service)s on host ' + '%(host)s.', + {'service': self.topic, 'host': self.host}) + + def _handle_signal(self, signo, frame): + LOG.info('Got signal SIGUSR1. Not deregistering on next shutdown ' + 'of service %(service)s on host %(host)s.', + {'service': self.topic, 'host': self.host}) + self.deregister = False + + def handle_signal(self): + """Add a signal handler for SIGUSR1. + + The handler ensures that the manager is not deregistered when it is + shutdown. + """ + signal.signal(signal.SIGUSR1, self._handle_signal) diff --git a/inventory/inventory/inventory/common/service.py b/inventory/inventory/inventory/common/service.py new file mode 100644 index 00000000..8c54a491 --- /dev/null +++ b/inventory/inventory/inventory/common/service.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# Copyright © 2012 eNovance +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from inventory.common import config +from inventory.conf import opts +from inventory import objects +from oslo_config import cfg +from oslo_log import log +from oslo_service import service + + +cfg.CONF.register_opts([ + cfg.IntOpt('periodic_interval', + default=60, + help='seconds between running periodic tasks'), +]) + +CONF = cfg.CONF + + +def prepare_service(argv=None): + argv = [] if argv is None else argv + + opts.update_opt_defaults() + log.register_options(CONF) + CONF(argv[1:], project='inventory') + config.parse_args(argv) + log.setup(CONF, 'inventory') + objects.register_all() + + +def process_launcher(): + return service.ProcessLauncher(CONF) diff --git a/inventory/inventory/inventory/common/sm_api.py b/inventory/inventory/inventory/common/sm_api.py new file mode 100644 index 00000000..b51c447e --- /dev/null +++ b/inventory/inventory/inventory/common/sm_api.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2016-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from inventory import objects +import json +from keystoneauth1.access import service_catalog as k_service_catalog +from oslo_log import log +from rest_api import rest_api_request + +LOG = log.getLogger(__name__) + + +def _get_region(context): + system = objects.System.get_one(context) + return system.region_name + + +def _get_endpoint(context): + # service_type, service_name, interface = \ + # CONF.smapi.catalog_info.split(':') + region_name = _get_region(context) + sc = k_service_catalog.ServiceCatalogV2(context.service_catalog) + service_parameters = {'service_type': 'smapi', + 'service_name': 'smapi', + 'interface': 'internalURL', + 'region_name': region_name} + endpoint = sc.url_for(**service_parameters) + return endpoint + + +def swact_pre_check(context, hostname, timeout=30): + """ + Sends a Swact Pre-Check command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/servicenode/%s" % hostname + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + api_cmd_payload = dict() + api_cmd_payload['origin'] = "inventory" + api_cmd_payload['action'] = "swact-pre-check" + api_cmd_payload['admin'] = "unknown" + api_cmd_payload['oper'] = "unknown" + api_cmd_payload['avail'] = "" + + response = rest_api_request(context, "PATCH", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def lock_pre_check(context, hostname, timeout=30): + """ + Sends a Lock Pre-Check command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/servicenode/%s" % hostname + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + api_cmd_payload = dict() + api_cmd_payload['origin'] = "inventory" + api_cmd_payload['action'] = "lock-pre-check" + api_cmd_payload['admin'] = "unknown" + api_cmd_payload['oper'] = "unknown" + api_cmd_payload['avail'] = "" + + response = rest_api_request(context, "PATCH", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + + return response + + +def service_list(context): + """ + Sends a service list command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/services" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['Accept'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, None) + + return response + + +def service_show(context, hostname): + """ + Sends a service show command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/services/%s" % hostname + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['Accept'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, None) + return response + + +def servicenode_list(context): + """ + Sends a service list command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/nodes" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['Accept'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, None) + + return response + + +def servicenode_show(context, hostname): + """ + Sends a service show command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/nodes/%s" % hostname + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['Accept'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, None) + + return response + + +def sm_servicegroup_list(context): + """ + Sends a service list command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/sm_sda" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['Accept'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, None) + + # rename the obsolete sm_sda to sm_servicegroups + if isinstance(response, dict): + if 'sm_sda' in response: + response['sm_servicegroup'] = response.pop('sm_sda') + + return response + + +def sm_servicegroup_show(context, hostname): + """ + Sends a service show command to SM. + """ + api_cmd = _get_endpoint(context) + api_cmd += "/v1/sm_sda/%s" % hostname + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['Accept'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, None) + + return response diff --git a/inventory/inventory/inventory/common/storage_backend_conf.py b/inventory/inventory/inventory/common/storage_backend_conf.py new file mode 100644 index 00000000..3619c0a9 --- /dev/null +++ b/inventory/inventory/inventory/common/storage_backend_conf.py @@ -0,0 +1,450 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Copyright (c) 2016-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# All Rights Reserved. +# + +""" System Inventory Storage Backend Utilities and helper functions.""" + + +import ast +import pecan +import wsme + +from inventory.common import constants +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_host +from oslo_log import log + +LOG = log.getLogger(__name__) + + +class StorageBackendConfig(object): + + @staticmethod + def get_backend(api, target): + """Get the primary backend. """ + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if (backend.backend == target and + backend.name == constants.SB_DEFAULT_NAMES[target]): + return backend + + @staticmethod + def get_backend_conf(api, target): + """Get the polymorphic primary backend. """ + + if target == constants.SB_TYPE_FILE: + # Only support a single file backend + storage_files = api.storage_file_get_list() + if storage_files: + return storage_files[0] + elif target == constants.SB_TYPE_LVM: + # Only support a single LVM backend + storage_lvms = api.storage_lvm_get_list() + if storage_lvms: + return storage_lvms[0] + elif target == constants.SB_TYPE_CEPH: + # Support multiple ceph backends + storage_cephs = api.storage_ceph_get_list() + primary_backends = filter( + lambda b: b['name'] == constants.SB_DEFAULT_NAMES[ + constants.SB_TYPE_CEPH], + storage_cephs) + if primary_backends: + return primary_backends[0] + elif target == constants.SB_TYPE_EXTERNAL: + # Only support a single external backend + storage_externals = api.storage_external_get_list() + if storage_externals: + return storage_externals[0] + elif target == constants.SB_TYPE_CEPH_EXTERNAL: + # Support multiple ceph external backends + storage_ceph_externals = api.storage_ceph_external_get_list() + if storage_ceph_externals: + return storage_ceph_externals[0] + + return None + + @staticmethod + def get_configured_backend_conf(api, target): + """Return the configured polymorphic primary backend + of a given type. + """ + + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if backend.state == constants.SB_STATE_CONFIGURED and \ + backend.backend == target and \ + backend.name == constants.SB_DEFAULT_NAMES[target]: + return StorageBackendConfig.get_backend_conf(api, target) + return None + + @staticmethod + def get_configured_backend_list(api): + """Get the list of all configured backends. """ + + backends = [] + try: + backend_list = api.storage_backend_get_list() + except Exception: + backend_list = [] + + for backend in backend_list: + if backend.state == constants.SB_STATE_CONFIGURED: + backends.append(backend.backend) + return backends + + @staticmethod + def get_configured_backend(api, target): + """Return the configured primary backend of a given type.""" + + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if backend.state == constants.SB_STATE_CONFIGURED and \ + backend.backend == target and \ + backend.name == constants.SB_DEFAULT_NAMES[target]: + return backend + return None + + @staticmethod + def get_configuring_backend(api): + """Get the primary backend that is configuring. """ + + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if (backend.state == constants.SB_STATE_CONFIGURING and + backend.name == + constants.SB_DEFAULT_NAMES[backend.backend]): + # At this point we can have but only max 1 configuring backend + # at any moment + return backend + + # it is normal there isn't one being configured + return None + + @staticmethod + def get_configuring_target_backend(api, target): + """Get the primary backend that is configuring. """ + + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if (backend.state == constants.SB_STATE_CONFIGURING and + backend.backend == target): + # At this point we can have but only max 1 configuring backend + # at any moment + return backend + + # it is normal there isn't one being configured + return None + + @staticmethod + def has_backend_configured(dbapi, target, service=None, + check_only_defaults=True, rpcapi=None): + """Check is a backend is configured. """ + # If cinder is a shared service on another region and + # we want to know if the ceph backend is configured, + # send a rpc to conductor which sends a query to the primary + system = dbapi.system_get_one() + shared_services = system.capabilities.get('shared_services', None) + configured = False + if (shared_services is not None and + constants.SERVICE_TYPE_VOLUME in shared_services and + target == constants.SB_TYPE_CEPH and + rpcapi is not None): + return rpcapi.region_has_ceph_backend( + pecan.request.context) + else: + backend_list = dbapi.storage_backend_get_list() + for backend in backend_list: + if (backend.state == constants.SB_STATE_CONFIGURED and + backend.backend == target): + configured = True + break + + # Supplementary semantics + if configured: + if check_only_defaults and \ + backend.name != constants.SB_DEFAULT_NAMES[target]: + configured = False + if service and service not in backend.services: + configured = False + + return configured + + @staticmethod + def has_backend(api, target): + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if backend.backend == target: + return True + return False + + @staticmethod + def update_backend_states(api, target, state=None, task='N/A'): + """Update primary backend state. """ + + values = dict() + if state: + values['state'] = state + if task != 'N/A': + values['task'] = task + backend = StorageBackendConfig.get_backend(api, target) + if backend: + api.storage_backend_update(backend.uuid, values) + else: + raise exception.InvalidStorageBackend(backend=target) + + @staticmethod + def get_ceph_mon_ip_addresses(dbapi): + try: + dbapi.network_get_by_type( + constants.NETWORK_TYPE_INFRA + ) + network_type = constants.NETWORK_TYPE_INFRA + except exception.NetworkTypeNotFound: + network_type = constants.NETWORK_TYPE_MGMT + + targets = { + '%s-%s' % (k_host.CONTROLLER_0_HOSTNAME, + network_type): 'ceph-mon-0-ip', + '%s-%s' % (k_host.CONTROLLER_1_HOSTNAME, + network_type): 'ceph-mon-1-ip', + '%s-%s' % (k_host.STORAGE_0_HOSTNAME, + network_type): 'ceph-mon-2-ip' + } + results = {} + addrs = dbapi.addresses_get_all() + for addr in addrs: + if addr.name in targets: + results[targets[addr.name]] = addr.address + if len(results) != len(targets): + raise exception.IncompleteCephMonNetworkConfig( + targets=targets, results=results) + return results + + @staticmethod + def is_ceph_backend_ready(api): + """ + check if ceph primary backend is ready, i,e, when a ceph backend + is configured after config_controller, it is considered ready when + both controller nodes and 1st pair of storage nodes are reconfigured + with ceph + :param api: + :return: + """ + ceph_backend = None + backend_list = api.storage_backend_get_list() + for backend in backend_list: + if backend.backend == constants.SB_TYPE_CEPH and \ + backend.name == constants.SB_DEFAULT_NAMES[ + constants.SB_TYPE_CEPH]: + ceph_backend = backend + break + if not ceph_backend: + return False + + if ceph_backend.state != constants.SB_STATE_CONFIGURED: + return False + + if ceph_backend.task == constants.SB_TASK_PROVISION_STORAGE: + return False + + # if both controllers are reconfigured and 1st pair storage nodes + # are provisioned, the task will be either reconfig_compute or none + return True + + @staticmethod + def get_ceph_tier_size(dbapi, rpcapi, tier_name): + try: + # Make sure the default ceph backend is configured + if not StorageBackendConfig.has_backend_configured( + dbapi, + constants.SB_TYPE_CEPH + ): + return 0 + + tier_size = \ + rpcapi.get_ceph_tier_size(pecan.request.context, + tier_name) + return int(tier_size) + except Exception as exp: + LOG.exception(exp) + return 0 + + @staticmethod + def get_ceph_pool_replication(api): + """ + return the values of 'replication' and 'min_replication' + capabilities as configured in ceph backend + :param api: + :return: replication, min_replication + """ + # Get ceph backend from db + ceph_backend = StorageBackendConfig.get_backend( + api, + constants.CINDER_BACKEND_CEPH + ) + + # Workaround for upgrade from R4 to R5, where 'capabilities' field + # does not exist in R4 backend entry + if hasattr(ceph_backend, 'capabilities'): + if (constants.CEPH_BACKEND_REPLICATION_CAP in + ceph_backend.capabilities): + pool_size = int(ceph_backend.capabilities[ + constants.CEPH_BACKEND_REPLICATION_CAP]) + + pool_min_size = \ + constants.CEPH_REPLICATION_MAP_DEFAULT[pool_size] + else: + # Should not get here + pool_size = constants.CEPH_REPLICATION_FACTOR_DEFAULT + pool_min_size = \ + constants.CEPH_REPLICATION_MAP_DEFAULT[pool_size] + else: + # upgrade compatibility with R4 + pool_size = constants.CEPH_REPLICATION_FACTOR_DEFAULT + pool_min_size = constants.CEPH_REPLICATION_MAP_DEFAULT[pool_size] + + return pool_size, pool_min_size + + @staticmethod + def get_ceph_backend_task(api): + """ + return current ceph backend task + :param: api + :return: + """ + # Get ceph backend from db + ceph_backend = StorageBackendConfig.get_backend( + api, + constants.CINDER_BACKEND_CEPH + ) + + return ceph_backend.task + + @staticmethod + def get_ceph_backend_state(api): + """ + return current ceph backend state + :param: api + :return: + """ + # Get ceph backend from db + ceph_backend = StorageBackendConfig.get_backend( + api, + constants.CINDER_BACKEND_CEPH + ) + + return ceph_backend.state + + @staticmethod + def is_ceph_backend_restore_in_progress(api): + """ + check ceph primary backend has a restore task set + :param api: + :return: + """ + for backend in api.storage_backend_get_list(): + if (backend.backend == constants.SB_TYPE_CEPH and + backend.name == constants.SB_DEFAULT_NAMES[ + constants.SB_TYPE_CEPH]): + return backend.task == constants.SB_TASK_RESTORE + + @staticmethod + def set_img_conversions_defaults(dbapi, controller_fs_api): + """ + initialize img_conversion partitions with default values if not + already done + :param dbapi + :param controller_fs_api + """ + # Img conversions identification + values = {'name': constants.FILESYSTEM_NAME_IMG_CONVERSIONS, + 'logical_volume': constants.FILESYSTEM_LV_DICT[ + constants.FILESYSTEM_NAME_IMG_CONVERSIONS], + 'replicated': False} + + # Abort if is already defined + controller_fs_list = dbapi.controller_fs_get_list() + for fs in controller_fs_list: + if values['name'] == fs.name: + LOG.info("Image conversions already defined, " + "avoiding reseting values") + return + + # Check if there is enough space available + rootfs_max_GiB, cgtsvg_max_free_GiB = \ + controller_fs_api.get_controller_fs_limit() + args = {'avail': cgtsvg_max_free_GiB, + 'min': constants.DEFAULT_SMALL_IMG_CONVERSION_STOR_SIZE, + 'lvg': constants.LVG_CGTS_VG} + if cgtsvg_max_free_GiB >= constants.DEFAULT_IMG_CONVERSION_STOR_SIZE: + img_conversions_gib = constants.DEFAULT_IMG_CONVERSION_STOR_SIZE + elif (cgtsvg_max_free_GiB >= + constants.DEFAULT_SMALL_IMG_CONVERSION_STOR_SIZE): + img_conversions_gib = \ + constants.DEFAULT_SMALL_IMG_CONVERSION_STOR_SIZE + else: + msg = _("Not enough space for image conversion partition. " + "Please ensure that '%(lvg)s' VG has " + "at least %(min)s GiB free space." + "Currently available: %(avail)s GiB.") % args + raise wsme.exc.ClientSideError(msg) + + args['size'] = img_conversions_gib + LOG.info("Available space in '%(lvg)s' is %(avail)s GiB " + "from which img_conversions will use %(size)s GiB." % args) + + # Create entry + values['size'] = img_conversions_gib + dbapi.controller_fs_create(values) + + @staticmethod + def get_enabled_services(dbapi, filter_unconfigured=True, + filter_shared=False): + """Get the list of enabled services + :param dbapi + :param filter_unconfigured: Determine weather to ignore + unconfigured services + :param filter_shared: Determine weather to ignore shared services + :returns: list of services + """ + services = [] + if not filter_shared: + system = dbapi.system_get_one() + shared_services = system.capabilities.get('shared_services', None) + services = [] if shared_services is None \ + else ast.literal_eval(shared_services) + + backend_list = dbapi.storage_backend_get_list() + for backend in backend_list: + backend_services = [] if backend.services is None \ + else backend.services.split(',') + for service in backend_services: + if (backend.state == constants.SB_STATE_CONFIGURED or + not filter_unconfigured): + if service not in services: + services.append(service) + return services + # TODO(oponcea): Check for external cinder backend & test multiregion + + @staticmethod + def is_service_enabled(dbapi, service, filter_unconfigured=True, + filter_shared=False): + """Checks if a service is enabled + :param dbapi + :param service: service name, one of constants.SB_SVC_* + :param unconfigured: check also unconfigured/failed services + :returns: True or false + """ + if service in StorageBackendConfig.get_enabled_services( + dbapi, filter_unconfigured, filter_shared): + return True + else: + return False diff --git a/inventory/inventory/inventory/common/utils.py b/inventory/inventory/inventory/common/utils.py new file mode 100644 index 00000000..da0a89ad --- /dev/null +++ b/inventory/inventory/inventory/common/utils.py @@ -0,0 +1,1263 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# Copyright (c) 2012 NTT DOCOMO, INC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Utilities and helper functions.""" + +import collections +import contextlib +import datetime +import errno +import fcntl +import functools +import glob +import hashlib +import itertools as it +import netaddr +import os +import random +import re +import shutil +import signal +import six +import socket +import tempfile +import time +import uuid +import wsme + +from eventlet.green import subprocess +from eventlet import greenthread + +from oslo_concurrency import lockutils +from oslo_log import log + +from inventory.common import constants +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.conf import CONF +from six import text_type as unicode +from tsconfig.tsconfig import SW_VERSION + +LOG = log.getLogger(__name__) + +# Used for looking up extensions of text +# to their 'multiplied' byte amount +BYTE_MULTIPLIERS = { + '': 1, + 't': 1024 ** 4, + 'g': 1024 ** 3, + 'm': 1024 ** 2, + 'k': 1024, +} + + +class memoized(object): + """Decorator to cache a functions' return value. + + Decorator. Caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned + (not reevaluated). + + WARNING: This function should not be used for class methods since it + does not provide weak references; thus would prevent the instance from + being garbage collected. + """ + def __init__(self, func): + self.func = func + self.cache = {} + + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + else: + value = self.func(*args) + self.cache[args] = value + return value + + def __repr__(self): + '''Return the function's docstring.''' + return self.func.__doc__ + + def __get__(self, obj, objtype): + '''Support instance methods.''' + return functools.partial(self.__call__, obj) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def execute(*cmd, **kwargs): + """Helper method to execute command with optional retry. + + If you add a run_as_root=True command, don't forget to add the + corresponding filter to etc/inventory/rootwrap.d ! + + :param cmd: Passed to subprocess.Popen. + :param process_input: Send to opened process. + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + exception.ProcessExecutionError unless + program exits with one of these code. + :param delay_on_retry: True | False. Defaults to True. If set to + True, wait a short amount of time + before retrying. + :param attempts: How many times to retry cmd. + :param run_as_root: True | False. Defaults to False. If set to True, + the command is run with rootwrap. + + :raises exception.InventoryException: on receiving unknown arguments + :raises exception.ProcessExecutionError: + + :returns: a tuple, (stdout, stderr) from the spawned process, or None if + the command fails. + """ + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) + shell = kwargs.pop('shell', False) + + if len(kwargs): + raise exception.InventoryException( + _('Got unknown keyword args to utils.execute: %r') % kwargs) + + if run_as_root and os.geteuid() != 0: + cmd = (['sudo', 'inventory-rootwrap', CONF.rootwrap_config] + + list(cmd)) + + cmd = map(str, cmd) + + while attempts > 0: + attempts -= 1 + try: + LOG.debug(_('Running cmd (subprocess): %s'), ' '.join(cmd)) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.debug(_('Result was %s') % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + raise exception.ProcessExecutionError( + exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + return result + except exception.ProcessExecutionError: + if not attempts: + raise + else: + LOG.debug(_('%r failed. Retrying.'), cmd) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except exception.ProcessExecutionError as exn: + out, err = '', str(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def is_int_like(val): + """Check if a value looks like an int.""" + try: + return str(int(val)) == str(val) + except Exception: + return False + + +def is_float_like(val): + """Check if a value looks like a float.""" + try: + return str(float(val)) == str(val) + except Exception: + return False + + +def is_valid_boolstr(val): + """Check if the provided string is a valid bool string or not.""" + boolstrs = ('true', 'false', 'yes', 'no', 'y', 'n', '1', '0') + return str(val).lower() in boolstrs + + +def is_valid_mac(address): + """Verify the format of a MAC addres.""" + m = "[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$" + if isinstance(address, six.string_types) and re.match(m, address.lower()): + return True + return False + + +def validate_and_normalize_mac(address): + """Validate a MAC address and return normalized form. + + Checks whether the supplied MAC address is formally correct and + normalize it to all lower case. + + :param address: MAC address to be validated and normalized. + :returns: Normalized and validated MAC address. + :raises: InvalidMAC If the MAC address is not valid. + :raises: ClonedInterfaceNotFound If MAC address is not updated + while installing a cloned image. + + """ + if not is_valid_mac(address): + if constants.CLONE_ISO_MAC in address: + # get interface name from the label + intf_name = address.rsplit('-', 1)[1][1:] + raise exception.ClonedInterfaceNotFound(intf=intf_name) + else: + raise exception.InvalidMAC(mac=address) + return address.lower() + + +def is_valid_ipv4(address): + """Verify that address represents a valid IPv4 address.""" + try: + return netaddr.valid_ipv4(address) + except Exception: + return False + + +def is_valid_ipv6(address): + try: + return netaddr.valid_ipv6(address) + except Exception: + return False + + +def is_valid_ip(address): + if not is_valid_ipv4(address): + return is_valid_ipv6(address) + return True + + +def is_valid_ipv6_cidr(address): + try: + str(netaddr.IPNetwork(address, version=6).cidr) + return True + except Exception: + return False + + +def get_shortened_ipv6(address): + addr = netaddr.IPAddress(address, version=6) + return str(addr.ipv6()) + + +def get_shortened_ipv6_cidr(address): + net = netaddr.IPNetwork(address, version=6) + return str(net.cidr) + + +def is_valid_cidr(address): + """Check if the provided ipv4 or ipv6 address is a valid CIDR address.""" + try: + # Validate the correct CIDR Address + netaddr.IPNetwork(address) + except netaddr.core.AddrFormatError: + return False + except UnboundLocalError: + # NOTE(MotoKen): work around bug in netaddr 0.7.5 (see detail in + # https://github.com/drkjam/netaddr/issues/2) + return False + + # Prior validation partially verify /xx part + # Verify it here + ip_segment = address.split('/') + + if len(ip_segment) <= 1 or ip_segment[1] == '': + return False + + return True + + +def is_valid_hex(num): + try: + int(num, 16) + except ValueError: + return False + return True + + +def is_valid_pci_device_vendor_id(id): + """Check if the provided id is a valid 16 bit hexadecimal.""" + val = id.replace('0x', '').strip() + if not is_valid_hex(id): + return False + if len(val) > 4: + return False + return True + + +def is_valid_pci_class_id(id): + """Check if the provided id is a valid 16 bit hexadecimal.""" + val = id.replace('0x', '').strip() + if not is_valid_hex(id): + return False + if len(val) > 6: + return False + return True + + +def is_system_usable_block_device(pydev_device): + """Check if a block device is local and can be used for partitioning + + Example devices: + o local block devices: local HDDs, SSDs, RAID arrays + o remote devices: iscsi mounted, LIO, EMC + o non permanent devices: USB stick + :return bool: True if device can be used else False + """ + if pydev_device.get("ID_BUS") == "usb": + # Skip USB devices + return False + if pydev_device.get("DM_VG_NAME") or pydev_device.get("DM_LV_NAME"): + # Skip LVM devices + return False + id_path = pydev_device.get("ID_PATH", "") + if "iqn." in id_path or "eui." in id_path: + # Skip all iSCSI devices, they are links for volume storage. + # As per https://www.ietf.org/rfc/rfc3721.txt, "iqn." or "edu." + # have to be present when constructing iSCSI names. + return False + if pydev_device.get("ID_VENDOR") == constants.VENDOR_ID_LIO: + # LIO devices are iSCSI, should be skipped above! + LOG.error("Invalid id_path. Device %s (%s) is iSCSI!" % + (id_path, pydev_device.get('DEVNAME'))) + return False + return True + + +def get_ip_version(network): + """Returns the IP version of a network (IPv4 or IPv6). + + :raises: AddrFormatError if invalid network. + """ + if netaddr.IPNetwork(network).version == 6: + return "IPv6" + elif netaddr.IPNetwork(network).version == 4: + return "IPv4" + + +def convert_to_list_dict(lst, label): + """Convert a value or list into a list of dicts.""" + if not lst: + return None + if not isinstance(lst, list): + lst = [lst] + return [{label: x} for x in lst] + + +def sanitize_hostname(hostname): + """Return a hostname which conforms to RFC-952 and RFC-1123 specs.""" + if isinstance(hostname, unicode): + hostname = hostname.encode('latin-1', 'ignore') + + hostname = re.sub('[ _]', '-', hostname) + hostname = re.sub('[^\w.-]+', '', hostname) + hostname = hostname.lower() + hostname = hostname.strip('.-') + + return hostname + + +def hash_file(file_like_object): + """Generate a hash for the contents of a file.""" + checksum = hashlib.sha1() + for chunk in iter(lambda: file_like_object.read(32768), b''): + checksum.update(chunk) + return checksum.hexdigest() + + +@contextlib.contextmanager +def tempdir(**kwargs): + tempfile.tempdir = CONF.tempdir + tmpdir = tempfile.mkdtemp(**kwargs) + try: + yield tmpdir + finally: + try: + shutil.rmtree(tmpdir) + except OSError as e: + LOG.error(_('Could not remove tmpdir: %s'), str(e)) + + +def mkfs(fs, path, label=None): + """Format a file or block device + + :param fs: Filesystem type (examples include 'swap', 'ext3', 'ext4' + 'btrfs', etc.) + :param path: Path to file or block device to format + :param label: Volume label to use + """ + if fs == 'swap': + args = ['mkswap'] + else: + args = ['mkfs', '-t', fs] + # add -F to force no interactive execute on non-block device. + if fs in ('ext3', 'ext4'): + args.extend(['-F']) + if label: + if fs in ('msdos', 'vfat'): + label_opt = '-n' + else: + label_opt = '-L' + args.extend([label_opt, label]) + args.append(path) + execute(*args) + + +def safe_rstrip(value, chars=None): + """Removes trailing characters from a string if that does not make it empty + + :param value: A string value that will be stripped. + :param chars: Characters to remove. + :return: Stripped value. + + """ + if not isinstance(value, six.string_types): + LOG.warn(_("Failed to remove trailing character. Returning original " + "object. Supplied object is not a string: %s,") % value) + return value + + return value.rstrip(chars) or value + + +def generate_uuid(): + return str(uuid.uuid4()) + + +def is_uuid_like(val): + """Returns validation of a value as a UUID. + + For our purposes, a UUID is a canonical form string: + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa + + """ + try: + return str(uuid.UUID(val)) == val + except (TypeError, ValueError, AttributeError): + return False + + +def removekey(d, key): + r = dict(d) + del r[key] + return r + + +def removekeys_nonmtce(d, keepkeys=None): + if not keepkeys: + keepkeys = [] + + nonmtce_keys = ['created_at', + 'updated_at', + 'host_action', + 'vim_progress_status', + 'task', + 'uptime', + 'location', + 'serialid', + 'config_status', + 'config_applied', + 'config_target', + 'reserved', + 'system_id'] + # 'action_state', + r = dict(d) + + for k in nonmtce_keys: + if r.get(k) and (k not in keepkeys): + del r[k] + return r + + +def removekeys_nonhwmon(d, keepkeys=None): + if not keepkeys: + keepkeys = [] + + nonmtce_keys = ['created_at', + 'updated_at', + ] + r = dict(d) + + for k in nonmtce_keys: + if r.get(k) and (k not in keepkeys): + del r[k] + return r + + +def touch(fname): + with open(fname, 'a'): + os.utime(fname, None) + + +def symlink_force(source, link_name): + """Force creation of a symlink + + :param: source: path to the source + :param: link_name: symbolic link name + """ + try: + os.symlink(source, link_name) + except OSError as e: + if e.errno == errno.EEXIST: + os.remove(link_name) + os.symlink(source, link_name) + + +@contextlib.contextmanager +def mounted(remote_dir, local_dir): + local_dir = os.path.abspath(local_dir) + try: + subprocess.check_output( + ["/bin/nfs-mount", remote_dir, local_dir], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + raise OSError(("mount operation failed: " + "command={}, retcode={}, output='{}'").format( + e.cmd, e.returncode, e.output)) + try: + yield + finally: + try: + subprocess.check_output( + ["/bin/umount", local_dir], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + raise OSError(("umount operation failed: " + "command={}, retcode={}, output='{}'").format( + e.cmd, e.returncode, e.output)) + + +def timestamped(dname, fmt='{dname}_%Y-%m-%d-%H-%M-%S'): + return datetime.datetime.now().strftime(fmt).format(dname=dname) + + +def host_has_function(host, function): + return function in (host.get('subfunctions') or + host.get('personality') or '') + + +@memoized +def is_virtual(): + '''Determines if the system is virtualized or not''' + subp = subprocess.Popen(['facter', 'is_virtual'], + stdout=subprocess.PIPE) + if subp.wait(): + raise Exception("Failed to read virtualization status from facter") + output = subp.stdout.readlines() + if len(output) != 1: + raise Exception("Unexpected number of lines: %d" % len(output)) + result = output[0].strip() + return bool(result == 'true') + + +def is_virtual_compute(ihost): + if not(os.path.isdir("/etc/inventory/.virtual_compute_nodes")): + return False + try: + ip = ihost['mgmt_ip'] + return os.path.isfile("/etc/inventory/.virtual_compute_nodes/%s" % + ip) + except AttributeError: + return False + + +def is_low_core_system(ihost, dba): + """Determine whether a low core cpu count system. + + Determine if the hosts core count is less than or equal to a xeon-d cpu + used with get_required_platform_reserved_memory to set the the required + platform memory for xeon-d systems + """ + cpu_list = dba.cpu_get_by_host(ihost['uuid']) + number_physical_cores = 0 + for cpu in cpu_list: + if int(cpu['thread']) == 0: + number_physical_cores += 1 + return number_physical_cores <= constants.NUMBER_CORES_XEOND + + +def get_minimum_platform_reserved_memory(ihost, numa_node): + """Returns the minimum amount of memory to be reserved by the platform + + For a given NUMA node. Compute nodes require reserved memory because the + balance of the memory is allocated to VM instances. Other node types + have exclusive use of the memory so no explicit reservation is + required. Memory required by platform core is not included here. + """ + reserved = 0 + if numa_node is None: + return reserved + if is_virtual() or is_virtual_compute(ihost): + # minimal memory requirements for VirtualBox + if host_has_function(ihost, k_host.COMPUTE): + if numa_node == 0: + reserved += 1200 + if host_has_function(ihost, k_host.CONTROLLER): + reserved += 5000 + else: + reserved += 500 + else: + if host_has_function(ihost, k_host.COMPUTE): + # Engineer 2G per numa node for disk IO RSS overhead + reserved += constants.DISK_IO_RESIDENT_SET_SIZE_MIB + return reserved + + +def get_required_platform_reserved_memory(ihost, numa_node, low_core=False): + """Returns the amount of memory to be reserved by the platform. + + For a a given NUMA node. Compute nodes require reserved memory because the + balance of the memory is allocated to VM instances. Other node types + have exclusive use of the memory so no explicit reservation is + required. + """ + required_reserved = 0 + if numa_node is None: + return required_reserved + if is_virtual() or is_virtual_compute(ihost): + # minimal memory requirements for VirtualBox + required_reserved += constants.DISK_IO_RESIDENT_SET_SIZE_MIB_VBOX + if host_has_function(ihost, k_host.COMPUTE): + if numa_node == 0: + required_reserved += \ + constants.PLATFORM_CORE_MEMORY_RESERVED_MIB_VBOX + if host_has_function(ihost, k_host.CONTROLLER): + required_reserved += \ + constants.COMBINED_NODE_CONTROLLER_MEMORY_RESERVED_MIB_VBOX # noqa + else: + # If not a controller, add overhead for + # metadata and vrouters + required_reserved += \ + constants.NETWORK_METADATA_OVERHEAD_MIB_VBOX + else: + required_reserved += \ + constants.DISK_IO_RESIDENT_SET_SIZE_MIB_VBOX + else: + if host_has_function(ihost, k_host.COMPUTE): + # Engineer 2G per numa node for disk IO RSS overhead + required_reserved += constants.DISK_IO_RESIDENT_SET_SIZE_MIB + if numa_node == 0: + # Engineer 2G for compute to give some headroom; + # typically requires 650 MB PSS + required_reserved += \ + constants.PLATFORM_CORE_MEMORY_RESERVED_MIB + if host_has_function(ihost, k_host.CONTROLLER): + # Over-engineer controller memory. + # Typically require 5GB PSS; accommodate 2GB headroom. + # Controller memory usage depends on number of workers. + if low_core: + required_reserved += \ + constants.COMBINED_NODE_CONTROLLER_MEMORY_RESERVED_MIB_XEOND # noqa + else: + required_reserved += \ + constants.COMBINED_NODE_CONTROLLER_MEMORY_RESERVED_MIB # noqa + else: + # If not a controller, + # add overhead for metadata and vrouters + required_reserved += \ + constants.NETWORK_METADATA_OVERHEAD_MIB + return required_reserved + + +def get_network_type_list(interface): + if interface['networktype']: + return [n.strip() for n in interface['networktype'].split(",")] + else: + return [] + + +def is_pci_network_types(networktypelist): + """Check whether pci network types in list + + Check if the network type consists of the combined PCI passthrough + and SRIOV network types. + """ + return (len(constants.PCI_NETWORK_TYPES) == len(networktypelist) and + all(i in networktypelist for i in constants.PCI_NETWORK_TYPES)) + + +def get_sw_version(): + return SW_VERSION + + +class ISO(object): + + def __init__(self, iso_path, mount_dir): + self.iso_path = iso_path + self.mount_dir = mount_dir + self._iso_mounted = False + self._mount_iso() + + def __del__(self): + if self._iso_mounted: + self._umount_iso() + + def _mount_iso(self): + with open(os.devnull, "w") as fnull: + subprocess.check_call(['mkdir', '-p', self.mount_dir], + stdout=fnull, + stderr=fnull) + subprocess.check_call(['mount', '-r', '-o', 'loop', self.iso_path, + self.mount_dir], + stdout=fnull, + stderr=fnull) + self._iso_mounted = True + + def _umount_iso(self): + try: + # Do a lazy unmount to handle cases where a file in the mounted + # directory is open when the umount is done. + subprocess.check_call(['umount', '-l', self.mount_dir]) + self._iso_mounted = False + except subprocess.CalledProcessError as e: + # If this fails for some reason, there's not a lot we can do + # Just log the exception and keep going + LOG.exception(e) + + +def get_active_load(loads): + active_load = None + for db_load in loads: + if db_load.state == constants.ACTIVE_LOAD_STATE: + active_load = db_load + + if active_load is None: + raise exception.InventoryException(_("No active load found")) + + return active_load + + +def get_imported_load(loads): + imported_load = None + for db_load in loads: + if db_load.state == constants.IMPORTED_LOAD_STATE: + imported_load = db_load + + if imported_load is None: + raise exception.InventoryException(_("No imported load found")) + + return imported_load + + +def validate_loads_for_import(loads): + for db_load in loads: + if db_load.state == constants.IMPORTED_LOAD_STATE: + raise exception.InventoryException(_("Imported load exists.")) + + +def validate_load_for_delete(load): + if not load: + raise exception.InventoryException(_("Load not found")) + + valid_delete_states = [ + constants.IMPORTED_LOAD_STATE, + constants.ERROR_LOAD_STATE, + constants.DELETING_LOAD_STATE + ] + + if load.state not in valid_delete_states: + raise exception.InventoryException( + _("Only a load in an imported or error state can be deleted")) + + +def gethostbyname(hostname): + return socket.getaddrinfo(hostname, None)[0][4][0] + + +def get_local_controller_hostname(): + try: + local_hostname = socket.gethostname() + except Exception as e: + raise exception.InventoryException(_( + "Failed to get the local hostname: %s") % str(e)) + return local_hostname + + +def get_mate_controller_hostname(hostname=None): + if not hostname: + try: + hostname = socket.gethostname() + except Exception as e: + raise exception.InventoryException(_( + "Failed to get the local hostname: %s") % str(e)) + + if hostname == k_host.CONTROLLER_0_HOSTNAME: + mate_hostname = k_host.CONTROLLER_1_HOSTNAME + elif hostname == k_host.CONTROLLER_1_HOSTNAME: + mate_hostname = k_host.CONTROLLER_0_HOSTNAME + else: + raise exception.InventoryException(_( + "Unknown local hostname: %s)") % hostname) + + return mate_hostname + + +def format_address_name(hostname, network_type): + return "%s-%s" % (hostname, network_type) + + +def validate_yes_no(name, value): + if value.lower() not in ['y', 'n']: + raise wsme.exc.ClientSideError(( + "Parameter '%s' must be a y/n value." % name)) + + +def get_interface_os_ifname(interface, interfaces, ports): + """Returns the operating system name for an interface. + + The user is allowed to override the inventory DB interface name for + convenience, but that name is not used at the operating system level for + all interface types. + For ethernet and VLAN interfaces the name follows the native interface + names while for AE interfaces the user defined name is used. + """ + if interface['iftype'] == constants.INTERFACE_TYPE_VLAN: + # VLAN interface names are built-in using the o/s name of the lower + # interface object. + lower_iface = interfaces[interface['uses'][0]] + lower_ifname = get_interface_os_ifname(lower_iface, interfaces, ports) + return '{}.{}'.format(lower_ifname, interface['vlan_id']) + elif interface['iftype'] == constants.INTERFACE_TYPE_ETHERNET: + # Ethernet interface names are always based on the port name which is + # just the normal o/s name of the original network interface + lower_ifname = ports[interface['id']]['name'] + return lower_ifname + else: + # All other interfaces default to the user-defined name + return interface['ifname'] + + +def get_dhcp_cid(hostname, network_type, mac): + """Create the CID for use with dnsmasq. + + We use a unique identifier for a client since different networks can + operate over the same device (and hence same MAC addr) when VLAN interfaces + are concerned. The format is different based on network type because the + mgmt network uses a default because it needs to exist before the board + is handled by inventory (i.e., the CID needs + to exist in the dhclient.conf file at build time) while the infra network + is built dynamically to avoid colliding with the mgmt CID. + + Example: + Format = 'id:' + colon-separated-hex(hostname:network_type) + ":" + mac + """ + if network_type == constants.NETWORK_TYPE_INFRA: + prefix = '{}:{}'.format(hostname, network_type) + prefix = ':'.join(x.encode('hex') for x in prefix) + elif network_type == constants.NETWORK_TYPE_MGMT: + # Our default dhclient.conf files requests a prefix of '00:03:00' to + # which dhclient adds a hardware address type of 01 to make final + # prefix of '00:03:00:01'. + prefix = '00:03:00:01' + else: + raise Exception("Network type {} does not support DHCP".format( + network_type)) + return '{}:{}'.format(prefix, mac) + + +def get_personalities(host_obj): + """Determine the personalities from host_obj""" + personalities = host_obj.subfunctions.split(',') + if k_host.LOWLATENCY in personalities: + personalities.remove(k_host.LOWLATENCY) + return personalities + + +def is_cpe(host_obj): + return (host_has_function(host_obj, k_host.CONTROLLER) and + host_has_function(host_obj, k_host.COMPUTE)) + + +def output_to_dict(output): + dict = {} + output = filter(None, output.split('\n')) + + for row in output: + values = row.split() + if len(values) != 2: + raise Exception("The following output does not respect the " + "format: %s" % row) + dict[values[1]] = values[0] + + return dict + + +def bytes_to_GiB(bytes_number): + return bytes_number / float(1024 ** 3) + + +def bytes_to_MiB(bytes_number): + return bytes_number / float(1024 ** 2) + + +def synchronized(name, external=True): + if external: + lock_path = constants.INVENTORY_LOCK_PATH + else: + lock_path = None + return lockutils.synchronized(name, + lock_file_prefix='inventory-', + external=external, + lock_path=lock_path) + + +def skip_udev_partition_probe(function): + def wrapper(*args, **kwargs): + """Decorator to skip partition rescanning in udev (fix for CGTS-8957) + + When reading partitions we have to avoid rescanning them as this + will temporarily delete their dev nodes causing devastating effects + for commands that rely on them (e.g. ceph-disk). + + UDEV triggers a partition rescan when a device node opened in write + mode is closed. To avoid this, we have to acquire a shared lock on the + device before other close operations do. + + Since both parted and sgdisk always open block devices in RW mode we + must disable udev from triggering the rescan when we just need to get + partition information. + + This happens due to a change in udev v214. For details see: + http://tracker.ceph.com/issues/14080 + http://tracker.ceph.com/issues/15176 + https://github.com/systemd/systemd/commit/02ba8fb3357 + daf57f6120ac512fb464a4c623419 + + :param device_node: dev node or path of the device + :returns decorated function + """ + device_node = kwargs.get('device_node', None) + if device_node: + with open(device_node, 'r') as f: + fcntl.flock(f, fcntl.LOCK_SH | fcntl.LOCK_NB) + try: + return function(*args, **kwargs) + finally: + # Since events are asynchronous we have to wait for udev + # to pick up the change. + time.sleep(0.1) + fcntl.flock(f, fcntl.LOCK_UN) + else: + return function(*args, **kwargs) + return wrapper + + +def disk_is_gpt(device_node): + """Checks if a device node is of GPT format. + + :param device_node: the disk's device node + :returns: True if partition table on disk is GPT + False if partition table on disk is not GPT + """ + parted_command = '{} {} {}'.format('parted -s', device_node, 'print') + parted_process = subprocess.Popen( + parted_command, stdout=subprocess.PIPE, shell=True) + parted_output = parted_process.stdout.read() + if re.search('Partition Table: gpt', parted_output): + return True + + return False + + +def partitions_are_in_order(disk_partitions, requested_partitions): + """Check if the disk partitions are in order with requested. + + Determine if a list of requested partitions can be created on a disk + with other existing partitions. + """ + + partitions_nr = [] + + for dp in disk_partitions: + part_number = re.match('.*?([0-9]+)$', dp.get('device_path')).group(1) + partitions_nr.append(int(part_number)) + + for rp in requested_partitions: + part_number = re.match('.*?([0-9]+)$', rp.get('device_path')).group(1) + partitions_nr.append(int(part_number)) + + return sorted(partitions_nr) == range(min(partitions_nr), + max(partitions_nr) + 1) + + +# TODO(oponcea): Remove once sm supports in-service configuration reload. +def is_single_controller(dbapi): + # Check the number of provisioned/provisioning hosts. If there is + # only one then we have a single controller (AIO-SX, single AIO-DX, or + # single std controller). If this is the case reset sm after adding + # cinder so that cinder DRBD/processes are managed. + hosts = dbapi.ihost_get_list() + prov_hosts = [h for h in hosts + if h.invprovision in [k_host.PROVISIONED, + k_host.PROVISIONING]] + if len(prov_hosts) == 1: + return True + return False + + +def is_partition_the_last(dbapi, partition): + """Check that the partition is the last partition. + + Used on check prior to delete. + """ + idisk_uuid = partition.get('idisk_uuid') + onidisk_parts = dbapi.partition_get_by_idisk(idisk_uuid) + part_number = re.match('.*?([0-9]+)$', + partition.get('device_path')).group(1) + + if int(part_number) != len(onidisk_parts): + return False + + return True + + +def _check_upgrade(dbapi, host_obj=None): + """Check whether partition operation may be allowed. + + If there is an upgrade in place, reject the operation if the + host was not created after upgrade start. + """ + try: + upgrade = dbapi.software_upgrade_get_one() + except exception.NotFound: + return + + if host_obj: + if host_obj.created_at > upgrade.created_at: + LOG.info("New host %s created after upgrade, allow partition" % + host_obj.hostname) + return + + raise wsme.exc.ClientSideError( + _("ERROR: Disk partition operations are not allowed during a " + "software upgrade. Try again after the upgrade is completed.")) + + +def disk_wipe(device): + """Wipe GPT table entries. + + We ignore exit codes in case disk is toasted or not present. + Note: Assumption is that entire disk is used + :param device: disk device node or device path + """ + LOG.info("Wiping device: %s " % device) + + # Wipe well known GPT table entries, if any. + trycmd('wipefs', '-f', '-a', device) + execute('udevadm', 'settle') + + # Wipe any other tables at the beginning of the device. + out, err = trycmd( + 'dd', 'if=/dev/zero', + 'of=%s' % device, + 'bs=512', 'count=2048', + 'conv=fdatasync') + LOG.info("Wiped beginning of disk: %s - %s" % (out, err)) + + # Get size of disk. + size, __ = trycmd('blockdev', '--getsz', + device) + size = size.rstrip() + + if size and size.isdigit(): + # Wipe at the end of device. + out, err = trycmd( + 'dd', 'if=/dev/zero', + 'of=%s' % device, + 'bs=512', 'count=2048', + 'seek=%s' % (int(size) - 2048), + 'conv=fdatasync') + LOG.info("Wiped end of disk: %s - %s" % (out, err)) + + LOG.info("Device %s zapped" % device) + + +def get_dhcp_client_iaid(mac_address): + """Retrieves the client IAID from its MAC address.""" + hwaddr = list(int(byte, 16) for byte in mac_address.split(':')) + return hwaddr[2] << 24 | hwaddr[3] << 16 | hwaddr[4] << 8 | hwaddr[5] + + +def get_cgts_vg_free_space(): + """Determine free space in cgts-vg""" + + try: + # Determine space in cgts-vg in GiB + vg_free_str = subprocess.check_output( + ['vgdisplay', '-C', '--noheadings', '--nosuffix', + '-o', 'vg_free', '--units', 'g', 'cgts-vg'], + close_fds=True).rstrip() + cgts_vg_free = int(float(vg_free_str)) + except subprocess.CalledProcessError: + LOG.error("Command vgdisplay failed") + raise Exception("Command vgdisplay failed") + + return cgts_vg_free + + +def read_filtered_directory_content(dirpath, *filters): + """Reads the content of a directory, filtered on glob like expressions. + + Returns a dictionary, with the "key" being the filename + and the "value" being the content of that file. + """ + def filter_directory_files(dirpath, *filters): + return it.chain.from_iterable(glob.iglob(dirpath + '/' + filter) + for filter in filters) + + content_dict = {} + for filename in filter_directory_files(dirpath, *filters): + content = "" + with open(os.path.join(filename), 'rb') as obj: + content = obj.read() + try: + # If the filter specified binary files then + # these will need to be base64 encoded so that + # they can be transferred over RPC and stored in DB + content.decode('utf-8') + except UnicodeError: + content = content.encode('base64') + content_dict['base64_encoded_files'] = \ + content_dict.get("base64_encoded_files", []) + [filename] + + content_dict[filename] = content + return content_dict + + +def get_disk_capacity_mib(device_node): + # Run command + fdisk_command = 'fdisk -l %s | grep "^Disk %s:"' % ( + device_node, device_node) + + try: + fdisk_output, _ = execute(fdisk_command, check_exit_code=[0], + run_as_root=True, attempts=3, + shell=True) + except exception.ProcessExecutionError: + LOG.error("Error running fdisk command: %s" % + fdisk_command) + return 0 + + # Parse output + second_half = fdisk_output.split(',')[1] + size_bytes = second_half.split()[0].strip() + + # Convert bytes to MiB (1 MiB = 1024*1024 bytes) + int_size = int(size_bytes) + size_mib = int_size / (1024 ** 2) + + return int(size_mib) + + +def format_range_set(items): + # Generate a pretty-printed value of ranges, such as 3-6,8-9,12-17 + ranges = [] + for k, iterable in it.groupby(enumerate(sorted(items)), + lambda x: x[1] - x[0]): + rng = list(iterable) + if len(rng) == 1: + s = str(rng[0][1]) + else: + s = "%s-%s" % (rng[0][1], rng[-1][1]) + ranges.append(s) + return ','.join(ranges) + + +def get_numa_index_list(obj): + """Create map of objects indexed by numa node""" + obj_lists = collections.defaultdict(list) + for index, o in enumerate(obj): + o["_index"] = index + obj_lists[o.numa_node].append(o) + return obj_lists + + +def compare(a, b): + return (a > b) - (a < b) diff --git a/inventory/inventory/inventory/common/vim_api.py b/inventory/inventory/inventory/common/vim_api.py new file mode 100644 index 00000000..7a12ad50 --- /dev/null +++ b/inventory/inventory/inventory/common/vim_api.py @@ -0,0 +1,156 @@ +# +# Copyright (c) 2015-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from inventory.common import constants +from inventory.common import k_host +from inventory import objects +import json +from keystoneauth1.access import service_catalog as k_service_catalog +from oslo_log import log +from rest_api import rest_api_request + +LOG = log.getLogger(__name__) + + +def _get_region(context): + system = objects.System.get_one(context) + return system.region_name + + +def _get_endpoint(context): + # service_type, service_name, interface = \ + # CONF.nfv.catalog_info.split(':') + region_name = _get_region(context) + sc = k_service_catalog.ServiceCatalogV2(context.service_catalog) + service_parameters = {'service_type': 'nfv', + 'service_name': 'vim', + 'interface': 'internalURL', + 'region_name': region_name} + endpoint = sc.url_for(**service_parameters) + LOG.info("NFV endpoint=%s" % endpoint) + return endpoint + + +def vim_host_add(context, uuid, hostname, subfunctions, + admininistrative, operational, availability, + subfunction_oper, subfunction_avail, + timeout=constants.VIM_DEFAULT_TIMEOUT_IN_SECS): + """ + Requests VIM to add a host. + """ + LOG.info("vim_host_add hostname=%s, subfunctions=%s " + "%s-%s-%s subfunction_oper=%s subfunction_avail=%s" % + (hostname, subfunctions, admininistrative, operational, + availability, subfunction_oper, subfunction_avail)) + + api_cmd = _get_endpoint(context) + api_cmd += "/nfvi-plugins/v1/hosts/" + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + api_cmd_payload = dict() + api_cmd_payload['uuid'] = uuid + api_cmd_payload['hostname'] = hostname + api_cmd_payload['subfunctions'] = subfunctions + api_cmd_payload['administrative'] = admininistrative + api_cmd_payload['operational'] = operational + api_cmd_payload['availability'] = availability + api_cmd_payload['subfunction_oper'] = subfunction_oper + api_cmd_payload['subfunction_avail'] = subfunction_avail + + LOG.warn("vim_host_add api_cmd=%s headers=%s payload=%s" % + (api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(context, "POST", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + return response + + +def vim_host_action(context, uuid, hostname, action, + timeout=constants.VIM_DEFAULT_TIMEOUT_IN_SECS): + """ + Request VIM to perform host action. + """ + + response = None + _valid_actions = [k_host.ACTION_UNLOCK, + k_host.ACTION_LOCK, + k_host.ACTION_FORCE_LOCK] + + if action not in _valid_actions: + LOG.error("Unrecognized vim_host_action=%s" % action) + return response + + LOG.warn("vim_host_action hostname=%s, action=%s" % (hostname, action)) + + api_cmd = _get_endpoint(context) + api_cmd += "/nfvi-plugins/v1/hosts/%s" % uuid + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + api_cmd_payload = dict() + api_cmd_payload['uuid'] = uuid + api_cmd_payload['hostname'] = hostname + api_cmd_payload['action'] = action + + LOG.warn("vim_host_action hostname=%s, action=%s api_cmd=%s " + "headers=%s payload=%s" % + (hostname, action, api_cmd, api_cmd_headers, api_cmd_payload)) + + response = rest_api_request(context, "PATCH", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + return response + + +def vim_host_delete(context, uuid, hostname, + timeout=constants.VIM_DEFAULT_TIMEOUT_IN_SECS): + """ + Asks VIM to delete a host + """ + + api_cmd = _get_endpoint(context) + api_cmd += "/nfvi-plugins/v1/hosts/%s" % uuid + + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + api_cmd_payload = dict() + api_cmd_payload['uuid'] = uuid + api_cmd_payload['hostname'] = hostname + api_cmd_payload['action'] = 'delete' + + response = rest_api_request(context, "DELETE", api_cmd, + api_cmd_headers, + json.dumps(api_cmd_payload), + timeout=timeout) + return response + + +def vim_host_get_instances(context, uuid, hostname, + timeout=constants.VIM_DEFAULT_TIMEOUT_IN_SECS): + """ + Returns instance counts for a given host + """ + + response = None + + api_cmd = _get_endpoint(context) + api_cmd += "/nfvi-plugins/v1/hosts" + api_cmd_headers = dict() + api_cmd_headers['Content-type'] = "application/json" + api_cmd_headers['User-Agent'] = "inventory/1.0" + + api_cmd_payload = dict() + api_cmd_payload['uuid'] = uuid + api_cmd_payload['hostname'] = hostname + + response = rest_api_request(context, "GET", api_cmd, api_cmd_headers, + json.dumps(api_cmd_payload), timeout) + return response diff --git a/inventory/inventory/inventory/conductor/__init__.py b/inventory/inventory/inventory/conductor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/conductor/base_manager.py b/inventory/inventory/inventory/conductor/base_manager.py new file mode 100644 index 00000000..404685ef --- /dev/null +++ b/inventory/inventory/inventory/conductor/base_manager.py @@ -0,0 +1,118 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +"""Base agent manager functionality.""" + +import inspect + +import futurist +from futurist import periodics +from futurist import rejection +from oslo_config import cfg +from oslo_log import log + +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.db import api as dbapi + +LOG = log.getLogger(__name__) + + +class BaseConductorManager(object): + + def __init__(self, host, topic): + super(BaseConductorManager, self).__init__() + if not host: + host = cfg.CONF.host + self.host = host + self.topic = topic + self._started = False + + def init_host(self, admin_context=None): + """Initialize the conductor host. + + :param admin_context: the admin context to pass to periodic tasks. + :raises: RuntimeError when conductor is already running. + """ + if self._started: + raise RuntimeError(_('Attempt to start an already running ' + 'conductor manager')) + + self.dbapi = dbapi.get_instance() + + rejection_func = rejection.reject_when_reached(64) + # CONF.conductor.workers_pool_size) + self._executor = futurist.GreenThreadPoolExecutor( + 64, check_and_reject=rejection_func) + """Executor for performing tasks async.""" + + # Collect driver-specific periodic tasks. + # Conductor periodic tasks accept context argument, + LOG.info('Collecting periodic tasks') + self._periodic_task_callables = [] + self._collect_periodic_tasks(self, (admin_context,)) + + self._periodic_tasks = periodics.PeriodicWorker( + self._periodic_task_callables, + executor_factory=periodics.ExistingExecutor(self._executor)) + + # Start periodic tasks + self._periodic_tasks_worker = self._executor.submit( + self._periodic_tasks.start, allow_empty=True) + self._periodic_tasks_worker.add_done_callback( + self._on_periodic_tasks_stop) + + self._started = True + + def del_host(self, deregister=True): + # Conductor deregistration fails if called on non-initialized + # conductor (e.g. when rpc server is unreachable). + if not hasattr(self, 'conductor'): + return + + self._periodic_tasks.stop() + self._periodic_tasks.wait() + self._executor.shutdown(wait=True) + self._started = False + + def _collect_periodic_tasks(self, obj, args): + """Collect periodic tasks from a given object. + + Populates self._periodic_task_callables with tuples + (callable, args, kwargs). + + :param obj: object containing periodic tasks as methods + :param args: tuple with arguments to pass to every task + """ + for name, member in inspect.getmembers(obj): + if periodics.is_periodic(member): + LOG.debug('Found periodic task %(owner)s.%(member)s', + {'owner': obj.__class__.__name__, + 'member': name}) + self._periodic_task_callables.append((member, args, {})) + + def _on_periodic_tasks_stop(self, fut): + try: + fut.result() + except Exception as exc: + LOG.critical('Periodic tasks worker has failed: %s', exc) + else: + LOG.info('Successfully shut down periodic tasks') + + def _spawn_worker(self, func, *args, **kwargs): + + """Create a greenthread to run func(*args, **kwargs). + + Spawns a greenthread if there are free slots in pool, otherwise raises + exception. Execution control returns immediately to the caller. + + :returns: Future object. + :raises: NoFreeConductorWorker if worker pool is currently full. + + """ + try: + return self._executor.submit(func, *args, **kwargs) + except futurist.RejectedSubmission: + raise exception.NoFreeConductorWorker() diff --git a/inventory/inventory/inventory/conductor/manager.py b/inventory/inventory/inventory/conductor/manager.py new file mode 100644 index 00000000..9263ff8a --- /dev/null +++ b/inventory/inventory/inventory/conductor/manager.py @@ -0,0 +1,1946 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# Copyright 2013 International Business Machines Corporation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""Conduct all activity related Inventory. + +A single instance of :py:class:`inventory.conductor.manager.ConductorManager` +is created within the inventory-conductor process, and is responsible for +performing actions for hosts managed by inventory. + +Commands are received via RPC calls. +""" + +import grp +import keyring +import os +import oslo_messaging as messaging +import pwd +import socket +import subprocess +import tsconfig.tsconfig as tsc + +from fm_api import constants as fm_constants +from fm_api import fm_api +from futurist import periodics +from inventory.agent import rpcapi as agent_rpcapi +from inventory.api.controllers.v1 import cpu_utils +from inventory.api.controllers.v1 import utils +from inventory.common import constants +from inventory.common import exception +from inventory.common import fm +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import k_lldp +from inventory.common import mtce_api +from inventory.common import rpc as inventory_oslo_rpc +from inventory.common import utils as cutils +from inventory.conductor import base_manager +from inventory.conductor import openstack +from inventory.db import api as dbapi +from inventory import objects +from inventory.systemconfig import plugin as systemconfig_plugin +from netaddr import IPAddress +from netaddr import IPNetwork +from oslo_config import cfg +from oslo_log import log + +MANAGER_TOPIC = 'inventory.conductor_manager' + +LOG = log.getLogger(__name__) + +conductor_opts = [ + cfg.StrOpt('api_url', + default=None, + help=('Url of Inventory API service. If not set Inventory can ' + 'get current value from Keystone service catalog.')), + cfg.IntOpt('audit_interval', + default=60, + help='Interval to run conductor audit'), +] + +CONF = cfg.CONF +CONF.register_opts(conductor_opts, 'conductor') +MTC_ADDRESS = 'localhost' +MTC_PORT = 2112 + + +class ConductorManager(base_manager.BaseConductorManager): + """Inventory Conductor service main class.""" + + # Must be in sync with rpcapi.ConductorAPI's + RPC_API_VERSION = '1.0' + my_host_id = None + + target = messaging.Target(version=RPC_API_VERSION) + + def __init__(self, host, topic): + super(ConductorManager, self).__init__(host, topic) + self.dbapi = None + self.fm_api = None + self.fm_log = None + self.sc_op = None + + self._openstack = None + self._api_token = None + self._mtc_address = MTC_ADDRESS + self._mtc_port = MTC_PORT + + def start(self): + self._start() + LOG.info("Start inventory-conductor") + + def init_host(self, admin_context=None): + super(ConductorManager, self).init_host(admin_context) + self._start(admin_context) + + def del_host(self, deregister=True): + return + + def _start(self, context=None): + self.dbapi = dbapi.get_instance() + self.fm_api = fm_api.FaultAPIs() + self.fm_log = fm.FmCustomerLog() + self.sc_op = systemconfig_plugin.SystemConfigPlugin( + invoke_kwds={'context': context}) + self._openstack = openstack.OpenStackOperator(self.dbapi) + + # create /var/run/inventory if required. On DOR, the manifests + # may not run to create this volatile directory. + self._create_volatile_dir() + + system = self._populate_default_system(context) + + inventory_oslo_rpc.init(cfg.CONF) + LOG.info("inventory-conductor start system=%s" % system.as_dict()) + + def periodic_tasks(self, context, raise_on_error=False): + """Periodic tasks are run at pre-specified intervals. """ + return self.run_periodic_tasks(context, raise_on_error=raise_on_error) + + @periodics.periodic(spacing=CONF.conductor.audit_interval) + def _conductor_audit(self, context): + # periodically, perform audit of inventory + LOG.info("Inventory Conductor running periodic audit task.") + + system = self._populate_default_system(context) + LOG.info("Inventory Conductor from systemconfig system=%s" % + system.as_dict()) + + hosts = objects.Host.list(context) + + for host in hosts: + self._audit_install_states(host) + + if not host.personality: + continue + # audit of configured hosts + self._audit_host_action(host) + + LOG.debug("Inventory Conductor audited hosts=%s" % hosts) + + @staticmethod + def _create_volatile_dir(): + """Create the volatile directory required for inventory service""" + if not os.path.isdir(constants.INVENTORY_LOCK_PATH): + try: + uid = pwd.getpwnam(constants.INVENTORY_USERNAME).pw_uid + gid = grp.getgrnam(constants.INVENTORY_GRPNAME).gr_gid + os.makedirs(constants.INVENTORY_LOCK_PATH) + os.chown(constants.INVENTORY_LOCK_PATH, uid, gid) + LOG.info("Created directory=%s" % + constants.INVENTORY_LOCK_PATH) + except OSError as e: + LOG.exception("makedir %s OSError=%s encountered" % + (constants.INVENTORY_LOCK_PATH, e)) + pass + + def _populate_default_system(self, context): + """Populate the default system tables""" + + try: + system = self.dbapi.system_get_one() + # TODO(sc) return system # system already configured + except exception.NotFound: + pass # create default system + + # Get the system from systemconfig + system = self.sc_op.system_get_one() + LOG.info("system retrieved from systemconfig=%s" % system.as_dict()) + + if not system: + # The audit will need to populate system + return + + values = { + 'uuid': system.uuid, + 'name': system.name, + 'system_mode': system.system_mode, + 'region_name': system.region_name, + 'software_version': cutils.get_sw_version(), + 'capabilities': {}} + + try: + system = self.dbapi.system_create(values) + except exception.SystemAlreadyExists: + system = self.dbapi.system_update(system.uuid, values) + + return system + + def _using_static_ip(self, ihost, personality=None, hostname=None): + using_static = False + if ihost: + ipersonality = ihost['personality'] + ihostname = ihost['hostname'] or "" + else: + ipersonality = personality + ihostname = hostname or "" + + if ipersonality and ipersonality == k_host.CONTROLLER: + using_static = True + elif ipersonality and ipersonality == k_host.STORAGE: + # only storage-0 and storage-1 have static (later storage-2) + if (ihostname[:len(k_host.STORAGE_0_HOSTNAME)] in + [k_host.STORAGE_0_HOSTNAME, + k_host.STORAGE_1_HOSTNAME]): + using_static = True + + return using_static + + def handle_dhcp_lease(self, context, tags, mac, ip_address, cid=None): + """Synchronously, have a conductor handle a DHCP lease update. + + Handling depends on the interface: + - management interface: do nothing + - infrastructure interface: do nothing + - pxeboot interface: create i_host + + :param cid: + :param context: request context. + :param tags: specifies the interface type (mgmt or infra) + :param mac: MAC for the lease + :param ip_address: IP address for the lease + """ + + LOG.info("receiving dhcp_lease: %s %s %s %s %s" % + (context, tags, mac, ip_address, cid)) + # Get the first field from the tags + first_tag = tags.split()[0] + + if 'pxeboot' == first_tag: + mgmt_network = \ + self.sc_op.network_get_by_type( + constants.NETWORK_TYPE_MGMT) + if not mgmt_network.dynamic: + return + + # This is a DHCP lease for a node on the pxeboot network + # Create the ihost (if necessary). + ihost_dict = {'mgmt_mac': mac} + self.create_host(context, ihost_dict, reason='dhcp pxeboot') + + def handle_dhcp_lease_from_clone(self, context, mac): + """Handle dhcp request from a cloned controller-1. + If MAC address in DB is still set to well known + clone label, then this is the first boot of the + other controller. Real MAC address from PXE request + is updated in the DB. + """ + controller_hosts = \ + self.dbapi.host_get_by_personality(k_host.CONTROLLER) + for host in controller_hosts: + if (constants.CLONE_ISO_MAC in host.mgmt_mac and + host.personality == k_host.CONTROLLER and + host.administrative == k_host.ADMIN_LOCKED): + LOG.info("create_host (clone): Host found: {}:{}:{}->{}" + .format(host.hostname, host.personality, + host.mgmt_mac, mac)) + values = {'mgmt_mac': mac} + self.dbapi.host_update(host.uuid, values) + host.mgmt_mac = mac + self._configure_controller_host(context, host) + if host.personality and host.hostname: + ihost_mtc = host.as_dict() + ihost_mtc['operation'] = 'modify' + ihost_mtc = cutils.removekeys_nonmtce(ihost_mtc) + mtce_api.host_modify( + self._api_token, self._mtc_address, + self._mtc_port, ihost_mtc, + constants.MTC_DEFAULT_TIMEOUT_IN_SECS) + return host + return None + + def create_host(self, context, values, reason=None): + """Create an ihost with the supplied data. + + This method allows an ihost to be created. + + :param reason: + :param context: an admin context + :param values: initial values for new ihost object + :returns: updated ihost object, including all fields. + """ + + if 'mgmt_mac' not in values: + raise exception.InventoryException(_( + "Invalid method call: create_host requires mgmt_mac.")) + + try: + mgmt_update_required = False + mac = values['mgmt_mac'] + mac = mac.rstrip() + mac = cutils.validate_and_normalize_mac(mac) + ihost = self.dbapi.host_get_by_mgmt_mac(mac) + LOG.info("Not creating ihost for mac: %s because it " + "already exists with uuid: %s" % (values['mgmt_mac'], + ihost['uuid'])) + mgmt_ip = values.get('mgmt_ip') or "" + + if mgmt_ip and not ihost.mgmt_ip: + LOG.info("%s create_host setting mgmt_ip to %s" % + (ihost.uuid, mgmt_ip)) + mgmt_update_required = True + elif mgmt_ip and ihost.mgmt_ip and \ + (ihost.mgmt_ip.strip() != mgmt_ip.strip()): + # Changing the management IP on an already configured + # host should not occur nor be allowed. + LOG.error("DANGER %s create_host mgmt_ip dnsmasq change " + "detected from %s to %s." % + (ihost.uuid, ihost.mgmt_ip, mgmt_ip)) + + if mgmt_update_required: + ihost = self.dbapi.host_update(ihost.uuid, values) + + if ihost.personality and ihost.hostname: + ihost_mtc = ihost.as_dict() + ihost_mtc['operation'] = 'modify' + ihost_mtc = cutils.removekeys_nonmtce(ihost_mtc) + LOG.info("%s create_host update mtce %s " % + (ihost.hostname, ihost_mtc)) + mtce_api.host_modify( + self._api_token, self._mtc_address, self._mtc_port, + ihost_mtc, + constants.MTC_DEFAULT_TIMEOUT_IN_SECS) + + return ihost + except exception.HostNotFound: + # If host is not found, check if this is cloning scenario. + # If yes, update management MAC in the DB and create PXE config. + clone_host = self.handle_dhcp_lease_from_clone(context, mac) + if clone_host: + return clone_host + + # assign default system + system = self.dbapi.system_get_one() + values.update({'system_id': system.id}) + values.update({k_host.HOST_ACTION_STATE: + k_host.HAS_REINSTALLING}) + + # get tboot value from the active controller + active_controller = None + hosts = self.dbapi.host_get_by_personality(k_host.CONTROLLER) + for h in hosts: + if utils.is_host_active_controller(h): + active_controller = h + break + + if active_controller is not None: + tboot_value = active_controller.get('tboot') + if tboot_value is not None: + values.update({'tboot': tboot_value}) + + host = objects.Host(context, **values).create() + + # A host is being created, generate discovery log. + self._log_host_create(host, reason) + + ihost_id = host.get('uuid') + LOG.info("RPC create_host called and created ihost %s." % ihost_id) + + return host + + def update_host(self, context, ihost_obj): + """Update an ihost with the supplied data. + + This method allows an ihost to be updated. + + :param context: an admin context + :param ihost_obj: a changed (but not saved) ihost object + :returns: updated ihost object, including all fields. + """ + + delta = ihost_obj.obj_what_changed() + if ('id' in delta) or ('uuid' in delta): + raise exception.InventoryException(_( + "Invalid method call: update_host cannot change id or uuid ")) + + ihost_obj.save(context) + return ihost_obj + + def _dnsmasq_host_entry_to_string(self, ip_addr, hostname, + mac_addr=None, cid=None): + if IPNetwork(ip_addr).version == constants.IPV6_FAMILY: + ip_addr = "[%s]" % ip_addr + if cid: + line = "id:%s,%s,%s,1d\n" % (cid, hostname, ip_addr) + elif mac_addr: + line = "%s,%s,%s,1d\n" % (mac_addr, hostname, ip_addr) + else: + line = "%s,%s\n" % (hostname, ip_addr) + return line + + def _dnsmasq_addn_host_entry_to_string(self, ip_addr, hostname, + aliases=[]): + line = "%s %s" % (ip_addr, hostname) + for alias in aliases: + line = "%s %s" % (line, alias) + line = "%s\n" % line + return line + + def get_my_host_id(self): + if not ConductorManager.my_host_id: + local_hostname = socket.gethostname() + controller = self.dbapi.host_get(local_hostname) + ConductorManager.my_host_id = controller['id'] + return ConductorManager.my_host_id + + def get_dhcp_server_duid(self): + """Retrieves the server DUID from the local DHCP server lease file.""" + lease_filename = tsc.CONFIG_PATH + 'dnsmasq.leases' + with open(lease_filename, 'r') as lease_file: + for columns in (line.strip().split() for line in lease_file): + if len(columns) != 2: + continue + keyword, value = columns + if keyword.lower() == "duid": + return value + + def _dhcp_release(self, interface, ip_address, mac_address, cid=None): + """Release a given DHCP lease""" + params = [interface, ip_address, mac_address] + if cid: + params += [cid] + if IPAddress(ip_address).version == 6: + params = ["--ip", ip_address, + "--iface", interface, + "--server-id", self.get_dhcp_server_duid(), + "--client-id", cid, + "--iaid", str(cutils.get_dhcp_client_iaid(mac_address))] + LOG.warning("Invoking dhcp_release6 for {}".format(params)) + subprocess.call(["dhcp_release6"] + params) + else: + LOG.warning("Invoking dhcp_release for {}".format(params)) + subprocess.call(["dhcp_release"] + params) + + def _find_networktype_for_address(self, ip_address): + LOG.info("SC to be queried from systemconfig") + # TODO(sc) query from systemconfig + + def _find_local_interface_name(self, network_type): + """Lookup the local interface name for a given network type.""" + host_id = self.get_my_host_id() + interface_list = self.dbapi.iinterface_get_all(host_id, expunge=True) + ifaces = dict((i['ifname'], i) for i in interface_list) + port_list = self.dbapi.port_get_all(host_id) + ports = dict((p['interface_id'], p) for p in port_list) + for interface in interface_list: + if interface.networktype == network_type: + return cutils.get_interface_os_ifname(interface, ifaces, ports) + + def _find_local_mgmt_interface_vlan_id(self): + """Lookup the local interface name for a given network type.""" + host_id = self.get_my_host_id() + interface_list = self.dbapi.iinterface_get_all(host_id, expunge=True) + for interface in interface_list: + if interface.networktype == constants.NETWORK_TYPE_MGMT: + if 'vlan_id' not in interface: + return 0 + else: + return interface['vlan_id'] + + def _remove_leases_by_mac_address(self, mac_address): + """Remove any leases that were added without a CID that we were not + able to delete. This is specifically looking for leases on the pxeboot + network that may still be present but will also handle the unlikely + event of deleting an old host during an upgrade. Hosts on previous + releases did not register a CID on the mgmt interface. + """ + lease_filename = tsc.CONFIG_PATH + 'dnsmasq.leases' + try: + with open(lease_filename, 'r') as lease_file: + for columns in (line.strip().split() for line in lease_file): + if len(columns) != 5: + continue + timestamp, address, ip_address, hostname, cid = columns + if address != mac_address: + continue + network_type = self._find_networktype_for_address( + ip_address) + if not network_type: + # Not one of our managed networks + LOG.warning("Lease for unknown network found in " + "dnsmasq.leases file: {}".format(columns)) + continue + interface_name = self._find_local_interface_name( + network_type + ) + self._dhcp_release(interface_name, ip_address, mac_address) + except Exception as e: + LOG.error("Failed to remove leases for %s: %s" % (mac_address, + str(e))) + + def configure_host(self, context, host_obj, + do_compute_apply=False): + """Configure a host. + + :param context: an admin context + :param host_obj: the host object + :param do_compute_apply: configure the compute subfunctions of the host + """ + + LOG.info("rpc conductor configure_host %s" % host_obj.uuid) + + # Request systemconfig plugin to configure_host + sc_host = self.sc_op.host_configure( + host_uuid=host_obj.uuid, + do_compute_apply=do_compute_apply) + + LOG.info("sc_op sc_host=%s" % sc_host) + + if sc_host: + return sc_host.as_dict() + + def unconfigure_host(self, context, host_obj): + """Unconfigure a host. + + :param context: an admin context. + :param host_obj: a host object. + """ + LOG.info("unconfigure_host %s." % host_obj.uuid) + + # Request systemconfig plugin to unconfigure_host + self.sc_op.host_unconfigure(host_obj.uuid) + + def port_update_by_host(self, context, + host_uuid, inic_dict_array): + """Create iports for an ihost with the supplied data. + + This method allows records for iports for ihost to be created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param inic_dict_array: initial values for iport objects + :returns: pass or fail + """ + + LOG.debug("Entering port_update_by_host %s %s" % + (host_uuid, inic_dict_array)) + host_uuid.strip() + try: + ihost = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + + for inic in inic_dict_array: + LOG.info("Processing inic %s" % inic) + bootp = None + port = None + # ignore port if no MAC address present, this will + # occur for data port after they are configured via DPDK driver + if not inic['mac']: + continue + + try: + inic_dict = {'host_id': ihost['id']} + inic_dict.update(inic) + if cutils.is_valid_mac(inic['mac']): + # Is this the port that the management interface is on? + if inic['mac'].strip() == ihost['mgmt_mac'].strip(): + # SKIP auto create management/pxeboot network + # was for all nodes but the active controller + bootp = 'True' + inic_dict.update({'bootp': bootp}) + + try: + LOG.debug("Attempting to create new port %s on host %s" % + (inic_dict, ihost['id'])) + + port = self.dbapi.ethernet_port_get_by_mac(inic['mac']) + # update existing port with updated attributes + try: + port_dict = { + 'sriov_totalvfs': inic['sriov_totalvfs'], + 'sriov_numvfs': inic['sriov_numvfs'], + 'sriov_vfs_pci_address': + inic['sriov_vfs_pci_address'], + 'driver': inic['driver'], + 'dpdksupport': inic['dpdksupport'], + 'speed': inic['speed'], + } + + LOG.info("port %s update attr: %s" % + (port.uuid, port_dict)) + self.dbapi.ethernet_port_update(port.uuid, port_dict) + except Exception: + LOG.exception("Failed to update port %s" % inic['mac']) + pass + + except Exception: + # adjust for field naming differences between the NIC + # dictionary returned by the agent and the Port model + port_dict = inic_dict.copy() + port_dict['name'] = port_dict.pop('pname', None) + port_dict['namedisplay'] = port_dict.pop('pnamedisplay', + None) + + LOG.info("Attempting to create new port %s " + "on host %s" % (inic_dict, ihost.uuid)) + port = self.dbapi.ethernet_port_create( + ihost.uuid, port_dict) + + except exception.HostNotFound: + raise exception.InventoryException( + _("Invalid host_uuid: host not found: %s") % + host_uuid) + + except Exception: + pass + + if ihost.invprovision not in [k_host.PROVISIONED, + k_host.PROVISIONING]: + value = {'invprovision': k_host.UNPROVISIONED} + self.dbapi.host_update(host_uuid, value) + + def lldp_tlv_dict(self, agent_neighbour_dict): + tlv_dict = {} + for k, v in agent_neighbour_dict.iteritems(): + if v is not None and k in k_lldp.LLDP_TLV_VALID_LIST: + tlv_dict.update({k: v}) + return tlv_dict + + def lldp_agent_tlv_update(self, tlv_dict, agent): + tlv_update_list = [] + tlv_create_list = [] + agent_id = agent['id'] + agent_uuid = agent['uuid'] + + tlvs = self.dbapi.lldp_tlv_get_by_agent(agent_uuid) + for k, v in tlv_dict.iteritems(): + for tlv in tlvs: + if tlv['type'] == k: + tlv_value = tlv_dict.get(tlv['type']) + entry = {'type': tlv['type'], + 'value': tlv_value} + if tlv['value'] != tlv_value: + tlv_update_list.append(entry) + break + else: + tlv_create_list.append({'type': k, + 'value': v}) + + if tlv_update_list: + try: + tlvs = self.dbapi.lldp_tlv_update_bulk(tlv_update_list, + agentid=agent_id) + except Exception as e: + LOG.exception("Error during bulk TLV update for agent %s: %s", + agent_id, str(e)) + raise + if tlv_create_list: + try: + self.dbapi.lldp_tlv_create_bulk(tlv_create_list, + agentid=agent_id) + except Exception as e: + LOG.exception("Error during bulk TLV create for agent %s: %s", + agent_id, str(e)) + raise + + def lldp_neighbour_tlv_update(self, tlv_dict, neighbour): + tlv_update_list = [] + tlv_create_list = [] + neighbour_id = neighbour['id'] + neighbour_uuid = neighbour['uuid'] + + tlvs = self.dbapi.lldp_tlv_get_by_neighbour(neighbour_uuid) + for k, v in tlv_dict.iteritems(): + for tlv in tlvs: + if tlv['type'] == k: + tlv_value = tlv_dict.get(tlv['type']) + entry = {'type': tlv['type'], + 'value': tlv_value} + if tlv['value'] != tlv_value: + tlv_update_list.append(entry) + break + else: + tlv_create_list.append({'type': k, + 'value': v}) + + if tlv_update_list: + try: + tlvs = self.dbapi.lldp_tlv_update_bulk( + tlv_update_list, + neighbourid=neighbour_id) + except Exception as e: + LOG.exception("Error during bulk TLV update for neighbour" + "%s: %s", neighbour_id, str(e)) + raise + if tlv_create_list: + try: + self.dbapi.lldp_tlv_create_bulk(tlv_create_list, + neighbourid=neighbour_id) + except Exception as e: + LOG.exception("Error during bulk TLV create for neighbour" + "%s: %s", + neighbour_id, str(e)) + raise + + def lldp_agent_update_by_host(self, context, + host_uuid, agent_dict_array): + """Create or update lldp agents for an host with the supplied data. + + This method allows records for lldp agents for ihost to be created or + updated. + + :param context: an admin context + :param host_uuid: host uuid unique id + :param agent_dict_array: initial values for lldp agent objects + :returns: pass or fail + """ + LOG.debug("Entering lldp_agent_update_by_host %s %s" % + (host_uuid, agent_dict_array)) + host_uuid.strip() + try: + db_host = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + raise exception.InventoryException(_( + "Invalid host_uuid: %s") % host_uuid) + + try: + db_ports = self.dbapi.port_get_by_host(host_uuid) + except Exception: + raise exception.InventoryException(_( + "Error getting ports for host %s") % host_uuid) + + try: + db_agents = self.dbapi.lldp_agent_get_by_host(host_uuid) + except Exception: + raise exception.InventoryException(_( + "Error getting LLDP agents for host %s") % host_uuid) + + for agent in agent_dict_array: + port_found = None + for db_port in db_ports: + if (db_port['name'] == agent['name_or_uuid'] or + db_port['uuid'] == agent['name_or_uuid']): + port_found = db_port + break + + if not port_found: + LOG.debug("Could not find port for agent %s", + agent['name_or_uuid']) + return + + hostid = db_host['id'] + portid = db_port['id'] + + agent_found = None + for db_agent in db_agents: + if db_agent['port_id'] == portid: + agent_found = db_agent + break + + LOG.debug("Processing agent %s" % agent) + + agent_dict = {'host_id': hostid, + 'port_id': portid, + 'status': agent['status']} + update_tlv = False + try: + if not agent_found: + LOG.info("Attempting to create new LLDP agent " + "%s on host %s" % (agent_dict, hostid)) + if agent['state'] != k_lldp.LLDP_AGENT_STATE_REMOVED: + db_agent = self.dbapi.lldp_agent_create(portid, + hostid, + agent_dict) + update_tlv = True + else: + # If the agent exists, try to update some of the fields + # or remove it + agent_uuid = db_agent['uuid'] + if agent['state'] == k_lldp.LLDP_AGENT_STATE_REMOVED: + db_agent = self.dbapi.lldp_agent_destroy(agent_uuid) + else: + attr = {'status': agent['status'], + 'system_name': agent['system_name']} + db_agent = self.dbapi.lldp_agent_update(agent_uuid, + attr) + update_tlv = True + + if update_tlv: + tlv_dict = self.lldp_tlv_dict(agent) + self.lldp_agent_tlv_update(tlv_dict, db_agent) + + except exception.InvalidParameterValue: + raise exception.InventoryException(_( + "Failed to update/delete non-existing" + "lldp agent %s") % agent_uuid) + except exception.LLDPAgentExists: + raise exception.InventoryException(_( + "Failed to add LLDP agent %s. " + "Already exists") % agent_uuid) + except exception.HostNotFound: + raise exception.InventoryException(_( + "Invalid host_uuid: host not found: %s") % + host_uuid) + except exception.PortNotFound: + raise exception.InventoryException(_( + "Invalid port id: port not found: %s") % + portid) + except Exception as e: + raise exception.InventoryException(_( + "Failed to update lldp agent: %s") % e) + + def lldp_neighbour_update_by_host(self, context, + host_uuid, neighbour_dict_array): + """Create or update lldp neighbours for an ihost with the supplied data. + + This method allows records for lldp neighbours for ihost to be created + or updated. + + :param context: an admin context + :param host_uuid: host uuid unique id + :param neighbour_dict_array: initial values for lldp neighbour objects + :returns: pass or fail + """ + LOG.debug("Entering lldp_neighbour_update_by_host %s %s" % + (host_uuid, neighbour_dict_array)) + host_uuid.strip() + try: + db_host = self.dbapi.host_get(host_uuid) + except Exception: + raise exception.InventoryException(_( + "Invalid host_uuid: %s") % host_uuid) + + try: + db_ports = self.dbapi.port_get_by_host(host_uuid) + except Exception: + raise exception.InventoryException(_( + "Error getting ports for host %s") % host_uuid) + + try: + db_neighbours = self.dbapi.lldp_neighbour_get_by_host(host_uuid) + except Exception: + raise exception.InventoryException(_( + "Error getting LLDP neighbours for host %s") % host_uuid) + + reported = set([(d['msap']) for d in neighbour_dict_array]) + stale = [d for d in db_neighbours if (d['msap']) not in reported] + for neighbour in stale: + db_neighbour = self.dbapi.lldp_neighbour_destroy( + neighbour['uuid']) + + for neighbour in neighbour_dict_array: + port_found = None + for db_port in db_ports: + if (db_port['name'] == neighbour['name_or_uuid'] or + db_port['uuid'] == neighbour['name_or_uuid']): + port_found = db_port + break + + if not port_found: + LOG.debug("Could not find port for neighbour %s", + neighbour['name']) + return + + LOG.debug("Processing lldp neighbour %s" % neighbour) + + hostid = db_host['id'] + portid = db_port['id'] + msap = neighbour['msap'] + state = neighbour['state'] + + neighbour_dict = {'host_id': hostid, + 'port_id': portid, + 'msap': msap} + + neighbour_found = False + for db_neighbour in db_neighbours: + if db_neighbour['msap'] == msap: + neighbour_found = db_neighbour + break + + update_tlv = False + try: + if not neighbour_found: + LOG.info("Attempting to create new lldp neighbour " + "%r on host %s" % (neighbour_dict, hostid)) + db_neighbour = self.dbapi.lldp_neighbour_create( + portid, hostid, neighbour_dict) + update_tlv = True + else: + # If the neighbour exists, remove it if requested by + # the agent. Otherwise, trigger a TLV update. There + # are currently no neighbour attributes that need to + # be updated. + if state == k_lldp.LLDP_NEIGHBOUR_STATE_REMOVED: + db_neighbour = self.dbapi.lldp_neighbour_destroy( + db_neighbour['uuid']) + else: + update_tlv = True + if update_tlv: + tlv_dict = self.lldp_tlv_dict(neighbour) + self.lldp_neighbour_tlv_update(tlv_dict, + db_neighbour) + except exception.InvalidParameterValue: + raise exception.InventoryException(_( + "Failed to update/delete lldp neighbour. " + "Invalid parameter: %r") % tlv_dict) + except exception.LLDPNeighbourExists: + raise exception.InventoryException(_( + "Failed to add lldp neighbour %r. " + "Already exists") % neighbour_dict) + except exception.HostNotFound: + raise exception.InventoryException(_( + "Invalid host_uuid: host not found: %s") % + host_uuid) + except exception.PortNotFound: + raise exception.InventoryException( + _("Invalid port id: port not found: %s") % + portid) + except Exception as e: + raise exception.InventoryException(_( + "Couldn't update LLDP neighbour: %s") % e) + + def pci_device_update_by_host(self, context, + host_uuid, pci_device_dict_array): + """Create devices for an ihost with the supplied data. + + This method allows records for devices for ihost to be created. + + :param context: an admin context + :param host_uuid: host uuid unique id + :param pci_device_dict_array: initial values for device objects + :returns: pass or fail + """ + LOG.debug("Entering device_update_by_host %s %s" % + (host_uuid, pci_device_dict_array)) + host_uuid.strip() + try: + host = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + for pci_dev in pci_device_dict_array: + LOG.debug("Processing dev %s" % pci_dev) + try: + pci_dev_dict = {'host_id': host['id']} + pci_dev_dict.update(pci_dev) + dev_found = None + try: + dev = self.dbapi.pci_device_get(pci_dev['pciaddr'], + hostid=host['id']) + dev_found = dev + if not dev: + LOG.info("Attempting to create new device " + "%s on host %s" % (pci_dev_dict, host['id'])) + dev = self.dbapi.pci_device_create(host['id'], + pci_dev_dict) + except Exception: + LOG.info("Attempting to create new device " + "%s on host %s" % (pci_dev_dict, host['id'])) + dev = self.dbapi.pci_device_create(host['id'], + pci_dev_dict) + + # If the device exists, try to update some of the fields + if dev_found: + try: + attr = { + 'pclass_id': pci_dev['pclass_id'], + 'pvendor_id': pci_dev['pvendor_id'], + 'pdevice_id': pci_dev['pdevice_id'], + 'pclass': pci_dev['pclass'], + 'pvendor': pci_dev['pvendor'], + 'psvendor': pci_dev['psvendor'], + 'psdevice': pci_dev['psdevice'], + 'sriov_totalvfs': pci_dev['sriov_totalvfs'], + 'sriov_numvfs': pci_dev['sriov_numvfs'], + 'sriov_vfs_pci_address': + pci_dev['sriov_vfs_pci_address'], + 'driver': pci_dev['driver']} + LOG.info("attr: %s" % attr) + dev = self.dbapi.pci_device_update(dev['uuid'], attr) + except Exception: + LOG.exception("Failed to update port %s" % + dev['pciaddr']) + pass + + except exception.HostNotFound: + raise exception.InventoryException( + _("Invalid host_uuid: host not found: %s") % + host_uuid) + except Exception: + pass + + def numas_update_by_host(self, context, + host_uuid, inuma_dict_array): + """Create inumas for an ihost with the supplied data. + + This method allows records for inumas for ihost to be created. + Updates the port node_id once its available. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param inuma_dict_array: initial values for inuma objects + :returns: pass or fail + """ + + host_uuid.strip() + try: + ihost = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + + try: + # Get host numa nodes which may already be in db + mynumas = self.dbapi.node_get_by_host(host_uuid) + except exception.HostNotFound: + raise exception.InventoryException(_( + "Invalid host_uuid: host not found: %s") % host_uuid) + + mynuma_nodes = [n.numa_node for n in mynumas] + + # perform update for ports + ports = self.dbapi.ethernet_port_get_by_host(host_uuid) + for i in inuma_dict_array: + if 'numa_node' in i and i['numa_node'] in mynuma_nodes: + LOG.info("Already in db numa_node=%s mynuma_nodes=%s" % + (i['numa_node'], mynuma_nodes)) + continue + + try: + inuma_dict = {'host_id': ihost['id']} + + inuma_dict.update(i) + + inuma = self.dbapi.node_create(ihost['id'], inuma_dict) + + for port in ports: + port_node = port['numa_node'] + if port_node == -1: + port_node = 0 # special handling + + if port_node == inuma['numa_node']: + attr = {'node_id': inuma['id']} + self.dbapi.ethernet_port_update(port['uuid'], attr) + + except exception.HostNotFound: + raise exception.InventoryException( + _("Invalid host_uuid: host not found: %s") % + host_uuid) + except Exception: # this info may have been posted previously + pass + + def _get_default_platform_cpu_count(self, ihost, node, + cpu_count, hyperthreading): + """Return the initial number of reserved logical cores for platform + use. This can be overridden later by the end user. + """ + cpus = 0 + if cutils.host_has_function(ihost, k_host.COMPUTE) and node == 0: + cpus += 1 if not hyperthreading else 2 + if cutils.host_has_function(ihost, k_host.CONTROLLER): + cpus += 1 if not hyperthreading else 2 + return cpus + + def _get_default_vswitch_cpu_count(self, ihost, node, + cpu_count, hyperthreading): + """Return the initial number of reserved logical cores for vswitch use. + This can be overridden later by the end user. + """ + if cutils.host_has_function(ihost, k_host.COMPUTE) and node == 0: + physical_cores = (cpu_count / 2) if hyperthreading else cpu_count + system_mode = self.dbapi.system_get_one().system_mode + if system_mode == constants.SYSTEM_MODE_SIMPLEX: + return 1 if not hyperthreading else 2 + else: + if physical_cores > 4: + return 2 if not hyperthreading else 4 + elif physical_cores > 1: + return 1 if not hyperthreading else 2 + return 0 + + def _get_default_shared_cpu_count(self, ihost, node, + cpu_count, hyperthreading): + """Return the initial number of reserved logical cores for shared + use. This can be overridden later by the end user. + """ + return 0 + + def _sort_by_socket_and_coreid(self, icpu_dict): + """Sort a list of cpu dict objects such that lower numbered sockets + appear first and that threads of the same core are adjacent in the + list with the lowest thread number appearing first. + """ + return int(icpu_dict['numa_node']), int(icpu_dict['core']), int(icpu_dict['thread']) # noqa + + def _get_hyperthreading_enabled(self, cpu_list): + """Determine if hyperthreading is enabled based on whether any threads + exist with a threadId greater than 0 + """ + for cpu in cpu_list: + if int(cpu['thread']) > 0: + return True + return False + + def _get_node_cpu_count(self, cpu_list, node): + count = 0 + for cpu in cpu_list: + count += 1 if int(cpu['numa_node']) == node else 0 + return count + + def _get_default_cpu_functions(self, host, node, cpu_list, hyperthreading): + """Return the default list of CPU functions to be reserved for this + host on the specified numa node. + """ + functions = [] + cpu_count = self._get_node_cpu_count(cpu_list, node) + # Determine how many platform cpus need to be reserved + count = self._get_default_platform_cpu_count( + host, node, cpu_count, hyperthreading) + for i in range(0, count): + functions.append(constants.PLATFORM_FUNCTION) + # Determine how many vswitch cpus need to be reserved + count = self._get_default_vswitch_cpu_count( + host, node, cpu_count, hyperthreading) + for i in range(0, count): + functions.append(constants.VSWITCH_FUNCTION) + # Determine how many shared cpus need to be reserved + count = self._get_default_shared_cpu_count( + host, node, cpu_count, hyperthreading) + for i in range(0, count): + functions.append(constants.SHARED_FUNCTION) + # Assign the default function to the remaining cpus + for i in range(0, (cpu_count - len(functions))): + functions.append(cpu_utils.get_default_function(host)) + return functions + + def print_cpu_topology(self, hostname=None, subfunctions=None, + reference=None, + sockets=None, cores=None, threads=None): + """Print logical cpu topology table (for debug reasons). + + :param hostname: hostname + :param subfunctions: subfunctions + :param reference: reference label + :param sockets: dictionary of socket_ids, sockets[cpu_id] + :param cores: dictionary of core_ids, cores[cpu_id] + :param threads: dictionary of thread_ids, threads[cpu_id] + :returns: None + """ + if sockets is None or cores is None or threads is None: + LOG.error("print_cpu_topology: topology not defined. " + "sockets=%s, cores=%s, threads=%s" + % (sockets, cores, threads)) + return + + # calculate overall cpu topology stats + n_sockets = len(set(sockets.values())) + n_cores = len(set(cores.values())) + n_threads = len(set(threads.values())) + if n_sockets < 1 or n_cores < 1 or n_threads < 1: + LOG.error("print_cpu_topology: unexpected topology. " + "n_sockets=%d, n_cores=%d, n_threads=%d" + % (n_sockets, n_cores, n_threads)) + return + + # build each line of output + ll = '' + s = '' + c = '' + t = '' + for cpu in sorted(cores.keys()): + ll += '%3d' % cpu + s += '%3d' % sockets[cpu] + c += '%3d' % cores[cpu] + t += '%3d' % threads[cpu] + + LOG.info('Logical CPU topology: host:%s (%s), ' + 'sockets:%d, cores/socket=%d, threads/core=%d, reference:%s' + % (hostname, subfunctions, n_sockets, n_cores, n_threads, + reference)) + LOG.info('%9s : %s' % ('cpu_id', ll)) + LOG.info('%9s : %s' % ('socket_id', s)) + LOG.info('%9s : %s' % ('core_id', c)) + LOG.info('%9s : %s' % ('thread_id', t)) + + def update_cpu_config(self, context, host_uuid): + LOG.info("TODO send to systemconfig update_cpu_config") + + def cpus_update_by_host(self, context, + host_uuid, icpu_dict_array, + force_grub_update=False): + """Create cpus for an ihost with the supplied data. + + This method allows records for cpus for ihost to be created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param icpu_dict_array: initial values for cpu objects + :param force_grub_update: bool value to force grub update + :returns: pass or fail + """ + + host_uuid.strip() + try: + ihost = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + + host_id = ihost['id'] + ihost_inodes = self.dbapi.node_get_by_host(host_uuid) + + icpus = self.dbapi.cpu_get_by_host(host_uuid) + + num_cpus_dict = len(icpu_dict_array) + num_cpus_db = len(icpus) + + # Capture 'current' topology in dictionary format + cs = {} + cc = {} + ct = {} + if num_cpus_dict > 0: + for icpu in icpu_dict_array: + cpu_id = icpu.get('cpu') + cs[cpu_id] = icpu.get('numa_node') + cc[cpu_id] = icpu.get('core') + ct[cpu_id] = icpu.get('thread') + + # Capture 'previous' topology in dictionary format + ps = {} + pc = {} + pt = {} + if num_cpus_db > 0: + for icpu in icpus: + cpu_id = icpu.get('cpu') + core_id = icpu.get('core') + thread_id = icpu.get('thread') + node_id = icpu.get('node_id') + socket_id = None + for inode in ihost_inodes: + if node_id == inode.get('id'): + socket_id = inode.get('numa_node') + break + ps[cpu_id] = socket_id + pc[cpu_id] = core_id + pt[cpu_id] = thread_id + + if num_cpus_dict > 0 and num_cpus_db == 0: + self.print_cpu_topology(hostname=ihost.get('hostname'), + subfunctions=ihost.get('subfunctions'), + reference='current (initial)', + sockets=cs, cores=cc, threads=ct) + + if num_cpus_dict > 0 and num_cpus_db > 0: + LOG.debug("num_cpus_dict=%d num_cpus_db= %d. " + "icpud_dict_array= %s cpus.as_dict= %s" % + (num_cpus_dict, num_cpus_db, icpu_dict_array, icpus)) + + # Skip update if topology has not changed + if ps == cs and pc == cc and pt == ct: + self.print_cpu_topology(hostname=ihost.get('hostname'), + subfunctions=ihost.get('subfunctions'), + reference='current (unchanged)', + sockets=cs, cores=cc, threads=ct) + if ihost.administrative == k_host.ADMIN_LOCKED and \ + force_grub_update: + self.update_cpu_config(context, host_uuid) + return + + self.print_cpu_topology(hostname=ihost.get('hostname'), + subfunctions=ihost.get('subfunctions'), + reference='previous', + sockets=ps, cores=pc, threads=pt) + self.print_cpu_topology(hostname=ihost.get('hostname'), + subfunctions=ihost.get('subfunctions'), + reference='current (CHANGED)', + sockets=cs, cores=cc, threads=ct) + + # there has been an update. Delete db entries and replace. + for icpu in icpus: + self.dbapi.cpu_destroy(icpu.uuid) + + # sort the list of cpus by socket and coreid + cpu_list = sorted(icpu_dict_array, key=self._sort_by_socket_and_coreid) + + # determine if hyperthreading is enabled + hyperthreading = self._get_hyperthreading_enabled(cpu_list) + + # build the list of functions to be assigned to each cpu + functions = {} + for n in ihost_inodes: + numa_node = int(n.numa_node) + functions[numa_node] = self._get_default_cpu_functions( + ihost, numa_node, cpu_list, hyperthreading) + + for data in cpu_list: + try: + node_id = None + for n in ihost_inodes: + numa_node = int(n.numa_node) + if numa_node == int(data['numa_node']): + node_id = n['id'] + break + + cpu_dict = {'host_id': host_id, + 'node_id': node_id, + 'allocated_function': functions[numa_node].pop(0)} + + cpu_dict.update(data) + + self.dbapi.cpu_create(host_id, cpu_dict) + + except exception.HostNotFound: + raise exception.InventoryException( + _("Invalid host_uuid: host not found: %s") % + host_uuid) + except Exception: + # info may have already been posted + pass + + # if it is the first controller wait for the initial config to + # be completed + if ((utils.is_host_simplex_controller(ihost) and + os.path.isfile(tsc.INITIAL_CONFIG_COMPLETE_FLAG)) or + (not utils.is_host_simplex_controller(ihost) and + ihost.administrative == k_host.ADMIN_LOCKED)): + LOG.info("Update CPU grub config, host_uuid (%s), name (%s)" + % (host_uuid, ihost.get('hostname'))) + self.update_cpu_config(context, host_uuid) + + return + + def _get_platform_reserved_memory(self, ihost, node): + low_core = cutils.is_low_core_system(ihost, self.dbapi) + reserved = cutils.get_required_platform_reserved_memory( + ihost, node, low_core) + return {'platform_reserved_mib': reserved} if reserved else {} + + def memory_update_by_host(self, context, + host_uuid, imemory_dict_array, + force_update): + """Create or update memory for a host with the supplied data. + + This method allows records for memory for host to be created, + or updated. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param imemory_dict_array: initial values for cpu objects + :param force_update: force host memory update + :returns: pass or fail + """ + + host_uuid.strip() + try: + ihost = self.dbapi.host_get(host_uuid) + except exception.ServerNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + + if ihost['administrative'] == k_host.ADMIN_LOCKED and \ + ihost['invprovision'] == k_host.PROVISIONED and \ + not force_update: + LOG.debug("Ignore the host memory audit after the host is locked") + return + + forihostid = ihost['id'] + ihost_inodes = self.dbapi.node_get_by_host(host_uuid) + + for i in imemory_dict_array: + forinodeid = None + inode_uuid = None + for n in ihost_inodes: + numa_node = int(n.numa_node) + if numa_node == int(i['numa_node']): + forinodeid = n['id'] + inode_uuid = n['uuid'] + inode_uuid.strip() + break + else: + # not found in host_nodes, do not add memory element + continue + + mem_dict = {'forihostid': forihostid, + 'forinodeid': forinodeid} + + mem_dict.update(i) + + # Do not allow updates to the amounts of reserved memory. + mem_dict.pop('platform_reserved_mib', None) + + # numa_node is not stored against imemory table + mem_dict.pop('numa_node', None) + + # clear the pending hugepage number for unlocked nodes + if ihost.administrative == k_host.ADMIN_UNLOCKED: + mem_dict['vm_hugepages_nr_2M_pending'] = None + mem_dict['vm_hugepages_nr_1G_pending'] = None + + try: + imems = self.dbapi.memory_get_by_host_node(host_uuid, + inode_uuid) + if not imems: + # Set the amount of memory reserved for platform use. + mem_dict.update(self._get_platform_reserved_memory( + ihost, i['numa_node'])) + self.dbapi.memory_create(forihostid, mem_dict) + else: + for imem in imems: + # Include 4K pages in the displayed VM memtotal + if imem.vm_hugepages_nr_4K is not None: + vm_4K_mib = \ + (imem.vm_hugepages_nr_4K / + constants.NUM_4K_PER_MiB) + mem_dict['memtotal_mib'] += vm_4K_mib + mem_dict['memavail_mib'] += vm_4K_mib + self.dbapi.memory_update(imem['uuid'], + mem_dict) + except Exception: + # Set the amount of memory reserved for platform use. + mem_dict.update(self._get_platform_reserved_memory( + ihost, i['numa_node'])) + self.dbapi.memory_create(forihostid, mem_dict) + pass + + return + + def _get_disk_available_mib(self, disk, agent_disk_dict): + partitions = self.dbapi.partition_get_by_idisk(disk['uuid']) + + if not partitions: + LOG.debug("Disk %s has no partitions" % disk.uuid) + return agent_disk_dict['available_mib'] + + available_mib = agent_disk_dict['available_mib'] + for part in partitions: + if (part.status in + [constants.PARTITION_CREATE_IN_SVC_STATUS, + constants.PARTITION_CREATE_ON_UNLOCK_STATUS]): + available_mib = available_mib - part.size_mib + + LOG.debug("Disk available mib host - %s disk - %s av - %s" % + (disk.host_id, disk.device_node, available_mib)) + return available_mib + + def platform_update_by_host(self, context, + host_uuid, imsg_dict): + """Create or update imemory for an ihost with the supplied data. + + This method allows records for memory for ihost to be created, + or updated. + + This method is invoked on initialization once. Note, swact also + results in restart, but not of inventory-agent? + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param imsg_dict: inventory message + :returns: pass or fail + """ + + host_uuid.strip() + try: + ihost = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + + availability = imsg_dict.get('availability') + + val = {} + action_state = imsg_dict.get(k_host.HOST_ACTION_STATE) + if action_state and action_state != ihost.action_state: + LOG.info("%s updating action_state=%s" % (ihost.hostname, + action_state)) + val[k_host.HOST_ACTION_STATE] = action_state + + iscsi_initiator_name = imsg_dict.get('iscsi_initiator_name') + if (iscsi_initiator_name and + ihost.iscsi_initiator_name is None): + LOG.info("%s updating iscsi initiator=%s" % + (ihost.hostname, iscsi_initiator_name)) + val['iscsi_initiator_name'] = iscsi_initiator_name + + if val: + ihost = self.dbapi.host_update(host_uuid, val) + + if not availability: + return + + if cutils.host_has_function(ihost, k_host.COMPUTE): + if availability == k_host.VIM_SERVICES_ENABLED: + # TODO(sc) report to systemconfig platform available, it will + # independently also update with config applied + LOG.info("Need report to systemconfig iplatform available " + "for ihost=%s imsg=%s" + % (host_uuid, imsg_dict)) + elif availability == k_host.AVAILABILITY_OFFLINE: + # TODO(sc) report to systemconfig platform AVAILABILITY_OFFLINE + LOG.info("Need report iplatform not available for host=%s " + "imsg= %s" % (host_uuid, imsg_dict)) + + if ((ihost.personality == k_host.STORAGE and + ihost.hostname == k_host.STORAGE_0_HOSTNAME) or + (ihost.personality == k_host.CONTROLLER)): + # TODO(sc) report to systemconfig platform available + LOG.info("TODO report to systemconfig platform available") + + def subfunctions_update_by_host(self, context, + host_uuid, subfunctions): + """Update subfunctions for a host. + + This method allows records for subfunctions to be updated. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param subfunctions: subfunctions provided by the ihost + :returns: pass or fail + """ + host_uuid.strip() + + # Create the host entry in neutron to allow for data interfaces to + # be configured on a combined node + if (k_host.CONTROLLER in subfunctions and + k_host.COMPUTE in subfunctions): + try: + ihost = self.dbapi.host_get(host_uuid) + except exception.HostNotFound: + LOG.exception("Invalid host_uuid %s" % host_uuid) + return + + try: + neutron_host_id = \ + self._openstack.get_neutron_host_id_by_name( + context, ihost['hostname']) + if not neutron_host_id: + self._openstack.create_neutron_host(context, + host_uuid, + ihost['hostname']) + elif neutron_host_id != host_uuid: + self._openstack.delete_neutron_host(context, + neutron_host_id) + self._openstack.create_neutron_host(context, + host_uuid, + ihost['hostname']) + except Exception: # TODO(sc) Needs better exception + LOG.exception("Failed in neutron stuff") + + ihost_val = {'subfunctions': subfunctions} + self.dbapi.host_update(host_uuid, ihost_val) + + def get_host_by_macs(self, context, host_macs): + """Finds ihost db entry based upon the mac list + + This method returns an ihost if it matches a mac + + :param context: an admin context + :param host_macs: list of mac addresses + :returns: ihost object, including all fields. + """ + + ihosts = objects.Host.list(context) + + LOG.debug("Checking ihost db for macs: %s" % host_macs) + for mac in host_macs: + try: + mac = mac.rstrip() + mac = cutils.validate_and_normalize_mac(mac) + except Exception: + LOG.warn("get_host_by_macs invalid mac: %s" % mac) + continue + + for host in ihosts: + if host.mgmt_mac == mac: + LOG.info("Host found ihost db for macs: %s" % + host.hostname) + return host + LOG.debug("RPC get_host_by_macs called but found no ihost.") + + def get_host_by_hostname(self, context, hostname): + """Finds host db entry based upon the host hostname + + This method returns a host if it matches the host + hostname. + + :param context: an admin context + :param hostname: host hostname + :returns: host object, including all fields. + """ + + try: + return objects.Host.get_by_filters_one(context, + {'hostname': hostname}) + except exception.HostNotFound: + pass + + LOG.info("RPC host_get_by_hostname called but found no host.") + + def _audit_host_action(self, host): + """Audit whether the host_action needs to be terminated or escalated. + """ + + if host.administrative == k_host.ADMIN_UNLOCKED: + host_action_str = host.host_action or "" + + if (host_action_str.startswith(k_host.ACTION_FORCE_LOCK) or + host_action_str.startswith(k_host.ACTION_LOCK)): + + task_str = host.task or "" + if (('--' in host_action_str and + host_action_str.startswith( + k_host.ACTION_FORCE_LOCK)) or + ('----------' in host_action_str and + host_action_str.startswith(k_host.ACTION_LOCK))): + ihost_mtc = host.as_dict() + keepkeys = ['host_action', 'vim_progress_status'] + ihost_mtc = cutils.removekeys_nonmtce(ihost_mtc, + keepkeys) + + if host_action_str.startswith( + k_host.ACTION_FORCE_LOCK): + timeout_in_secs = 6 + ihost_mtc['operation'] = 'modify' + ihost_mtc['action'] = k_host.ACTION_FORCE_LOCK + ihost_mtc['task'] = k_host.FORCE_LOCKING + LOG.warn("host_action override %s" % + ihost_mtc) + mtce_api.host_modify( + self._api_token, self._mtc_address, self._mtc_port, + ihost_mtc, timeout_in_secs) + + # need time for FORCE_LOCK mtce to clear + if '----' in host_action_str: + host_action_str = "" + else: + host_action_str += "-" + + if (task_str.startswith(k_host.FORCE_LOCKING) or + task_str.startswith(k_host.LOCKING)): + val = {'task': "", + 'host_action': host_action_str, + 'vim_progress_status': ""} + else: + val = {'host_action': host_action_str, + 'vim_progress_status': ""} + else: + host_action_str += "-" + if (task_str.startswith(k_host.FORCE_LOCKING) or + task_str.startswith(k_host.LOCKING)): + task_str += "-" + val = {'task': task_str, + 'host_action': host_action_str} + else: + val = {'host_action': host_action_str} + + self.dbapi.host_update(host.uuid, val) + else: # Administrative locked already + task_str = host.task or "" + if (task_str.startswith(k_host.FORCE_LOCKING) or + task_str.startswith(k_host.LOCKING)): + val = {'task': ""} + self.dbapi.host_update(host.uuid, val) + + vim_progress_status_str = host.get('vim_progress_status') or "" + if (vim_progress_status_str and + (vim_progress_status_str != k_host.VIM_SERVICES_ENABLED) and + (vim_progress_status_str != k_host.VIM_SERVICES_DISABLED)): + if '..' in vim_progress_status_str: + LOG.info("Audit clearing vim_progress_status=%s" % + vim_progress_status_str) + vim_progress_status_str = "" + else: + vim_progress_status_str += ".." + + val = {'vim_progress_status': vim_progress_status_str} + self.dbapi.host_update(host.uuid, val) + + def _audit_install_states(self, host): + # A node could shutdown during it's installation and the install_state + # for example could get stuck at the value "installing". To avoid + # this situation we audit the sanity of the states by appending the + # character '+' to the states in the database. After 15 minutes of the + # states not changing, set the install_state to failed. + + # The audit's interval is 60sec + MAX_COUNT = 15 + + # Allow longer duration for booting phase + MAX_COUNT_BOOTING = 40 + + LOG.info("Auditing %s, install_state is %s", + host.hostname, host.install_state) + LOG.debug("Auditing %s, availability is %s", + host.hostname, host.availability) + + if (host.administrative == k_host.ADMIN_LOCKED and + host.install_state is not None): + + install_state = host.install_state.rstrip('+') + + if host.install_state != constants.INSTALL_STATE_FAILED: + if (install_state == constants.INSTALL_STATE_BOOTING and + host.availability != + k_host.AVAILABILITY_OFFLINE): + host.install_state = constants.INSTALL_STATE_COMPLETED + + if (install_state != constants.INSTALL_STATE_INSTALLED and + install_state != + constants.INSTALL_STATE_COMPLETED): + if (install_state == + constants.INSTALL_STATE_INSTALLING and + host.install_state_info is not None): + if host.install_state_info.count('+') >= MAX_COUNT: + LOG.info( + "Auditing %s, install_state changed from " + "'%s' to '%s'", host.hostname, + host.install_state, + constants.INSTALL_STATE_FAILED) + host.install_state = \ + constants.INSTALL_STATE_FAILED + else: + host.install_state_info += "+" + else: + if (install_state == + constants.INSTALL_STATE_BOOTING): + max_count = MAX_COUNT_BOOTING + else: + max_count = MAX_COUNT + if host.install_state.count('+') >= max_count: + LOG.info( + "Auditing %s, install_state changed from " + "'%s' to '%s'", host.hostname, + host.install_state, + constants.INSTALL_STATE_FAILED) + host.install_state = \ + constants.INSTALL_STATE_FAILED + else: + host.install_state += "+" + + # It is possible we get stuck in an installed failed state. For + # example if a node gets powered down during an install booting + # state and then powered on again. Clear it if the node is + # online. + elif (host.availability == k_host.AVAILABILITY_ONLINE and + host.install_state == constants.INSTALL_STATE_FAILED): + host.install_state = constants.INSTALL_STATE_COMPLETED + + self.dbapi.host_update(host.uuid, + {'install_state': host.install_state, + 'install_state_info': + host.install_state_info}) + + def configure_systemname(self, context, systemname): + """Configure the systemname with the supplied data. + + :param context: an admin context. + :param systemname: the systemname + """ + + LOG.debug("configure_systemname: sending systemname to agent(s)") + rpcapi = agent_rpcapi.AgentAPI() + rpcapi.configure_systemname(context, systemname=systemname) + + return + + @staticmethod + def _get_fm_entity_instance_id(host_obj): + """ + Create 'entity_instance_id' from host_obj data + """ + + entity_instance_id = "%s=%s" % (fm_constants.FM_ENTITY_TYPE_HOST, + host_obj.hostname) + return entity_instance_id + + def _log_host_create(self, host, reason=None): + """ + Create host discovery event customer log. + """ + if host.hostname: + hostid = host.hostname + else: + hostid = host.mgmt_mac + + if reason is not None: + reason_text = ("%s has been 'discovered' on the network. (%s)" % + (hostid, reason)) + else: + reason_text = ("%s has been 'discovered'." % hostid) + + # action event -> FM_ALARM_TYPE_4 = 'equipment' + # FM_ALARM_SEVERITY_CLEAR to be consistent with 200.x series Info + log_data = {'hostid': hostid, + 'event_id': fm_constants.FM_LOG_ID_HOST_DISCOVERED, + 'entity_type': fm_constants.FM_ENTITY_TYPE_HOST, + 'entity': 'host=%s.event=discovered' % hostid, + 'fm_severity': fm_constants.FM_ALARM_SEVERITY_CLEAR, + 'fm_event_type': fm_constants.FM_ALARM_TYPE_4, + 'reason_text': reason_text, + } + self.fm_log.customer_log(log_data) + + def _update_subfunctions(self, context, ihost_obj): + """Update subfunctions.""" + + ihost_obj.invprovision = k_host.PROVISIONED + ihost_obj.save(context) + + def notify_subfunctions_config(self, context, + host_uuid, ihost_notify_dict): + """ + Notify inventory of host subfunctions configuration status + """ + + subfunctions_configured = ihost_notify_dict.get( + 'subfunctions_configured') or "" + try: + ihost_obj = self.dbapi.host_get(host_uuid) + except Exception as e: + LOG.exception("notify_subfunctions_config e=%s " + "ihost=%s subfunctions=%s" % + (e, host_uuid, subfunctions_configured)) + return False + + if not subfunctions_configured: + self._update_subfunctions(context, ihost_obj) + + def _add_port_to_list(self, interface_id, networktype, port_list): + info = {} + ports = self.dbapi.port_get_all(interfaceid=interface_id) + if ports: + info['name'] = ports[0]['name'] + info['numa_node'] = ports[0]['numa_node'] + info['networktype'] = networktype + if info not in port_list: + port_list.append(info) + return port_list + + def bm_deprovision_by_host(self, context, host_uuid, ibm_msg_dict): + """Update ihost upon notification of board management controller + deprovisioning. + + This method also allows a dictionary of values to be passed in to + affort additional controls, if and as needed. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param ibm_msg_dict: values for additional controls or changes + :returns: pass or fail + """ + LOG.info("bm_deprovision_by_host=%s msg=%s" % + (host_uuid, ibm_msg_dict)) + + isensorgroups = self.dbapi.sensorgroup_get_by_host(host_uuid) + + for isensorgroup in isensorgroups: + isensors = self.dbapi.sensor_get_by_sensorgroup(isensorgroup.uuid) + for isensor in isensors: + self.dbapi.sensor_destroy(isensor.uuid) + + self.dbapi.sensorgroup_destroy(isensorgroup.uuid) + + isensors = self.dbapi.sensor_get_by_host(host_uuid) + if isensors: + LOG.info("bm_deprovision_by_host=%s Non-group sensors=%s" % + (host_uuid, isensors)) + for isensor in isensors: + self.dbapi.sensor_destroy(isensor.uuid) + + isensors = self.dbapi.sensor_get_by_host(host_uuid) + + return True + + def configure_ttys_dcd(self, context, uuid, ttys_dcd): + """Notify agent to configure the dcd with the supplied data. + + :param context: an admin context. + :param uuid: the host uuid + :param ttys_dcd: the flag to enable/disable dcd + """ + + LOG.debug("ConductorManager.configure_ttys_dcd: sending dcd update %s " + "%s to agents" % (ttys_dcd, uuid)) + rpcapi = agent_rpcapi.AgentAPI() + rpcapi.configure_ttys_dcd(context, uuid=uuid, ttys_dcd=ttys_dcd) + + def get_host_ttys_dcd(self, context, ihost_id): + """ + Retrieve the serial line carrier detect state for a given host + """ + ihost = self.dbapi.host_get(ihost_id) + if ihost: + return ihost.ttys_dcd + else: + LOG.error("Host: %s not found in database" % ihost_id) + return None + + def _get_cinder_address_name(self, network_type): + ADDRESS_FORMAT_ARGS = (k_host.CONTROLLER_HOSTNAME, + network_type) + return "%s-cinder-%s" % ADDRESS_FORMAT_ARGS + + def configure_keystore_account(self, context, service_name, + username, password): + """Synchronously, have a conductor configure a ks(keyring) account. + + Does the following tasks: + - call keyring API to create an account under a service. + + :param context: request context. + :param service_name: the keystore service. + :param username: account username + :param password: account password + """ + if not service_name.strip(): + raise exception.InventoryException(_( + "Keystore service is a blank value")) + + keyring.set_password(service_name, username, password) + + def unconfigure_keystore_account(self, context, service_name, username): + """Synchronously, have a conductor unconfigure a ks(keyring) account. + + Does the following tasks: + - call keyring API to delete an account under a service. + + :param context: request context. + :param service_name: the keystore service. + :param username: account username + """ + try: + keyring.delete_password(service_name, username) + except keyring.errors.PasswordDeleteError: + pass diff --git a/inventory/inventory/inventory/conductor/openstack.py b/inventory/inventory/inventory/conductor/openstack.py new file mode 100644 index 00000000..399db0d8 --- /dev/null +++ b/inventory/inventory/inventory/conductor/openstack.py @@ -0,0 +1,878 @@ +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +""" Inventory Openstack Utilities and helper functions.""" + +from cinderclient.v2 import client as cinder_client_v2 +from inventory.common import constants +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import k_host +from inventory.common import k_host_agg +from inventory.common.storage_backend_conf import StorageBackendConfig +from keystoneclient.auth.identity import v3 +from keystoneclient import exceptions as identity_exc +from keystoneclient import session +from keystoneclient.v3 import client as keystone_client +from neutronclient.v2_0 import client as neutron_client_v2_0 +from novaclient.v2 import client as nova_client_v2 +from oslo_concurrency import lockutils +from oslo_config import cfg +from oslo_log import log +from sqlalchemy.orm import exc + + +LOG = log.getLogger(__name__) + +keystone_opts = [ + cfg.StrOpt('auth_host', + default='controller', + help=_("Authentication host server")), + cfg.IntOpt('auth_port', + default=5000, + help=_("Authentication host port number")), + cfg.StrOpt('auth_protocol', + default='http', + help=_("Authentication protocol")), + cfg.StrOpt('admin_user', + default='admin', + help=_("Admin user")), + cfg.StrOpt('admin_password', + default='admin', # this is usually some value + help=_("Admin password"), + secret=True), + cfg.StrOpt('admin_tenant_name', + default='services', + help=_("Admin tenant name")), + cfg.StrOpt('auth_uri', + default='http://192.168.204.2:5000/', + help=_("Authentication URI")), + cfg.StrOpt('auth_url', + default='http://127.0.0.1:5000/', + help=_("Admin Authentication URI")), + cfg.StrOpt('region_name', + default='RegionOne', + help=_("Region Name")), + cfg.StrOpt('neutron_region_name', + default='RegionOne', + help=_("Neutron Region Name")), + cfg.StrOpt('cinder_region_name', + default='RegionOne', + help=_("Cinder Region Name")), + cfg.StrOpt('nova_region_name', + default='RegionOne', + help=_("Nova Region Name")), + cfg.StrOpt('username', + default='inventory', + help=_("Inventory keystone user name")), + cfg.StrOpt('password', + default='inventory', + help=_("Inventory keystone user password")), + cfg.StrOpt('project_name', + default='services', + help=_("Inventory keystone user project name")), + cfg.StrOpt('user_domain_name', + default='Default', + help=_("Inventory keystone user domain name")), + cfg.StrOpt('project_domain_name', + default='Default', + help=_("Inventory keystone user project domain name")) +] + +# Register the configuration options +cfg.CONF.register_opts(keystone_opts, "KEYSTONE_AUTHTOKEN") + + +class OpenStackOperator(object): + """Class to encapsulate OpenStack operations for Inventory""" + + def __init__(self, dbapi): + self.dbapi = dbapi + self.cinder_client = None + self.keystone_client = None + self.keystone_session = None + self.nova_client = None + self.neutron_client = None + self._neutron_extension_list = [] + self.auth_url = cfg.CONF.KEYSTONE_AUTHTOKEN.auth_url + "/v3" + + ################# + # NEUTRON + ################# + + def _get_neutronclient(self): + if not self.neutron_client: # should not cache this forever + # neutronclient doesn't yet support v3 keystone auth + # use keystoneauth.session + self.neutron_client = neutron_client_v2_0.Client( + session=self._get_keystone_session(), + auth_url=self.auth_url, + endpoint_type='internalURL', + region_name=cfg.CONF.KEYSTONE_AUTHTOKEN.neutron_region_name) + return self.neutron_client + + def get_providernetworksdict(self, pn_names=None, quiet=False): + """ + Returns names and MTU values of neutron's providernetworks + """ + pn_dict = {} + + # Call neutron + try: + pn_list = self._get_neutronclient().list_providernets().get( + 'providernets', []) + except Exception as e: + if not quiet: + LOG.error("Failed to access Neutron client") + LOG.error(e) + return pn_dict + + # Get dict + # If no names specified, will add all providenets to dict + for pn in pn_list: + if pn_names and pn['name'] not in pn_names: + continue + else: + pn_dict.update({pn['name']: pn}) + + return pn_dict + + def neutron_extension_list(self, context): + """ + Send a request to neutron to query the supported extension list. + """ + if not self._neutron_extension_list: + client = self._get_neutronclient() + extensions = client.list_extensions().get('extensions', []) + self._neutron_extension_list = [e['alias'] for e in extensions] + return self._neutron_extension_list + + def bind_interface(self, context, host_uuid, interface_uuid, + network_type, providernets, mtu, + vlans=None, test=False): + """ + Send a request to neutron to bind an interface to a set of provider + networks, and inform neutron of some key attributes of the interface + for semantic checking purposes. + + Any remote exceptions from neutron are allowed to pass-through and are + expected to be handled by the caller. + """ + client = self._get_neutronclient() + body = {'interface': {'uuid': interface_uuid, + 'providernets': providernets, + 'network_type': network_type, + 'mtu': mtu}} + if vlans: + body['interface']['vlans'] = vlans + if test: + body['interface']['test'] = True + client.host_bind_interface(host_uuid, body=body) + return True + + def unbind_interface(self, context, host_uuid, interface_uuid): + """ + Send a request to neutron to unbind an interface from a set of + provider networks. + + Any remote exceptions from neutron are allowed to pass-through and are + expected to be handled by the caller. + """ + client = self._get_neutronclient() + body = {'interface': {'uuid': interface_uuid}} + client.host_unbind_interface(host_uuid, body=body) + return True + + def get_neutron_host_id_by_name(self, context, name): + """ + Get a neutron host + """ + + client = self._get_neutronclient() + + hosts = client.list_hosts() + + if not hosts: + return "" + + for host in hosts['hosts']: + if host['name'] == name: + return host['id'] + + return "" + + def create_neutron_host(self, context, host_uuid, name, + availability='down'): + """ + Send a request to neutron to create a host + """ + client = self._get_neutronclient() + body = {'host': {'id': host_uuid, + 'name': name, + 'availability': availability + }} + client.create_host(body=body) + return True + + def delete_neutron_host(self, context, host_uuid): + """ + Delete a neutron host + """ + client = self._get_neutronclient() + + client.delete_host(host_uuid) + + return True + + ################# + # NOVA + ################# + + def _get_novaclient(self): + if not self.nova_client: # should not cache this forever + # novaclient doesn't yet support v3 keystone auth + # use keystoneauth.session + self.nova_client = nova_client_v2.Client( + session=self._get_keystone_session(), + auth_url=self.auth_url, + endpoint_type='internalURL', + direct_use=False, + region_name=cfg.CONF.KEYSTONE_AUTHTOKEN.nova_region_name) + return self.nova_client + + def try_interface_get_by_host(self, host_uuid): + try: + interfaces = self.dbapi.iinterface_get_by_ihost(host_uuid) + except exc.DetachedInstanceError: + # A rare DetachedInstanceError exception may occur, retry + LOG.exception("Detached Instance Error, retry " + "iinterface_get_by_ihost %s" % host_uuid) + interfaces = self.dbapi.iinterface_get_by_ihost(host_uuid) + + return interfaces + + @lockutils.synchronized('update_nova_local_aggregates', 'inventory-') + def update_nova_local_aggregates(self, ihost_uuid, aggregates=None): + """ + Update nova_local aggregates for a host + """ + availability_zone = None + + if not aggregates: + try: + aggregates = self._get_novaclient().aggregates.list() + except Exception: + self.nova_client = None # password may have updated + aggregates = self._get_novaclient().aggregates.list() + + nova_aggset_provider = set() + for aggregate in aggregates: + nova_aggset_provider.add(aggregate.name) + + aggset_storage = set([ + k_host_agg.HOST_AGG_NAME_LOCAL_LVM, + k_host_agg.HOST_AGG_NAME_LOCAL_IMAGE, + k_host_agg.HOST_AGG_NAME_REMOTE]) + agglist_missing = list(aggset_storage - nova_aggset_provider) + LOG.debug("AGG Storage agglist_missing = %s." % agglist_missing) + + # Only add the ones that don't exist + for agg_name in agglist_missing: + # Create the aggregate + try: + aggregate = self._get_novaclient().aggregates.create( + agg_name, availability_zone) + LOG.info("AGG-AS Storage aggregate= %s created. " % ( + aggregate)) + except Exception: + LOG.error("AGG-AS EXCEPTION Storage aggregate " + "agg_name=%s not created" % (agg_name)) + raise + + # Add the metadata + try: + if agg_name == k_host_agg.HOST_AGG_NAME_LOCAL_LVM: + metadata = {'storage': k_host_agg.HOST_AGG_META_LOCAL_LVM} + elif agg_name == k_host_agg.HOST_AGG_NAME_LOCAL_IMAGE: + metadata = \ + {'storage': k_host_agg.HOST_AGG_META_LOCAL_IMAGE} + else: + metadata = {'storage': k_host_agg.HOST_AGG_META_REMOTE} + LOG.debug("AGG-AS storage aggregate metadata = %s." % metadata) + aggregate = self._get_novaclient().aggregates.set_metadata( + aggregate.id, metadata) + except Exception: + LOG.error("AGG-AS EXCEPTION Storage aggregate " + "=%s metadata not added" % aggregate) + raise + + # refresh the aggregate list + try: + aggregates = dict([(agg.name, agg) for agg in + self._get_novaclient().aggregates.list()]) + except Exception: + self.nova_client = None # password may have updated + aggregates = dict([(agg.name, agg) for agg in + self._get_novaclient().aggregates.list()]) + + # Add the host to the local or remote aggregate group + # determine if this host is configured for local storage + host_has_lvg = False + lvg_backing = False + try: + ilvgs = self.dbapi.ilvg_get_by_ihost(ihost_uuid) + for lvg in ilvgs: + if lvg.lvm_vg_name == constants.LVG_NOVA_LOCAL and \ + lvg.vg_state == k_host.PROVISIONED: + host_has_lvg = True + lvg_backing = lvg.capabilities.get( + constants.LVG_NOVA_PARAM_BACKING) + break + else: + LOG.debug("AGG-AS Found LVG %s with state %s " + "for host %s." % ( + lvg.lvm_vg_name, + lvg.vg_state, + ihost_uuid)) + except Exception: + LOG.error("AGG-AS ilvg_get_by_ihost failed " + "for %s." % ihost_uuid) + raise + + LOG.debug("AGG-AS ihost (%s) %s in a local storage configuration." % + (ihost_uuid, + "is not" + if (lvg_backing == constants.LVG_NOVA_BACKING_REMOTE) else + "is")) + + # Select the appropriate aggregate id based on the presence of an LVG + # + agg_add_to = "" + if host_has_lvg: + agg_add_to = { + constants.LVG_NOVA_BACKING_IMAGE: + k_host_agg.HOST_AGG_NAME_LOCAL_IMAGE, + constants.LVG_NOVA_BACKING_LVM: + k_host_agg.HOST_AGG_NAME_LOCAL_LVM, + constants.LVG_NOVA_BACKING_REMOTE: + k_host_agg.HOST_AGG_NAME_REMOTE + }.get(lvg_backing) + + if not agg_add_to: + LOG.error("The nova-local LVG for host: %s has an invalid " + "instance backing: " % (ihost_uuid, agg_add_to)) + + ihost = self.dbapi.ihost_get(ihost_uuid) + for aggregate in aggregates.values(): + if aggregate.name not in aggset_storage \ + or aggregate.name == agg_add_to: + continue + if hasattr(aggregate, 'hosts') \ + and ihost.hostname in aggregate.hosts: + try: + self._get_novaclient().aggregates.remove_host( + aggregate.id, + ihost.hostname) + LOG.info("AGG-AS remove ihost = %s from aggregate = %s." % + (ihost.hostname, aggregate.name)) + except Exception: + LOG.error(("AGG-AS EXCEPTION remove ihost= %s " + "from aggregate = %s.") % ( + ihost.hostname, + aggregate.name)) + raise + else: + LOG.info("skip removing host=%s not in storage " + "aggregate id=%s" % ( + ihost.hostname, + aggregate)) + if hasattr(aggregates[agg_add_to], 'hosts') \ + and ihost.hostname in aggregates[agg_add_to].hosts: + LOG.info(("skip adding host=%s already in storage " + "aggregate id=%s") % ( + ihost.hostname, + agg_add_to)) + else: + try: + self._get_novaclient().aggregates.add_host( + aggregates[agg_add_to].id, ihost.hostname) + LOG.info("AGG-AS add ihost = %s to aggregate = %s." % ( + ihost.hostname, agg_add_to)) + except Exception: + LOG.error("AGG-AS EXCEPTION add ihost= %s to aggregate = %s." % + (ihost.hostname, agg_add_to)) + raise + + def nova_host_available(self, ihost_uuid): + """ + Perform inventory driven nova operations for an available ihost + """ + # novaclient/v3 + # + # # On unlock, check whether exists: + # 1. nova aggregate-create provider_physnet0 nova + # cs.aggregates.create(args.name, args.availability_zone) + # e.g. create(provider_physnet0, None) + # + # can query it from do_aggregate_list + # ('Name', 'Availability Zone'); anyways it doesnt + # allow duplicates on Name. can be done prior to compute nodes? + # + # # On unlock, check whether exists: metadata is a key/value pair + # 2. nova aggregate-set-metadata provider_physnet0 \ + # provider:physical_network=physnet0 + # aggregate = _find_aggregate(cs, args.aggregate) + # metadata = _extract_metadata(args) + # cs.aggregates.set_metadata(aggregate.id, metadata) + # + # This can be run mutliple times regardless. + # + # 3. nova aggregate-add-host provider_physnet0 compute-0 + # cs.aggregates.add_host(aggregate.id, args.host) + # + # Can only be after nova knows about this resource!!! + # Doesnt allow duplicates,therefore agent must trigger conductor + # to perform the function. A single sync call upon init. + # On every unlock try for about 5 minutes? or check admin state + # and skip it. it needs to try several time though or needs to + # know that nova is up and running before sending it. + # e.g. agent audit look for and transitions + # /etc/platform/.initial_config_complete + # however, it needs to do this on every unlock may update + # + # Remove aggregates from provider network - on delete of host. + # 4. nova aggregate-remove-host provider_physnet0 compute-0 + # cs.aggregates.remove_host(aggregate.id, args.host) + # + # Do we ever need to do this? + # 5. nova aggregate-delete provider_physnet0 + # cs.aggregates.delete(aggregate) + # + # report to nova host aggregate groupings once node is available + + availability_zone = None + aggregate_name_prefix = 'provider_' + ihost_providernets = [] + + ihost_aggset_provider = set() + nova_aggset_provider = set() + + # determine which providernets are on this ihost + try: + iinterfaces = self.try_interface_get_by_host(ihost_uuid) + for interface in iinterfaces: + networktypelist = [] + if interface.networktype: + networktypelist = [ + network.strip() + for network in interface['networktype'].split(",")] + if constants.NETWORK_TYPE_DATA in networktypelist: + providernets = interface.providernetworks + for providernet in providernets.split(',') \ + if providernets else []: + ihost_aggset_provider.add(aggregate_name_prefix + + providernet) + + ihost_providernets = list(ihost_aggset_provider) + except Exception: + LOG.exception("AGG iinterfaces_get failed for %s." % ihost_uuid) + + try: + aggregates = self._get_novaclient().aggregates.list() + except Exception: + self.nova_client = None # password may have updated + aggregates = self._get_novaclient().aggregates.list() + pass + + for aggregate in aggregates: + nova_aggset_provider.add(aggregate.name) + + if ihost_providernets: + agglist_missing = \ + list(ihost_aggset_provider - nova_aggset_provider) + LOG.debug("AGG agglist_missing = %s." % agglist_missing) + + for i in agglist_missing: + # 1. nova aggregate-create provider_physnet0 + # use None for the availability zone + # cs.aggregates.create(args.name, args.availability_zone) + try: + aggregate = self._get_novaclient().aggregates.create( + i, availability_zone) + aggregates.append(aggregate) + LOG.debug("AGG6 aggregate= %s. aggregates= %s" % + (aggregate, aggregates)) + except Exception: + # do not continue i, redo as potential race condition + LOG.error("AGG6 EXCEPTION aggregate i=%s, aggregates=%s" % + (i, aggregates)) + + # let it try again, so it can rebuild the aggregates list + return False + + # 2. nova aggregate-set-metadata provider_physnet0 \ + # provider:physical_network=physnet0 + # aggregate = _find_aggregate(cs, args.aggregate) + # metadata = _extract_metadata(args) + # cs.aggregates.set_metadata(aggregate.id, metadata) + try: + metadata = {} + key = 'provider:physical_network' + metadata[key] = i[9:] + + # pre-check: only add/modify if aggregate is valid + if aggregate_name_prefix + metadata[key] == aggregate.name: + LOG.debug("AGG8 aggregate metadata = %s." % metadata) + aggregate = \ + self._get_novaclient().aggregates.set_metadata( + aggregate.id, metadata) + except Exception: + LOG.error("AGG8 EXCEPTION aggregate") + pass + + # 3. nova aggregate-add-host provider_physnet0 compute-0 + # cs.aggregates.add_host(aggregate.id, args.host) + + # aggregates = self._get_novaclient().aggregates.list() + ihost = self.dbapi.ihost_get(ihost_uuid) + + for i in aggregates: + if i.name in ihost_providernets: + metadata = self._get_novaclient().aggregates.get(int(i.id)) + + nhosts = [] + if hasattr(metadata, 'hosts'): + nhosts = metadata.hosts or [] + + if ihost.hostname in nhosts: + LOG.warn("host=%s in already in aggregate id=%s" % + (ihost.hostname, i.id)) + else: + try: + metadata = \ + self._get_novaclient().aggregates.add_host( + i.id, ihost.hostname) + except Exception: + LOG.warn("AGG10 EXCEPTION aggregate id = %s " + "ihost= %s." + % (i.id, ihost.hostname)) + return False + else: + LOG.warn("AGG ihost_providernets empty %s." % ihost_uuid) + + def nova_host_offline(self, ihost_uuid): + """ + Perform inventory driven nova operations for an unavailable ihost, + such as may occur when a host is locked, since if providers + may change before being unlocked again. + """ + # novaclient/v3 + # + # # On delete, check whether exists: + # + # Remove aggregates from provider network - on delete of host. + # 4. nova aggregate-remove-host provider_physnet0 compute-0 + # cs.aggregates.remove_host(aggregate.id, args.host) + # + # Do we ever need to do this? + # 5. nova aggregate-delete provider_physnet0 + # cs.aggregates.delete(aggregate) + # + + aggregate_name_prefix = 'provider_' + ihost_providernets = [] + + ihost_aggset_provider = set() + nova_aggset_provider = set() + + # determine which providernets are on this ihost + try: + iinterfaces = self.try_interface_get_by_host(ihost_uuid) + for interface in iinterfaces: + networktypelist = [] + if interface.networktype: + networktypelist = [network.strip() for network in + interface['networktype'].split(",")] + if constants.NETWORK_TYPE_DATA in networktypelist: + providernets = interface.providernetworks + for providernet in ( + providernets.split(',') if providernets else []): + ihost_aggset_provider.add(aggregate_name_prefix + + providernet) + ihost_providernets = list(ihost_aggset_provider) + except Exception: + LOG.exception("AGG iinterfaces_get failed for %s." % ihost_uuid) + + try: + aggregates = self._get_novaclient().aggregates.list() + except Exception: + self.nova_client = None # password may have updated + aggregates = self._get_novaclient().aggregates.list() + + if ihost_providernets: + for aggregate in aggregates: + nova_aggset_provider.add(aggregate.name) + else: + LOG.debug("AGG ihost_providernets empty %s." % ihost_uuid) + + # setup the valid set of storage aggregates for host removal + aggset_storage = set([ + k_host_agg.HOST_AGG_NAME_LOCAL_LVM, + k_host_agg.HOST_AGG_NAME_LOCAL_IMAGE, + k_host_agg.HOST_AGG_NAME_REMOTE]) + + # Remove aggregates from provider network. Anything with host in list. + # 4. nova aggregate-remove-host provider_physnet0 compute-0 + # cs.aggregates.remove_host(aggregate.id, args.host) + + ihost = self.dbapi.ihost_get(ihost_uuid) + + for aggregate in aggregates: + if aggregate.name in ihost_providernets or \ + aggregate.name in aggset_storage: # or just do it for all aggs + try: + LOG.debug("AGG10 remove aggregate id = %s ihost= %s." % + (aggregate.id, ihost.hostname)) + self._get_novaclient().aggregates.remove_host( + aggregate.id, ihost.hostname) + except Exception: + LOG.debug("AGG10 EXCEPTION remove aggregate") + pass + + return True + + ################# + # Keystone + ################# + def _get_keystone_session(self): + if not self.keystone_session: + auth = v3.Password(auth_url=self.auth_url, + username=cfg.CONF.KEYSTONE_AUTHTOKEN.username, + password=cfg.CONF.KEYSTONE_AUTHTOKEN.password, + user_domain_name=cfg.CONF.KEYSTONE_AUTHTOKEN. + user_domain_name, + project_name=cfg.CONF.KEYSTONE_AUTHTOKEN. + project_name, + project_domain_name=cfg.CONF.KEYSTONE_AUTHTOKEN. + project_domain_name) + self.keystone_session = session.Session(auth=auth) + return self.keystone_session + + def _get_keystoneclient(self): + if not self.keystone_client: # should not cache this forever + self.keystone_client = keystone_client.Client( + username=cfg.CONF.KEYSTONE_AUTHTOKEN.username, + user_domain_name=cfg.CONF.KEYSTONE_AUTHTOKEN.user_domain_name, + project_name=cfg.CONF.KEYSTONE_AUTHTOKEN.project_name, + project_domain_name=cfg.CONF.KEYSTONE_AUTHTOKEN + .project_domain_name, + password=cfg.CONF.KEYSTONE_AUTHTOKEN.password, + auth_url=self.auth_url, + region_name=cfg.CONF.KEYSTONE_AUTHTOKEN.region_name) + return self.keystone_client + + def _get_identity_id(self): + try: + LOG.debug("Search service id for : (%s)" % + constants.SERVICE_TYPE_IDENTITY) + service = self._get_keystoneclient().services.find( + type=constants.SERVICE_TYPE_IDENTITY) + except identity_exc.NotFound: + LOG.error("Could not find service id for (%s)" % + constants.SERVICE_TYPE_IDENTITY) + return None + except identity_exc.NoUniqueMatch: + LOG.error("Multiple service matches found for (%s)" % + constants.SERVICE_TYPE_IDENTITY) + return None + return service.id + + ################# + # Cinder + ################# + def _get_cinder_endpoints(self): + endpoint_list = [] + try: + # get region one name from platform.conf + region1_name = get_region_name('region_1_name') + if region1_name is None: + region1_name = 'RegionOne' + service_list = self._get_keystoneclient().services.list() + for s in service_list: + if s.name.find(constants.SERVICE_TYPE_CINDER) != -1: + endpoint_list += self._get_keystoneclient().endpoints.list( + service=s, region=region1_name) + except Exception: + LOG.error("Failed to get keystone endpoints for cinder.") + return endpoint_list + + def _get_cinderclient(self): + if not self.cinder_client: + self.cinder_client = cinder_client_v2.Client( + session=self._get_keystone_session(), + auth_url=self.auth_url, + endpoint_type='internalURL', + region_name=cfg.CONF.KEYSTONE_AUTHTOKEN.cinder_region_name) + + return self.cinder_client + + def get_cinder_pools(self): + pools = {} + + # Check to see if cinder is present + # TODO(rchurch): Need to refactor with storage backend + if ((StorageBackendConfig.has_backend_configured( + self.dbapi, constants.CINDER_BACKEND_CEPH)) or + (StorageBackendConfig.has_backend_configured( + self.dbapi, constants.CINDER_BACKEND_LVM))): + try: + pools = self._get_cinderclient().pools.list(detailed=True) + except Exception as e: + LOG.error("get_cinder_pools: Failed to access " + "Cinder client: %s" % e) + + return pools + + def get_cinder_volumes(self): + volumes = [] + + # Check to see if cinder is present + # TODO(rchurch): Need to refactor with storage backend + if ((StorageBackendConfig.has_backend_configured( + self.dbapi, constants.CINDER_BACKEND_CEPH)) or + (StorageBackendConfig.has_backend_configured( + self.dbapi, constants.CINDER_BACKEND_LVM))): + search_opts = { + 'all_tenants': 1 + } + try: + volumes = self._get_cinderclient().volumes.list( + search_opts=search_opts) + except Exception as e: + LOG.error("get_cinder_volumes: Failed to access " + "Cinder client: %s" % e) + + return volumes + + def get_cinder_services(self): + service_list = [] + + # Check to see if cinder is present + # TODO(rchurch): Need to refactor with storage backend + if ((StorageBackendConfig.has_backend_configured( + self.dbapi, constants.CINDER_BACKEND_CEPH)) or + (StorageBackendConfig.has_backend_configured( + self.dbapi, constants.CINDER_BACKEND_LVM))): + try: + service_list = self._get_cinderclient().services.list() + except Exception as e: + LOG.error("get_cinder_services:Failed to access " + "Cinder client: %s" % e) + + return service_list + + def get_cinder_volume_types(self): + """Obtain the current list of volume types.""" + volume_types_list = [] + + if StorageBackendConfig.is_service_enabled(self.dbapi, + constants.SB_SVC_CINDER, + filter_shared=True): + try: + volume_types_list = \ + self._get_cinderclient().volume_types.list() + except Exception as e: + LOG.error("get_cinder_volume_types: Failed to access " + "Cinder client: %s" % e) + + return volume_types_list + + ######################### + # Primary Region Inventory + # Region specific methods + ######################### + def _get_primary_cgtsclient(self): + # import the module in the function that uses it + # as the cgtsclient is only installed on the controllers + from cgtsclient.v1 import client as cgts_client + # get region one name from platform.conf + region1_name = get_region_name('region_1_name') + if region1_name is None: + region1_name = 'RegionOne' + auth_ref = self._get_keystoneclient().auth_ref + if auth_ref is None: + raise exception.InventoryException( + _("Unable to get auth ref from keystone client")) + auth_token = auth_ref.service_catalog.get_token() + endpoint = (auth_ref.service_catalog. + get_endpoints(service_type='platform', + endpoint_type='internal', + region_name=region1_name)) + endpoint = endpoint['platform'][0] + version = 1 + return cgts_client.Client(version=version, + endpoint=endpoint['url'], + auth_url=self.auth_url, + token=auth_token['id']) + + def get_ceph_mon_info(self): + ceph_mon_info = dict() + try: + cgtsclient = self._get_primary_cgtsclient() + clusters = cgtsclient.cluster.list() + if clusters: + ceph_mon_info['cluster_id'] = clusters[0].cluster_uuid + else: + LOG.error("Unable to get the cluster from the primary region") + return None + ceph_mon_ips = cgtsclient.ceph_mon.ip_addresses() + if ceph_mon_ips: + ceph_mon_info['ceph-mon-0-ip'] = ceph_mon_ips.get( + 'ceph-mon-0-ip', '') + ceph_mon_info['ceph-mon-1-ip'] = ceph_mon_ips.get( + 'ceph-mon-1-ip', '') + ceph_mon_info['ceph-mon-2-ip'] = ceph_mon_ips.get( + 'ceph-mon-2-ip', '') + else: + LOG.error("Unable to get the ceph mon IPs from the primary " + "region") + return None + except Exception as e: + LOG.error("Unable to get ceph info from the " + "primary region: %s" % e) + return None + return ceph_mon_info + + def region_has_ceph_backend(self): + ceph_present = False + try: + backend_list = \ + self._get_primary_cgtsclient().storage_backend.list() + for backend in backend_list: + if backend.backend == constants.CINDER_BACKEND_CEPH: + ceph_present = True + break + except Exception as e: + LOG.error("Unable to get storage backend list from the primary " + "region: %s" % e) + return ceph_present + + +def get_region_name(region): + # get region name from platform.conf + lines = [line.rstrip('\n') for line in + open('/etc/platform/platform.conf')] + for line in lines: + values = line.split('=') + if values[0] == region: + return values[1] + LOG.error("Unable to get %s from the platform.conf." % region) + return None diff --git a/inventory/inventory/inventory/conductor/rpcapi.py b/inventory/inventory/inventory/conductor/rpcapi.py new file mode 100644 index 00000000..ab500465 --- /dev/null +++ b/inventory/inventory/inventory/conductor/rpcapi.py @@ -0,0 +1,571 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 + +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +""" +Client side of the conductor RPC API. +""" + +from oslo_log import log +import oslo_messaging as messaging + +from inventory.common import rpc +from inventory.objects import base as objects_base + +LOG = log.getLogger(__name__) + +MANAGER_TOPIC = 'inventory.conductor_manager' + + +class ConductorAPI(object): + """Client side of the conductor RPC API. + + API version history: + + 1.0 - Initial version. + """ + + RPC_API_VERSION = '1.0' + + # The default namespace, which can be overridden in a subclass. + RPC_API_NAMESPACE = None + + def __init__(self, topic=None): + super(ConductorAPI, self).__init__() + self.topic = topic + if self.topic is None: + self.topic = MANAGER_TOPIC + target = messaging.Target(topic=self.topic, + version='1.0') + serializer = objects_base.InventoryObjectSerializer() + # release_ver = versions.RELEASE_MAPPING.get(CONF.pin_release_version) + # version_cap = (release_ver['rpc'] if release_ver + # else self.RPC_API_VERSION) + version_cap = self.RPC_API_VERSION + self.client = rpc.get_client(target, + version_cap=version_cap, + serializer=serializer) + + @staticmethod + def make_namespaced_msg(method, namespace, **kwargs): + return {'method': method, 'namespace': namespace, 'args': kwargs} + + def make_msg(self, method, **kwargs): + return self.make_namespaced_msg(method, self.RPC_API_NAMESPACE, + **kwargs) + + # This is to be in inventory? However, it'll need to know the ip_address! + def handle_dhcp_lease(self, context, tags, mac, ip_address, cid=None, + topic=None): + """Synchronously, have a conductor handle a DHCP lease update. + + Handling depends on the interface: + - management interface: creates an ihost + - infrastructure interface: just updated the dnsmasq config + + :param context: request context. + :param tags: specifies the interface type (mgmt or infra) + :param mac: MAC for the lease + :param ip_address: IP address for the lease + :param cid: Client ID for the lease + """ + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'handle_dhcp_lease', + tags=tags, + mac=mac, + ip_address=ip_address, + cid=cid) + + def create_host(self, context, values, topic=None): + """Synchronously, have a conductor create an ihost. + + Create an ihost in the database and return an object. + + :param context: request context. + :param values: dictionary with initial values for new ihost object + :returns: created ihost object, including all fields. + """ + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'create_host', + values=values) + + def update_host(self, context, ihost_obj, topic=None): + """Synchronously, have a conductor update the hosts's information. + + Update the ihost's information in the database and return an object. + + :param context: request context. + :param ihost_obj: a changed (but not saved) ihost object. + :returns: updated ihost object, including all fields. + """ + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'update_host', + ihost_obj=ihost_obj) + + def configure_host(self, context, host_obj, + do_compute_apply=False, + topic=None): + """Synchronously, have a conductor configure an ihost. + + Does the following tasks: + - invoke systemconfig to perform host configuration + - Update puppet hiera configuration files for the ihost. + - Add (or update) a host entry in the dnsmasq.conf file. + - Set up PXE configuration to run installer + + :param context: request context. + :param host_obj: an ihost object. + :param do_compute_apply: apply the newly created compute manifests. + """ + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'configure_host', + host_obj=host_obj, + do_compute_apply=do_compute_apply) + + def unconfigure_host(self, context, host_obj, topic=None): + """Synchronously, have a conductor unconfigure a host. + + Does the following tasks: + - Remove hiera config files for the host. + - Remove the host entry from the dnsmasq.conf file. + - Remove the PXE configuration + + :param context: request context. + :param host_obj: a host object. + """ + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'unconfigure_host', + host_obj=host_obj) + + def get_host_by_macs(self, context, host_macs, topic=None): + """Finds hosts db entry based upon the mac list + + This method returns a host if it matches a mac + + :param context: an admin context + :param host_macs: list of mac addresses + :returns: host object + """ + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + + return cctxt.call(context, + 'get_host_by_macs', + host_macs=host_macs) + + def get_host_by_hostname(self, context, hostname, topic=None): + """Finds host db entry based upon the ihost hostname + + This method returns an ihost if it matches the + hostname. + + :param context: an admin context + :param hostname: host hostname + :returns: host object, including all fields. + """ + + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'get_host_by_hostname', + hostname=hostname) + + def platform_update_by_host(self, context, + host_uuid, imsg_dict, topic=None): + """Create or update memory for an ihost with the supplied data. + + This method allows records for memory for ihost to be created, + or updated. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param imsg_dict: inventory message dict + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, + version='1.0') + return cctxt.call(context, + 'platform_update_by_host', + host_uuid=host_uuid, + imsg_dict=imsg_dict) + + def subfunctions_update_by_host(self, context, + host_uuid, subfunctions, topic=None): + """Create or update local volume group for an ihost with the supplied + data. + + This method allows records for a local volume group for ihost to be + created, or updated. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param subfunctions: subfunctions of the host + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'subfunctions_update_by_host', + host_uuid=host_uuid, + subfunctions=subfunctions) + + def mgmt_ip_set_by_host(self, + context, + host_uuid, + mgmt_ip, + topic=None): + """Call inventory to update host mgmt_ip (removes previous entry if + necessary) + + :param context: an admin context + :param host_uuid: ihost uuid + :param mgmt_ip: mgmt_ip + :returns: Address + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'mgmt_ip_set_by_host', + host_uuid=host_uuid, + mgmt_ip=mgmt_ip) + + def infra_ip_set_by_host(self, + context, + host_uuid, + infra_ip, topic=None): + """Call inventory to update host infra_ip (removes previous entry if + necessary) + + :param context: an admin context + :param host_uuid: ihost uuid + :param infra_ip: infra_ip + :returns: Address + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'infra_ip_set_by_host', + host_uuid=host_uuid, + infra_ip=infra_ip) + + def vim_host_add(self, context, api_token, host_uuid, + hostname, subfunctions, administrative, + operational, availability, + subfunction_oper, subfunction_avail, timeout, topic=None): + """ + Asynchronously, notify VIM of host add + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.cast(context, + 'vim_host_add', + api_token=api_token, + host_uuid=host_uuid, + hostname=hostname, + personality=subfunctions, + administrative=administrative, + operational=operational, + availability=availability, + subfunction_oper=subfunction_oper, + subfunction_avail=subfunction_avail, + timeout=timeout) + + def notify_subfunctions_config(self, context, + host_uuid, ihost_notify_dict, topic=None): + """ + Synchronously, notify inventory of host subfunctions config status + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'notify_subfunctions_config', + host_uuid=host_uuid, + ihost_notify_dict=ihost_notify_dict) + + def bm_deprovision_by_host(self, context, + host_uuid, ibm_msg_dict, topic=None): + """Update ihost upon notification of board management controller + deprovisioning. + + This method also allows a dictionary of values to be passed in to + affort additional controls, if and as needed. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param ibm_msg_dict: values for additional controls or changes + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'bm_deprovision_by_host', + host_uuid=host_uuid, + ibm_msg_dict=ibm_msg_dict) + + def configure_ttys_dcd(self, context, uuid, ttys_dcd, topic=None): + """Synchronously, have a conductor configure the dcd. + + Does the following tasks: + - sends a message to conductor + - who sends a message to all inventory agents + - who has the uuid updates dcd + + :param context: request context. + :param uuid: the host uuid + :param ttys_dcd: the flag to enable/disable dcd + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + LOG.debug("ConductorApi.configure_ttys_dcd: sending (%s %s) to " + "conductor" % (uuid, ttys_dcd)) + return cctxt.call(context, + 'configure_ttys_dcd', + uuid=uuid, + ttys_dcd=ttys_dcd) + + def get_host_ttys_dcd(self, context, ihost_id, topic=None): + """Synchronously, have a agent collect carrier detect state for this + ihost. + + :param context: request context. + :param ihost_id: id of this host + :returns: ttys_dcd. + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'get_host_ttys_dcd', + ihost_id=ihost_id) + + def port_update_by_host(self, context, + host_uuid, inic_dict_array, topic=None): + """Create iports for an ihost with the supplied data. + + This method allows records for iports for ihost to be created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param inic_dict_array: initial values for iport objects + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'port_update_by_host', + host_uuid=host_uuid, + inic_dict_array=inic_dict_array) + + def lldp_agent_update_by_host(self, context, host_uuid, agent_dict_array, + topic=None): + """Create lldp_agents for an ihost with the supplied data. + + This method allows records for lldp_agents for a host to be created. + + :param context: an admin context + :param host_uuid: host uuid unique id + :param agent_dict_array: initial values for lldp_agent objects + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'lldp_agent_update_by_host', + host_uuid=host_uuid, + agent_dict_array=agent_dict_array) + + def lldp_neighbour_update_by_host(self, context, + host_uuid, neighbour_dict_array, + topic=None): + """Create lldp_neighbours for an ihost with the supplied data. + + This method allows records for lldp_neighbours for a host to be + created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param neighbour_dict_array: initial values for lldp_neighbour objects + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call( + context, + 'lldp_neighbour_update_by_host', + host_uuid=host_uuid, + neighbour_dict_array=neighbour_dict_array) + + def pci_device_update_by_host(self, context, + host_uuid, pci_device_dict_array, + topic=None): + """Create pci_devices for an ihost with the supplied data. + + This method allows records for pci_devices for ihost to be created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param pci_device_dict_array: initial values for device objects + :returns: pass or fail + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'pci_device_update_by_host', + host_uuid=host_uuid, + pci_device_dict_array=pci_device_dict_array) + + def numas_update_by_host(self, + context, + host_uuid, + inuma_dict_array, + topic=None): + """Create inumas for an ihost with the supplied data. + + This method allows records for inumas for ihost to be created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param inuma_dict_array: initial values for inuma objects + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'numas_update_by_host', + host_uuid=host_uuid, + inuma_dict_array=inuma_dict_array) + + def cpus_update_by_host(self, + context, + host_uuid, + icpu_dict_array, + force_grub_update, + topic=None): + """Create cpus for an ihost with the supplied data. + + This method allows records for cpus for ihost to be created. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param icpu_dict_array: initial values for cpu objects + :param force_grub_update: bool value to force grub update + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'cpus_update_by_host', + host_uuid=host_uuid, + icpu_dict_array=icpu_dict_array, + force_grub_update=force_grub_update) + + def memory_update_by_host(self, context, + host_uuid, imemory_dict_array, + force_update=False, + topic=None): + """Create or update memory for an ihost with the supplied data. + + This method allows records for memory for ihost to be created, + or updated. + + :param context: an admin context + :param host_uuid: ihost uuid unique id + :param imemory_dict_array: initial values for memory objects + :param force_update: force a memory update + :returns: pass or fail + """ + + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'memory_update_by_host', + host_uuid=host_uuid, + imemory_dict_array=imemory_dict_array, + force_update=force_update) + + def update_cpu_config(self, context, topic=None): + """Synchronously, have the conductor update the cpu + configuration. + + :param context: request context. + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, 'update_cpu_config') + + def configure_keystore_account(self, context, service_name, + username, password, topic=None): + """Synchronously, have a conductor configure a ks(keyring) account. + + Does the following tasks: + - call keyring API to create an account under a service. + + :param context: request context. + :param service_name: the keystore service. + :param username: account username + :param password: account password + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'configure_keystore_account', + service_name=service_name, + username=username, password=password) + + def unconfigure_keystore_account(self, context, + service_name, username, topic=None): + """Synchronously, have a conductor unconfigure a ks(keyring) account. + + Does the following tasks: + - call keyring API to delete an account under a service. + + :param context: request context. + :param service_name: the keystore service. + :param username: account username + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'unconfigure_keystore_account', + service_name=service_name, + username=username) + + def reload_snmp_config(self, context, topic=None): + """Synchronously, have a conductor reload the SNMP configuration. + + Does the following tasks: + - sighup snmpd to reload the snmpd configuration. + + :param context: request context. + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, + 'reload_snmp_config') + + def region_has_ceph_backend(self, context, topic=None): + """ + Send a request to primary region to see if ceph backend is configured + """ + cctxt = self.client.prepare(topic=topic or self.topic, version='1.0') + return cctxt.call(context, 'region_has_ceph_backend') diff --git a/inventory/inventory/inventory/conf/__init__.py b/inventory/inventory/inventory/conf/__init__.py new file mode 100644 index 00000000..c6d440ed --- /dev/null +++ b/inventory/inventory/inventory/conf/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2016 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventory.conf import default +from oslo_config import cfg + +CONF = cfg.CONF +default.register_opts(CONF) diff --git a/inventory/inventory/inventory/conf/database.py b/inventory/inventory/inventory/conf/database.py new file mode 100644 index 00000000..b2fd16fd --- /dev/null +++ b/inventory/inventory/inventory/conf/database.py @@ -0,0 +1,31 @@ +# Copyright 2016 Intel Corporation +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from inventory.common.i18n import _ + +opts = [ + cfg.StrOpt('mysql_engine', + default='InnoDB', + help=_('MySQL engine to use.')), + cfg.StrOpt('sql_connection', + default='sqlite://', + help=_('sql connection to use.')) +] + + +def register_opts(conf): + conf.register_opts(opts, group='database') diff --git a/inventory/inventory/inventory/conf/default.py b/inventory/inventory/inventory/conf/default.py new file mode 100644 index 00000000..87b5569c --- /dev/null +++ b/inventory/inventory/inventory/conf/default.py @@ -0,0 +1,121 @@ +# Copyright 2016 Intel Corporation +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# Copyright 2013 Red Hat, Inc. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import socket +import tempfile + +from inventory.common.i18n import _ +from oslo_config import cfg + + +api_opts = [ + cfg.StrOpt( + 'auth_strategy', + default='keystone', + choices=['noauth', 'keystone'], + help=_('Authentication strategy used by inventory-api. "noauth" ' + 'should not be used in a production environment because all ' + 'authentication will be disabled.')), + cfg.BoolOpt('debug_tracebacks_in_api', + default=False, + help=_('Return server tracebacks in the API response for any ' + 'error responses. WARNING: this is insecure ' + 'and should not be used in a production environment.')), + cfg.BoolOpt('pecan_debug', + default=False, + help=_('Enable pecan debug mode. WARNING: this is insecure ' + 'and should not be used in a production environment.')), + cfg.StrOpt('default_resource_class', + help=_('Resource class to use for new nodes when no resource ' + 'class is provided in the creation request.')), +] + +exc_log_opts = [ + cfg.BoolOpt('fatal_exception_format_errors', + default=False, + help=_('Used if there is a formatting error when generating ' + 'an exception message (a programming error). If True, ' + 'raise an exception; if False, use the unformatted ' + 'message.')), +] + +# NOTE(mariojv) By default, accessing this option when it's unset will return +# None, indicating no notifications will be sent. oslo.config returns None by +# default for options without set defaults that aren't required. +notification_opts = [ + cfg.StrOpt('notification_level', + choices=[('debug', _('"debug" level')), + ('info', _('"info" level')), + ('warning', _('"warning" level')), + ('error', _('"error" level')), + ('critical', _('"critical" level'))], + help=_('Specifies the minimum level for which to send ' + 'notifications. If not set, no notifications will ' + 'be sent. The default is for this option to be unset.')) +] + +path_opts = [ + cfg.StrOpt( + 'pybasedir', + default=os.path.abspath(os.path.join(os.path.dirname(__file__), + '../')), + sample_default='/usr/lib64/python/site-packages/inventory', + help=_('Directory where the inventory python module is ' + 'installed.')), + cfg.StrOpt('bindir', + default='$pybasedir/bin', + help=_('Directory where inventory binaries are installed.')), + cfg.StrOpt('state_path', + default='$pybasedir', + help=_("Top-level directory for maintaining inventory's " + "state.")), +] + +service_opts = [ + cfg.StrOpt('host', + default=socket.getfqdn(), + sample_default='localhost', + help=_('Name of this node. This can be an opaque identifier. ' + 'It is not necessarily a hostname, FQDN, or IP address. ' + 'However, the node name must be valid within ' + 'an AMQP key, and if using ZeroMQ (will be removed in ' + 'the Stein release), a valid hostname, FQDN, ' + 'or IP address.')), +] + +utils_opts = [ + cfg.StrOpt('rootwrap_config', + default="/etc/inventory/rootwrap.conf", + help=_('Path to the rootwrap configuration file to use for ' + 'running commands as root.')), + cfg.StrOpt('tempdir', + default=tempfile.gettempdir(), + sample_default='/tmp', + help=_('Temporary working directory, default is Python temp ' + 'dir.')), +] + + +def register_opts(conf): + conf.register_opts(exc_log_opts) + conf.register_opts(notification_opts) + conf.register_opts(path_opts) + conf.register_opts(service_opts) + conf.register_opts(utils_opts) diff --git a/inventory/inventory/inventory/conf/opts.py b/inventory/inventory/inventory/conf/opts.py new file mode 100644 index 00000000..63578f74 --- /dev/null +++ b/inventory/inventory/inventory/conf/opts.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from oslo_log import log + + +def update_opt_defaults(): + log.set_defaults( + default_log_levels=[ + 'amqp=WARNING', + 'amqplib=WARNING', + 'qpid.messaging=INFO', + # TODO(therve): when bug #1685148 is fixed in oslo.messaging, we + # should be able to remove one of those 2 lines. + 'oslo_messaging=INFO', + 'oslo.messaging=INFO', + 'sqlalchemy=WARNING', + 'stevedore=INFO', + 'eventlet.wsgi.server=INFO', + 'iso8601=WARNING', + 'requests=WARNING', + 'neutronclient=WARNING', + 'urllib3.connectionpool=WARNING', + 'keystonemiddleware.auth_token=INFO', + 'keystoneauth.session=INFO', + ] + ) + +# 'glanceclient=WARNING', diff --git a/inventory/inventory/inventory/config-generator.conf b/inventory/inventory/inventory/config-generator.conf new file mode 100644 index 00000000..7ae653a3 --- /dev/null +++ b/inventory/inventory/inventory/config-generator.conf @@ -0,0 +1,18 @@ +[DEFAULT] +output_file = inventory.conf.sample +wrap_width = 79 +# namespace = inventory.api.config +namespace = inventory.agent.manager +namespace = inventory.conductor.manager +namespace = inventory.conductor.openstack +namespace = inventory.conf +namespace = inventory.default +# from setup.py +namespace = inventory.common.config +namespace = keystonemiddleware.auth_token +namespace = oslo.middleware +namespace = oslo.log +namespace = oslo.policy +namespace = oslo.db +namespace = oslo.messaging +namespace = oslo.service.service diff --git a/inventory/inventory/inventory/db/__init__.py b/inventory/inventory/inventory/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/db/api.py b/inventory/inventory/inventory/db/api.py new file mode 100644 index 00000000..8db55404 --- /dev/null +++ b/inventory/inventory/inventory/db/api.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Interface for database access. +""" + +import abc + +from oslo_config import cfg +from oslo_db import api as db_api +import six + +CONF = cfg.CONF + +_BACKEND_MAPPING = {'sqlalchemy': 'inventory.db.sqlalchemy.api'} +IMPL = db_api.DBAPI.from_config(CONF, backend_mapping=_BACKEND_MAPPING, + lazy=True) + + +def get_instance(): + """Return a DB API instance.""" + return IMPL + + +def get_engine(): + return IMPL.get_engine() + + +def get_session(): + return IMPL.get_session() + + +@six.add_metaclass(abc.ABCMeta) +class Connection(object): + """Base class for database connections.""" + + @abc.abstractmethod + def __init__(self): + """Constructor.""" + + # TODO(sc) Enforcement of required methods for db api diff --git a/inventory/inventory/inventory/db/migration.py b/inventory/inventory/inventory/db/migration.py new file mode 100644 index 00000000..eb360f51 --- /dev/null +++ b/inventory/inventory/inventory/db/migration.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# under the License. + +"""Database setup and migration commands.""" + +from inventory.db.sqlalchemy import api as db_api +import os +from oslo_config import cfg +from oslo_db import options +from stevedore import driver + +options.set_defaults(cfg.CONF) + + +_IMPL = None + +MIGRATE_REPO_PATH = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'sqlalchemy', + 'migrate_repo', +) + + +def get_backend(): + global _IMPL + if not _IMPL: + _IMPL = driver.DriverManager("inventory.database.migration_backend", + cfg.CONF.database.backend).driver + return _IMPL + + +def db_sync(version=None, engine=None): + """Migrate the database to `version` or the most recent version.""" + + if engine is None: + engine = db_api.get_engine() + return get_backend().db_sync(engine=engine, + abs_path=MIGRATE_REPO_PATH, + version=version + ) + + +def upgrade(version=None): + """Migrate the database to `version` or the most recent version.""" + return get_backend().upgrade(version) + + +def version(): + return get_backend().version() + + +def create_schema(): + return get_backend().create_schema() diff --git a/inventory/inventory/inventory/db/sqlalchemy/__init__.py b/inventory/inventory/inventory/db/sqlalchemy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/db/sqlalchemy/api.py b/inventory/inventory/inventory/db/sqlalchemy/api.py new file mode 100644 index 00000000..ff02de46 --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/api.py @@ -0,0 +1,2570 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +"""SQLAlchemy backend.""" + +import eventlet +import threading + +from oslo_config import cfg +from oslo_db import exception as db_exc +from oslo_db.sqlalchemy import enginefacade +from oslo_db.sqlalchemy import session as db_session +from oslo_db.sqlalchemy import utils as db_utils + +from oslo_log import log +from oslo_utils import uuidutils + +from sqlalchemy import inspect +from sqlalchemy import or_ +from sqlalchemy.orm.exc import MultipleResultsFound +from sqlalchemy.orm.exc import NoResultFound + +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.common import utils +from inventory.db import api +from inventory.db.sqlalchemy import models + +LOG = log.getLogger(__name__) +CONF = cfg.CONF + + +_LOCK = threading.Lock() +_FACADE = None + +context_manager = enginefacade.transaction_context() +context_manager.configure() + + +def _create_facade_lazily(): + global _LOCK + with _LOCK: + global _FACADE + if _FACADE is None: + _FACADE = db_session.EngineFacade( + CONF.database.connection, + **dict(CONF.database) + ) + return _FACADE + + +def get_engine(): + facade = _create_facade_lazily() + return facade.get_engine() + + +def get_session(**kwargs): + facade = _create_facade_lazily() + return facade.get_session(**kwargs) + + +def get_backend(): + """The backend is this module itself.""" + return Connection() + + +def _session_for_read(): + # _context = threading.local() + _context = eventlet.greenthread.getcurrent() + + return enginefacade.reader.using(_context) + + +def _session_for_write(): + _context = eventlet.greenthread.getcurrent() + + return enginefacade.writer.using(_context) + + +def _paginate_query(model, limit=None, marker=None, sort_key=None, + sort_dir=None, query=None): + if not query: + query = model_query(model) + + if not sort_key: + sort_keys = [] + elif not isinstance(sort_key, list): + sort_keys = [sort_key] + else: + sort_keys = sort_key + + if 'id' not in sort_keys: + sort_keys.append('id') + query = db_utils.paginate_query(query, model, limit, sort_keys, + marker=marker, sort_dir=sort_dir) + return query.all() + + +def model_query(model, *args, **kwargs): + """Query helper for simpler session usage. + + :param model: database model + :param session: if present, the session to use + """ + + session = kwargs.get('session') + if session: + query = session.query(model, *args) + else: + with _session_for_read() as session: + query = session.query(model, *args) + return query + + +def add_identity_filter(query, value, + use_ifname=False, + use_ipaddress=False, + use_community=False, + use_key=False, + use_name=False, + use_cname=False, + use_secname=False, + use_sensorgroupname=False, + use_sensorname=False, + use_pciaddr=False): + """Adds an identity filter to a query. + + Filters results by ID, if supplied value is a valid integer. + Otherwise attempts to filter results by UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(value): + return query.filter_by(id=value) + elif uuidutils.is_uuid_like(value): + return query.filter_by(uuid=value) + else: + if use_ifname: + return query.filter_by(ifname=value) + elif use_ipaddress: + return query.filter_by(ip_address=value) + elif use_community: + return query.filter_by(community=value) + elif use_name: + return query.filter_by(name=value) + elif use_cname: + return query.filter_by(cname=value) + elif use_secname: + return query.filter_by(secname=value) + elif use_key: + return query.filter_by(key=value) + elif use_pciaddr: + return query.filter_by(pciaddr=value) + elif use_sensorgroupname: + return query.filter_by(sensorgroupname=value) + elif use_sensorname: + return query.filter_by(sensorname=value) + else: + return query.filter_by(hostname=value) + + +def add_filter_by_many_identities(query, model, values): + """Adds an identity filter to a query for values list. + + Filters results by ID, if supplied values contain a valid integer. + Otherwise attempts to filter results by UUID. + + :param query: Initial query to add filter to. + :param model: Model for filter. + :param values: Values for filtering results by. + :return: tuple (Modified query, filter field name). + """ + if not values: + raise exception.InvalidIdentity(identity=values) + value = values[0] + if utils.is_int_like(value): + return query.filter(getattr(model, 'id').in_(values)), 'id' + elif uuidutils.is_uuid_like(value): + return query.filter(getattr(model, 'uuid').in_(values)), 'uuid' + else: + raise exception.InvalidIdentity(identity=value) + + +def add_node_filter_by_host(query, value): + if utils.is_int_like(value): + return query.filter_by(host_id=value) + else: + query = query.join(models.Hosts, + models.Nodes.host_id == models.Hosts.id) + return query.filter(models.Hosts.uuid == value) + + +def add_filter_by_host_node(query, ihostid, inodeid): + if utils.is_int_like(ihostid) and utils.is_int_like(inodeid): + return query.filter_by(host_id=ihostid, node_id=inodeid) + + if utils.is_uuid_like(ihostid) and utils.is_uuid_like(inodeid): + ihostq = model_query(models.Hosts).filter_by(uuid=ihostid).first() + inodeq = model_query(models.Nodes).filter_by(uuid=inodeid).first() + + query = query.filter_by(host_id=ihostq.id, + node_id=inodeq.id) + + return query + + +def add_cpu_filter_by_host(query, value): + if utils.is_int_like(value): + return query.filter_by(host_id=value) + else: + query = query.join(models.Hosts, + models.Cpus.host_id == models.Hosts.id) + return query.filter(models.Hosts.uuid == value) + + +def add_cpu_filter_by_host_node(query, ihostid, inodeid): + if utils.is_int_like(ihostid) and utils.is_int_like(inodeid): + return query.filter_by(host_id=ihostid, node_id=inodeid) + + # gives access to joined tables... nice to have unique col name + if utils.is_uuid_like(ihostid) and utils.is_uuid_like(inodeid): + query = query.join(models.Hosts, + models.Cpus.host_id == models.Hosts.id, + models.Nodes.host_id == models.Hosts.id) + + return query.filter(models.Hosts.uuid == ihostid, + models.Nodes.uuid == inodeid) + + LOG.error("cpu_filter_by_host_inode: No match for id int or ids uuid") + + +def add_cpu_filter_by_node(query, inodeid): + if utils.is_int_like(inodeid): + return query.filter_by(node_id=inodeid) + else: + query = query.join(models.Nodes, + models.Cpus.node_id == models.Nodes.id) + return query.filter(models.Nodes.uuid == inodeid) + + +def add_memory_filter_by_host(query, value): + if utils.is_int_like(value): + return query.filter_by(host_id=value) + else: + query = query.join(models.Hosts, + models.Memorys.host_id == models.Hosts.id) + return query.filter(models.Hosts.uuid == value) + + +def add_memory_filter_by_host_node(query, ihostid, inodeid): + if utils.is_int_like(ihostid) and utils.is_int_like(inodeid): + return query.filter_by(host_id=ihostid, node_id=inodeid) + + if utils.is_uuid_like(ihostid) and utils.is_uuid_like(inodeid): + ihostq = model_query(models.Hosts).filter_by(uuid=ihostid).first() + inodeq = model_query(models.Nodes).filter_by(uuid=inodeid).first() + + query = query.filter_by(host_id=ihostq.id, + node_id=inodeq.id) + + return query + + +def add_memory_filter_by_node(query, inodeid): + if utils.is_int_like(inodeid): + return query.filter_by(node_id=inodeid) + else: + query = query.join(models.Nodes, + models.Memorys.node_id == models.Nodes.id) + return query.filter(models.Nodes.uuid == inodeid) + + +def add_device_filter_by_host(query, hostid): + """Adds a device-specific ihost filter to a query. + + Filters results by host id if supplied value is an integer, + otherwise attempts to filter results by host uuid. + + :param query: Initial query to add filter to. + :param hostid: host id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(hostid): + return query.filter_by(host_id=hostid) + + elif utils.is_uuid_like(hostid): + query = query.join(models.Hosts) + return query.filter(models.Hosts.uuid == hostid) + + +def add_port_filter_by_numa_node(query, nodeid): + """Adds a port-specific numa node filter to a query. + + Filters results by numa node id if supplied nodeid is an integer, + otherwise attempts to filter results by numa node uuid. + + :param query: Initial query to add filter to. + :param nodeid: numa node id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(nodeid): + return query.filter_by(node_id=nodeid) + + elif utils.is_uuid_like(nodeid): + query = query.join(models.Nodes) + return query.filter(models.Nodes.uuid == nodeid) + + LOG.debug("port_filter_by_numa_node: " + "No match for supplied filter id (%s)" % str(nodeid)) + + +def add_port_filter_by_host(query, hostid): + """Adds a port-specific host filter to a query. + + Filters results by host id if supplied value is an integer, + otherwise attempts to filter results by host uuid. + + :param query: Initial query to add filter to. + :param hostid: host id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(hostid): + # Should not need join due to polymorphic ports table + # query = query.join(models.ports, + # models.EthernetPorts.id == models.ports.id) + # + # Query of ethernet_ports table should return data from + # corresponding ports table entry so should be able to + # use filter_by() rather than filter() + # + return query.filter_by(host_id=hostid) + + elif utils.is_uuid_like(hostid): + query = query.join(models.Hosts) + return query.filter(models.Hosts.uuid == hostid) + + LOG.debug("port_filter_by_host: " + "No match for supplied filter id (%s)" % str(hostid)) + + +def add_lldp_filter_by_host(query, hostid): + """Adds a lldp-specific ihost filter to a query. + + Filters results by host id if supplied value is an integer, + otherwise attempts to filter results by host uuid. + + :param query: Initial query to add filter to. + :param hostid: host id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(hostid): + return query.filter_by(host_id=hostid) + elif utils.is_uuid_like(hostid): + query = query.join(models.Hosts) + return query.filter(models.Hosts.uuid == hostid) + + LOG.debug("lldp_filter_by_host: " + "No match for supplied filter id (%s)" % str(hostid)) + + +def add_lldp_filter_by_port(query, portid): + """Adds a lldp-specific port filter to a query. + + Filters results by port id if supplied value is an integer, + otherwise attempts to filter results by port uuid. + + :param query: Initial query to add filter to. + :param portid: port id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(portid): + return query.filter_by(port_id=portid) + elif utils.is_uuid_like(portid): + query = query.join(models.Ports) + return query.filter(models.Ports.uuid == portid) + + +def add_lldp_filter_by_agent(query, value): + """Adds an lldp-specific filter to a query. + + Filters results by agent id if supplied value is an integer. + Filters results by agent UUID if supplied value is a UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(value): + return query.filter(models.LldpAgents.id == value) + elif uuidutils.is_uuid_like(value): + return query.filter(models.LldpAgents.uuid == value) + + +def add_lldp_filter_by_neighbour(query, value): + """Adds an lldp-specific filter to a query. + + Filters results by neighbour id if supplied value is an integer. + Filters results by neighbour UUID if supplied value is a UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(value): + return query.filter(models.LldpNeighbours.id == value) + elif uuidutils.is_uuid_like(value): + return query.filter(models.LldpNeighbours.uuid == value) + + +def add_lldp_tlv_filter_by_neighbour(query, neighbourid): + """Adds an lldp-specific filter to a query. + + Filters results by neighbour id if supplied value is an integer. + Filters results by neighbour UUID if supplied value is a UUID. + + :param query: Initial query to add filter to. + :param neighbourid: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(neighbourid): + return query.filter_by(neighbour_id=neighbourid) + elif uuidutils.is_uuid_like(neighbourid): + query = query.join( + models.LldpNeighbours, + models.LldpTlvs.neighbour_id == models.LldpNeighbours.id) + return query.filter(models.LldpNeighbours.uuid == neighbourid) + + +def add_lldp_tlv_filter_by_agent(query, agentid): + """Adds an lldp-specific filter to a query. + + Filters results by agent id if supplied value is an integer. + Filters results by agent UUID if supplied value is a UUID. + + :param query: Initial query to add filter to. + :param agentid: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(agentid): + return query.filter_by(agent_id=agentid) + elif uuidutils.is_uuid_like(agentid): + query = query.join(models.LldpAgents, + models.LldpTlvs.agent_id == models.LldpAgents.id) + return query.filter(models.LldpAgents.uuid == agentid) + + +# +# SENSOR FILTERS +# +def add_sensorgroup_filter(query, value): + """Adds a sensorgroup-specific filter to a query. + + Filters results by mac, if supplied value is a valid MAC + address. Otherwise attempts to filter results by identity. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if uuidutils.is_uuid_like(value): + return query.filter(or_(models.SensorGroupsAnalog.uuid == value, + models.SensorGroupsDiscrete.uuid == value)) + elif utils.is_int_like(value): + return query.filter(or_(models.SensorGroupsAnalog.id == value, + models.SensorGroupsDiscrete.id == value)) + else: + return add_identity_filter(query, value, use_sensorgroupname=True) + + +def add_sensorgroup_filter_by_sensor(query, value): + """Adds an sensorgroup-specific filter to a query. + + Filters results by sensor id if supplied value is an integer. + Filters results by sensor UUID if supplied value is a UUID. + Otherwise attempts to filter results by name + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + query = query.join(models.Sensors) + if utils.is_int_like(value): + return query.filter(models.Sensors.id == value) + elif uuidutils.is_uuid_like(value): + return query.filter(models.Sensors.uuid == value) + else: + return query.filter(models.Sensors.name == value) + + +def add_sensorgroup_filter_by_host(query, value): + """Adds an sensorgroup-specific filter to a query. + + Filters results by hostid, if supplied value is an integer. + Otherwise attempts to filter results by UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if utils.is_int_like(value): + return query.filter_by(host_id=value) + else: + query = query.join(models.Hosts, + models.SensorGroups.host_id == models.Hosts.id) + return query.filter(models.Hosts.uuid == value) + + +def add_sensor_filter(query, value): + """Adds a sensor-specific filter to a query. + + Filters results by identity. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + return add_identity_filter(query, value, use_sensorname=True) + + +def add_sensor_filter_by_host(query, hostid): + """Adds a sensor-specific ihost filter to a query. + + Filters results by host id if supplied value is an integer, + otherwise attempts to filter results by host uuid. + + :param query: Initial query to add filter to. + :param hostid: host id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(hostid): + return query.filter_by(host_id=hostid) + elif utils.is_uuid_like(hostid): + query = query.join(models.Hosts) + return query.filter(models.Hosts.uuid == hostid) + + LOG.debug("sensor_filter_by_host: " + "No match for supplied filter id (%s)" % str(hostid)) + + +def add_sensor_filter_by_sensorgroup(query, sensorgroupid): + """Adds a sensor-specific sensorgroup filter to a query. + + Filters results by sensorgroup id if supplied value is an integer, + otherwise attempts to filter results by sensorgroup uuid. + + :param query: Initial query to add filter to. + :param sensorgroupid: sensorgroup id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(sensorgroupid): + return query.filter_by(sensorgroup_id=sensorgroupid) + + elif utils.is_uuid_like(sensorgroupid): + query = query.join(models.SensorGroups, + models.Sensors.sensorgroup_id == + models.SensorGroups.id) + + return query.filter(models.SensorGroups.uuid == sensorgroupid) + + LOG.warn("sensor_filter_by_sensorgroup: " + "No match for supplied filter id (%s)" % str(sensorgroupid)) + + +def add_sensor_filter_by_host_sensorgroup(query, hostid, sensorgroupid): + """Adds a sensor-specific host and sensorgroup filter to a query. + + Filters results by host id and sensorgroup id if supplied hostid and + sensorgroupid are integers, otherwise attempts to filter results by + host uuid and sensorgroup uuid. + + :param query: Initial query to add filter to. + :param hostid: host id or uuid to filter results by. + :param sensorgroupid: sensorgroup id or uuid to filter results by. + :return: Modified query. + """ + if utils.is_int_like(hostid) and utils.is_int_like(sensorgroupid): + return query.filter_by(host_id=hostid, sensorgroup_id=sensorgroupid) + + elif utils.is_uuid_like(hostid) and utils.is_uuid_like(sensorgroupid): + query = query.join(models.Hosts, + models.SensorGroups) + return query.filter(models.Hosts.uuid == hostid, + models.SensorGroups.uuid == sensorgroupid) + + LOG.debug("sensor_filter_by_host_isensorgroup: " + "No match for supplied filter ids (%s, %s)" + % (str(hostid), str(sensorgroupid))) + + +class Connection(api.Connection): + """SQLAlchemy connection.""" + + def __init__(self): + pass + + def get_session(self, autocommit=True): + return get_session(autocommit) + + def get_engine(self): + return get_engine() + + def _system_get(self, system): + query = model_query(models.Systems) + query = add_identity_filter(query, system) + try: + result = query.one() + except NoResultFound: + raise exception.SystemNotFound(system=system) + return result + + def system_create(self, values): + if not values.get('uuid'): + # The system uuid comes from systemconfig + raise exception.SystemNotFound(system=values) + system = models.Systems() + system.update(values) + with _session_for_write() as session: + try: + session.add(system) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.SystemAlreadyExists(uuid=values['uuid']) + return self._system_get(values['uuid']) + + def system_get(self, system): + return self._system_get(system) + + def system_get_one(self): + query = model_query(models.Systems) + try: + return query.one() + except NoResultFound: + raise exception.NotFound() + + def system_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Systems) + return _paginate_query(models.Systems, limit, marker, + sort_key, sort_dir, query) + + def system_update(self, system, values): + with _session_for_write() as session: + query = model_query(models.Systems, session=session) + query = add_identity_filter(query, system) + + count = query.update(values, synchronize_session='fetch') + if count != 1: + raise exception.SystemNotFound(system=system) + return query.one() + + def system_delete(self, system): + with _session_for_write() as session: + query = model_query(models.Systems, session=session) + query = add_identity_filter(query, system) + try: + query.one() + except NoResultFound: + raise exception.SystemNotFound(system=system) + query.delete() + + # + # Hosts + # + + def _add_hosts_filters(self, query, filters): + if filters is None: + filters = dict() + supported_filters = {'hostname', + 'invprovision', + 'mgmt_mac', + 'personality', + } + unsupported_filters = set(filters).difference(supported_filters) + if unsupported_filters: + msg = _("SqlAlchemy API does not support " + "filtering by %s") % ', '.join(unsupported_filters) + raise ValueError(msg) + + for field in supported_filters: + if field in filters: + query = query.filter_by(**{field: filters[field]}) + + return query + + def _host_get(self, host): + query = model_query(models.Hosts) + if utils.is_uuid_like(host): + host.strip() + query = add_identity_filter(query, host) + try: + return query.one() + except NoResultFound: + raise exception.HostNotFound(host=host) + + def host_create(self, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + host = models.Hosts() + host.update(values) + with _session_for_write() as session: + try: + session.add(host) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.HostAlreadyExists(uuid=values['uuid']) + return self._host_get(values['uuid']) + + def host_get(self, host): + return self._host_get(host) + + def host_get_list(self, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Hosts) + query = self._add_hosts_filters(query, filters) + return _paginate_query(models.Hosts, limit, marker, + sort_key, sort_dir, query) + + def host_get_by_filters_one(self, filters): + query = model_query(models.Hosts) + query = self._add_hosts_filters(query, filters) + try: + return query.one() + except NoResultFound: + raise exception.HostNotFound(host=filters) + + def host_get_by_hostname(self, hostname): + query = model_query(models.Hosts) + query = query.filter_by(hostname=hostname) + try: + return query.one() + except NoResultFound: + raise exception.HostNotFound(host=hostname) + + def host_get_by_personality(self, personality, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Hosts) + query = query.filter_by(personality=personality) + return _paginate_query(models.Hosts, limit, marker, + sort_key, sort_dir, query) + + def host_get_by_mgmt_mac(self, mgmt_mac): + try: + mgmt_mac = mgmt_mac.rstrip() + mgmt_mac = utils.validate_and_normalize_mac(mgmt_mac) + except exception.InventoryException: + raise exception.HostNotFound(host=mgmt_mac) + + query = model_query(models.Hosts) + query = query.filter_by(mgmt_mac=mgmt_mac) + + try: + return query.one() + except NoResultFound: + raise exception.HostNotFound(host=mgmt_mac) + + def host_update(self, host, values, context=None): + with _session_for_write() as session: + query = model_query(models.Hosts, session=session) + query = add_identity_filter(query, host) + count = query.update(values, synchronize_session='fetch') + if count != 1: + raise exception.HostNotFound(host=host) + return self._host_get(host) + + def host_destroy(self, host): + with _session_for_write() as session: + query = model_query(models.Hosts, session=session) + query = add_identity_filter(query, host) + try: + query.one() + except NoResultFound: + raise exception.HostNotFound(host=host) + query.delete() + + # + # Ports + # + + def _port_get(self, portid, hostid=None): + query = model_query(models.Ports) + + if hostid: + query = query.filter_by(host_id=hostid) + + query = add_identity_filter(query, portid, use_name=True) + + try: + return query.one() + except NoResultFound: + raise exception.PortNotFound(port=portid) + + def port_get(self, portid, hostid=None): + return self._port_get(portid, hostid) + + def port_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.Ports, limit, marker, + sort_key, sort_dir) + + def port_get_all(self, hostid=None, interfaceid=None): + query = model_query(models.Ports, read_deleted="no") + if hostid: + query = query.filter_by(host_id=hostid) + if interfaceid: + query = query.filter_by(interface_id=interfaceid) + return query.all() + + def port_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Ports) + query = add_port_filter_by_host(query, host) + return _paginate_query(models.Ports, limit, marker, + sort_key, sort_dir, query) + + def port_get_by_numa_node(self, node, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Ports) + query = add_port_filter_by_numa_node(query, node) + return _paginate_query(models.Ports, limit, marker, + sort_key, sort_dir, query) + + def _ethernet_port_get(self, portid, hostid=None): + query = model_query(models.EthernetPorts) + + if hostid: + query = query.filter_by(host_id=hostid) + + query = add_identity_filter(query, portid, use_name=True) + + try: + return query.one() + except NoResultFound: + raise exception.PortNotFound(port=portid) + + def ethernet_port_create(self, hostid, values): + if utils.is_int_like(hostid): + host = self.host_get(int(hostid)) + elif utils.is_uuid_like(hostid): + host = self.host_get(hostid.strip()) + elif isinstance(hostid, models.Hosts): + host = hostid + else: + raise exception.HostNotFound(host=hostid) + + values['host_id'] = host['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + ethernet_port = models.EthernetPorts() + ethernet_port.update(values) + with _session_for_write() as session: + try: + session.add(ethernet_port) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add port %s (uuid: %s), port with MAC " + "address %s on host %s already exists" % + (values['name'], + values['uuid'], + values['mac'], + values['host_id'])) + raise exception.MACAlreadyExists(mac=values['mac'], + host=values['host_id']) + + return self._ethernet_port_get(values['uuid']) + + def ethernet_port_get(self, portid, hostid=None): + return self._ethernet_port_get(portid, hostid) + + def ethernet_port_get_by_mac(self, mac): + query = model_query(models.EthernetPorts).filter_by(mac=mac) + try: + return query.one() + except NoResultFound: + raise exception.PortNotFound(port=mac) + + def ethernet_port_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.EthernetPorts, limit, marker, + sort_key, sort_dir) + + def ethernet_port_get_all(self, hostid=None): + query = model_query(models.EthernetPorts, read_deleted="no") + if hostid: + query = query.filter_by(host_id=hostid) + return query.all() + + def ethernet_port_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.EthernetPorts) + query = add_port_filter_by_host(query, host) + return _paginate_query(models.EthernetPorts, limit, marker, + sort_key, sort_dir, query) + + def ethernet_port_get_by_numa_node(self, node, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.EthernetPorts) + query = add_port_filter_by_numa_node(query, node) + return _paginate_query(models.EthernetPorts, limit, marker, + sort_key, sort_dir, query) + + def ethernet_port_update(self, portid, values): + with _session_for_write() as session: + # May need to reserve in multi controller system; ref sysinv + query = model_query(models.EthernetPorts, read_deleted="no", + session=session) + query = add_identity_filter(query, portid) + + try: + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for port %s" % portid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for port %s" % portid) + + return query.one() + + def ethernet_port_destroy(self, portid): + with _session_for_write() as session: + # Delete port which should cascade to delete EthernetPort + if uuidutils.is_uuid_like(portid): + model_query(models.Ports, read_deleted="no", + session=session).\ + filter_by(uuid=portid).\ + delete() + else: + model_query(models.Ports, read_deleted="no", + session=session).\ + filter_by(id=portid).\ + delete() + + # + # Nodes + # + + def _node_get(self, node_id): + query = model_query(models.Nodes) + query = add_identity_filter(query, node_id) + + try: + result = query.one() + except NoResultFound: + raise exception.NodeNotFound(node=node_id) + + return result + + def node_create(self, host_id, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + values['host_id'] = int(host_id) + node = models.Nodes() + node.update(values) + with _session_for_write() as session: + try: + session.add(node) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.NodeAlreadyExists(uuid=values['uuid']) + + return self._node_get(values['uuid']) + + def node_get_all(self, host_id=None): + query = model_query(models.Nodes, read_deleted="no") + if host_id: + query = query.filter_by(host_id=host_id) + return query.all() + + def node_get(self, node_id): + return self._node_get(node_id) + + def node_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.Nodes, limit, marker, + sort_key, sort_dir) + + def node_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Nodes) + query = add_node_filter_by_host(query, host) + return _paginate_query(models.Nodes, limit, marker, + sort_key, sort_dir, query) + + def node_update(self, node_id, values): + with _session_for_write() as session: + # May need to reserve in multi controller system; ref sysinv + query = model_query(models.Nodes, read_deleted="no", + session=session) + query = add_identity_filter(query, node_id) + + count = query.update(values, synchronize_session='fetch') + if count != 1: + raise exception.NodeNotFound(node=node_id) + return query.one() + + def node_destroy(self, node_id): + with _session_for_write() as session: + # Delete physically since it has unique columns + if uuidutils.is_uuid_like(node_id): + model_query(models.Nodes, read_deleted="no", + session=session).\ + filter_by(uuid=node_id).\ + delete() + else: + model_query(models.Nodes, read_deleted="no").\ + filter_by(id=node_id).\ + delete() + + # + # Cpus + # + + def _cpu_get(self, cpu_id, host_id=None): + query = model_query(models.Cpus) + + if host_id: + query = query.filter_by(host_id=host_id) + + query = add_identity_filter(query, cpu_id) + + try: + result = query.one() + except NoResultFound: + raise exception.CPUNotFound(cpu=cpu_id) + + return result + + def cpu_create(self, host_id, values): + + if utils.is_int_like(host_id): + values['host_id'] = int(host_id) + else: + # this is not necessary if already integer following not work + host = self.host_get(host_id.strip()) + values['host_id'] = host['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + cpu = models.Cpus() + cpu.update(values) + + with _session_for_write() as session: + try: + session.add(cpu) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.CPUAlreadyExists(cpu=values['cpu']) + return self._cpu_get(values['uuid']) + + def cpu_get_all(self, host_id=None, fornodeid=None): + query = model_query(models.Cpus, read_deleted="no") + if host_id: + query = query.filter_by(host_id=host_id) + if fornodeid: + query = query.filter_by(fornodeid=fornodeid) + return query.all() + + def cpu_get(self, cpu_id, host_id=None): + return self._cpu_get(cpu_id, host_id) + + def cpu_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.Cpus, limit, marker, + sort_key, sort_dir) + + def cpu_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Cpus) + query = add_cpu_filter_by_host(query, host) + return _paginate_query(models.Cpus, limit, marker, + sort_key, sort_dir, query) + + def cpu_get_by_node(self, node, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Cpus) + query = add_cpu_filter_by_node(query, node) + return _paginate_query(models.Cpus, limit, marker, + sort_key, sort_dir, query) + + def cpu_get_by_host_node(self, host, node, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Cpus) + query = add_cpu_filter_by_host_node(query, host, node) + return _paginate_query(models.Cpus, limit, marker, + sort_key, sort_dir, query) + + def cpu_update(self, cpu_id, values, host_id=None): + with _session_for_write() as session: + # May need to reserve in multi controller system; ref sysinv + query = model_query(models.Cpus, read_deleted="no", + session=session) + if host_id: + query = query.filter_by(host_id=host_id) + + query = add_identity_filter(query, cpu_id) + + count = query.update(values, synchronize_session='fetch') + if count != 1: + raise exception.CPUNotFound(cpu=cpu_id) + return query.one() + + def cpu_destroy(self, cpu_id): + with _session_for_write() as session: + # Delete physically since it has unique columns + if uuidutils.is_uuid_like(cpu_id): + model_query(models.Cpus, read_deleted="no", session=session).\ + filter_by(uuid=cpu_id).\ + delete() + else: + model_query(models.Cpus, read_deleted="no").\ + filter_by(id=cpu_id).\ + delete() + + # + # Memory + # + + def _memory_get(self, memory_id, host_id=None): + query = model_query(models.Memorys) + + if host_id: + query = query.filter_by(host_id=host_id) + + query = add_identity_filter(query, memory_id) + + try: + result = query.one() + except NoResultFound: + raise exception.MemoryNotFound(memory=memory_id) + + return result + + def memory_create(self, host_id, values): + if utils.is_int_like(host_id): + values['host_id'] = int(host_id) + else: + # this is not necessary if already integer following not work + host = self.host_get(host_id.strip()) + values['host_id'] = host['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + values.pop('numa_node', None) + + memory = models.Memorys() + memory.update(values) + with _session_for_write() as session: + try: + session.add(memory) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.MemoryAlreadyExists(uuid=values['uuid']) + return self._memory_get(values['uuid']) + + def memory_get_all(self, host_id=None, fornodeid=None): + query = model_query(models.Memorys, read_deleted="no") + if host_id: + query = query.filter_by(host_id=host_id) + if fornodeid: + query = query.filter_by(fornodeid=fornodeid) + return query.all() + + def memory_get(self, memory_id, host_id=None): + return self._memory_get(memory_id, host_id) + + def memory_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.Memorys, limit, marker, + sort_key, sort_dir) + + def memory_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Memorys) + query = add_memory_filter_by_host(query, host) + return _paginate_query(models.Memorys, limit, marker, + sort_key, sort_dir, query) + + def memory_get_by_node(self, node, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Memorys) + query = add_memory_filter_by_node(query, node) + return _paginate_query(models.Memorys, limit, marker, + sort_key, sort_dir, query) + + def memory_get_by_host_node(self, host, node, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.Memorys) + query = add_memory_filter_by_host_node(query, host, node) + return _paginate_query(models.Memorys, limit, marker, + sort_key, sort_dir, query) + + def memory_update(self, memory_id, values, host_id=None): + with _session_for_write() as session: + # May need to reserve in multi controller system; ref sysinv + query = model_query(models.Memorys, read_deleted="no", + session=session) + if host_id: + query = query.filter_by(host_id=host_id) + + query = add_identity_filter(query, memory_id) + + values.pop('numa_node', None) + + count = query.update(values, synchronize_session='fetch') + if count != 1: + raise exception.MemoryNotFound(memory=memory_id) + return query.one() + + def memory_destroy(self, memory_id): + with _session_for_write() as session: + # Delete physically since it has unique columns + if uuidutils.is_uuid_like(memory_id): + model_query(models.Memorys, read_deleted="no", + session=session).\ + filter_by(uuid=memory_id).\ + delete() + else: + model_query(models.Memorys, read_deleted="no", + session=session).\ + filter_by(id=memory_id).\ + delete() + + # + # PciDevices + # + + def pci_device_create(self, hostid, values): + + if utils.is_int_like(hostid): + host = self.host_get(int(hostid)) + elif utils.is_uuid_like(hostid): + host = self.host_get(hostid.strip()) + elif isinstance(hostid, models.Hosts): + host = hostid + else: + raise exception.HostNotFound(host=hostid) + + values['host_id'] = host['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + pci_device = models.PciDevices() + pci_device.update(values) + with _session_for_write() as session: + try: + session.add(pci_device) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add pci device %s:%s (uuid: %s), " + "device with PCI address %s on host %s " + "already exists" % + (values['vendor'], + values['device'], + values['uuid'], + values['pciaddr'], + values['host_id'])) + raise exception.PCIAddrAlreadyExists(pciaddr=values['pciaddr'], + host=values['host_id']) + + def pci_device_get_all(self, hostid=None): + query = model_query(models.PciDevices, read_deleted="no") + if hostid: + query = query.filter_by(host_id=hostid) + return query.all() + + def pci_device_get(self, deviceid, hostid=None): + query = model_query(models.PciDevices) + if hostid: + query = query.filter_by(host_id=hostid) + query = add_identity_filter(query, deviceid, use_pciaddr=True) + try: + result = query.one() + except NoResultFound: + raise exception.PCIDeviceNotFound(pcidevice_id=deviceid) + + return result + + def pci_device_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.PciDevices, limit, marker, + sort_key, sort_dir) + + def pci_device_get_by_host(self, host, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.PciDevices) + query = add_device_filter_by_host(query, host) + return _paginate_query(models.PciDevices, limit, marker, + sort_key, sort_dir, query) + + def pci_device_update(self, device_id, values, host_id=None): + with _session_for_write() as session: + query = model_query(models.PciDevices, read_deleted="no", + session=session) + + if host_id: + query = query.filter_by(host_id=host_id) + + try: + query = add_identity_filter(query, device_id) + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for device %s" % device_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for device %s" % device_id) + + return query.one() + + def pci_device_destroy(self, device_id): + with _session_for_write() as session: + if uuidutils.is_uuid_like(device_id): + model_query(models.PciDevices, read_deleted="no", + session=session).\ + filter_by(uuid=device_id).\ + delete() + else: + model_query(models.PciDevices, read_deleted="no", + session=session).\ + filter_by(id=device_id).\ + delete() + + # + # LLDP + # + + def _lldp_agent_get(self, agentid, hostid=None): + query = model_query(models.LldpAgents) + + if hostid: + query = query.filter_by(host_id=hostid) + + query = add_lldp_filter_by_agent(query, agentid) + + try: + return query.one() + except NoResultFound: + raise exception.LldpAgentNotFound(agent=agentid) + + def lldp_agent_create(self, portid, hostid, values): + host = self.host_get(hostid) + port = self.port_get(portid) + + values['host_id'] = host['id'] + values['port_id'] = port['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + lldp_agent = models.LldpAgents() + lldp_agent.update(values) + with _session_for_write() as session: + try: + session.add(lldp_agent) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add lldp agent %s, on host %s:" + "already exists" % + (values['uuid'], + values['host_id'])) + raise exception.LLDPAgentExists(uuid=values['uuid'], + host=values['host_id']) + return self._lldp_agent_get(values['uuid']) + + def lldp_agent_get(self, agentid, hostid=None): + return self._lldp_agent_get(agentid, hostid) + + def lldp_agent_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.LldpAgents, limit, marker, + sort_key, sort_dir) + + def lldp_agent_get_all(self, hostid=None, portid=None): + query = model_query(models.LldpAgents, read_deleted="no") + if hostid: + query = query.filter_by(host_id=hostid) + if portid: + query = query.filter_by(port_id=portid) + return query.all() + + def lldp_agent_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.LldpAgents) + query = add_lldp_filter_by_host(query, host) + return _paginate_query(models.LldpAgents, limit, marker, + sort_key, sort_dir, query) + + def lldp_agent_get_by_port(self, port): + query = model_query(models.LldpAgents) + query = add_lldp_filter_by_port(query, port) + try: + return query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for agent on port %s" % port) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for agent on port %s" % port) + + def lldp_agent_update(self, uuid, values): + with _session_for_write(): + query = model_query(models.LldpAgents, read_deleted="no") + + try: + query = add_lldp_filter_by_agent(query, uuid) + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + return result + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for agent %s" % uuid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for agent %s" % uuid) + + def lldp_agent_destroy(self, agentid): + + with _session_for_write(): + query = model_query(models.LldpAgents, read_deleted="no") + query = add_lldp_filter_by_agent(query, agentid) + + try: + query.delete() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for agent %s" % agentid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for agent %s" % agentid) + + def _lldp_neighbour_get(self, neighbourid, hostid=None): + query = model_query(models.LldpNeighbours) + + if hostid: + query = query.filter_by(host_id=hostid) + + query = add_lldp_filter_by_neighbour(query, neighbourid) + + try: + return query.one() + except NoResultFound: + raise exception.LldpNeighbourNotFound(neighbour=neighbourid) + + def lldp_neighbour_create(self, portid, hostid, values): + if utils.is_int_like(hostid): + host = self.host_get(int(hostid)) + elif utils.is_uuid_like(hostid): + host = self.host_get(hostid.strip()) + elif isinstance(hostid, models.Hosts): + host = hostid + else: + raise exception.HostNotFound(host=hostid) + if utils.is_int_like(portid): + port = self.port_get(int(portid)) + elif utils.is_uuid_like(portid): + port = self.port_get(portid.strip()) + elif isinstance(portid, models.port): + port = portid + else: + raise exception.PortNotFound(port=portid) + + values['host_id'] = host['id'] + values['port_id'] = port['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + lldp_neighbour = models.LldpNeighbours() + lldp_neighbour.update(values) + with _session_for_write() as session: + try: + session.add(lldp_neighbour) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add lldp neighbour %s, on port %s:. " + "Already exists with msap %s" % + (values['uuid'], + values['port_id'], + values['msap'])) + raise exception.LLDPNeighbourExists(uuid=values['uuid']) + + return self._lldp_neighbour_get(values['uuid']) + + def lldp_neighbour_get(self, neighbourid, hostid=None): + return self._lldp_neighbour_get(neighbourid, hostid) + + def lldp_neighbour_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.LldpNeighbours, limit, marker, + sort_key, sort_dir) + + def lldp_neighbour_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.LldpNeighbours) + query = add_port_filter_by_host(query, host) + return _paginate_query(models.LldpNeighbours, limit, marker, + sort_key, sort_dir, query) + + def lldp_neighbour_get_by_port(self, port, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.LldpNeighbours) + query = add_lldp_filter_by_port(query, port) + return _paginate_query(models.LldpNeighbours, limit, marker, + sort_key, sort_dir, query) + + def lldp_neighbour_update(self, uuid, values): + with _session_for_write(): + query = model_query(models.LldpNeighbours, read_deleted="no") + + try: + query = add_lldp_filter_by_neighbour(query, uuid) + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + return result + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for uuid %s" % uuid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for uuid %s" % uuid) + + def lldp_neighbour_destroy(self, neighbourid): + with _session_for_write(): + query = model_query(models.LldpNeighbours, read_deleted="no") + query = add_lldp_filter_by_neighbour(query, neighbourid) + try: + query.delete() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for neighbour %s" % neighbourid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for neighbour %s" % + neighbourid) + + def _lldp_tlv_get(self, type, agentid=None, neighbourid=None, + session=None): + if not agentid and not neighbourid: + raise exception.InvalidParameterValue( + err="agent id and neighbour id not specified") + + query = model_query(models.LldpTlvs, session=session) + + if agentid: + query = query.filter_by(agent_id=agentid) + + if neighbourid: + query = query.filter_by(neighbour_id=neighbourid) + + query = query.filter_by(type=type) + + try: + return query.one() + except NoResultFound: + raise exception.LldpTlvNotFound(type=type) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found") + + def lldp_tlv_create(self, values, agentid=None, neighbourid=None): + if not agentid and not neighbourid: + raise exception.InvalidParameterValue( + err="agent id and neighbour id not specified") + + if agentid: + if utils.is_int_like(agentid): + agent = self.lldp_agent_get(int(agentid)) + elif utils.is_uuid_like(agentid): + agent = self.lldp_agent_get(agentid.strip()) + elif isinstance(agentid, models.lldp_agents): + agent = agentid + else: + raise exception.LldpAgentNotFound(agent=agentid) + + if neighbourid: + if utils.is_int_like(neighbourid): + neighbour = self.lldp_neighbour_get(int(neighbourid)) + elif utils.is_uuid_like(neighbourid): + neighbour = self.lldp_neighbour_get(neighbourid.strip()) + elif isinstance(neighbourid, models.lldp_neighbours): + neighbour = neighbourid + else: + raise exception.LldpNeighbourNotFound(neighbour=neighbourid) + + if agentid: + values['agent_id'] = agent['id'] + + if neighbourid: + values['neighbour_id'] = neighbour['id'] + + lldp_tlv = models.LldpTlvs() + lldp_tlv.update(values) + with _session_for_write() as session: + try: + session.add(lldp_tlv) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add lldp tlv %s" + "already exists" % (values['type'])) + raise exception.LLDPTlvExists(uuid=values['id']) + return self._lldp_tlv_get(values['type'], + agentid=values.get('agent_id'), + neighbourid=values.get('neighbour_id')) + + def lldp_tlv_create_bulk(self, values, agentid=None, neighbourid=None): + if not agentid and not neighbourid: + raise exception.InvalidParameterValue( + err="agent id and neighbour id not specified") + + if agentid: + if utils.is_int_like(agentid): + agent = self.lldp_agent_get(int(agentid)) + elif utils.is_uuid_like(agentid): + agent = self.lldp_agent_get(agentid.strip()) + elif isinstance(agentid, models.lldp_agents): + agent = agentid + else: + raise exception.LldpAgentNotFound(agent=agentid) + + if neighbourid: + if utils.is_int_like(neighbourid): + neighbour = self.lldp_neighbour_get(int(neighbourid)) + elif utils.is_uuid_like(neighbourid): + neighbour = self.lldp_neighbour_get(neighbourid.strip()) + elif isinstance(neighbourid, models.lldp_neighbours): + neighbour = neighbourid + else: + raise exception.LldpNeighbourNotFound(neighbour=neighbourid) + + tlvs = [] + with _session_for_write() as session: + for entry in values: + lldp_tlv = models.LldpTlvs() + if agentid: + entry['agent_id'] = agent['id'] + + if neighbourid: + entry['neighbour_id'] = neighbour['id'] + + lldp_tlv.update(entry) + session.add(lldp_tlv) + + lldp_tlv = self._lldp_tlv_get( + entry['type'], + agentid=entry.get('agent_id'), + neighbourid=entry.get('neighbour_id'), + session=session) + + tlvs.append(lldp_tlv) + + return tlvs + + def lldp_tlv_get(self, type, agentid=None, neighbourid=None): + return self._lldp_tlv_get(type, agentid, neighbourid) + + def lldp_tlv_get_by_id(self, id, agentid=None, neighbourid=None): + query = model_query(models.LldpTlvs) + + query = query.filter_by(id=id) + try: + result = query.one() + except NoResultFound: + raise exception.LldpTlvNotFound(id=id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found") + + return result + + def lldp_tlv_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.LldpTlvs, limit, marker, + sort_key, sort_dir) + + def lldp_tlv_get_all(self, agentid=None, neighbourid=None): + query = model_query(models.LldpTlvs, read_deleted="no") + if agentid: + query = query.filter_by(agent_id=agentid) + if neighbourid: + query = query.filter_by(neighbour_id=neighbourid) + return query.all() + + def lldp_tlv_get_by_agent(self, agent, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.LldpTlvs) + query = add_lldp_tlv_filter_by_agent(query, agent) + return _paginate_query(models.LldpTlvs, limit, marker, + sort_key, sort_dir, query) + + def lldp_tlv_get_by_neighbour(self, neighbour, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.LldpTlvs) + query = add_lldp_tlv_filter_by_neighbour(query, neighbour) + return _paginate_query(models.LldpTlvs, limit, marker, + sort_key, sort_dir, query) + + def lldp_tlv_update(self, values, agentid=None, neighbourid=None): + if not agentid and not neighbourid: + raise exception.InvalidParameterValue( + err="agent id and neighbour id not specified") + + with _session_for_write(): + query = model_query(models.LldpTlvs, read_deleted="no") + + if agentid: + query = add_lldp_tlv_filter_by_agent(query, agentid) + + if neighbourid: + query = add_lldp_tlv_filter_by_neighbour(query, + neighbourid) + + query = query.filter_by(type=values['type']) + + try: + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + return result + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for tlv") + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found") + + def lldp_tlv_update_bulk(self, values, agentid=None, neighbourid=None): + results = [] + + if not agentid and not neighbourid: + raise exception.InvalidParameterValue( + err="agent id and neighbour id not specified") + + with _session_for_write() as session: + for entry in values: + query = model_query(models.LldpTlvs, read_deleted="no") + + if agentid: + query = query.filter_by(agent_id=agentid) + + if neighbourid: + query = query.filter_by(neighbour_id=neighbourid) + + query = query.filter_by(type=entry['type']) + + try: + result = query.one() + result.update(entry) + session.merge(result) + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for tlv") + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found") + + results.append(result) + return results + + def lldp_tlv_destroy(self, id): + with _session_for_write(): + model_query(models.LldpTlvs, read_deleted="no").\ + filter_by(id=id).\ + delete() + + # + # SENSORS + # + + def _sensor_analog_create(self, hostid, values): + if utils.is_int_like(hostid): + host = self.host_get(int(hostid)) + elif utils.is_uuid_like(hostid): + host = self.host_get(hostid.strip()) + elif isinstance(hostid, models.Hosts): + host = hostid + else: + raise exception.HostNotFound(host=hostid) + + values['host_id'] = host['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + sensor_analog = models.SensorsAnalog() + sensor_analog.update(values) + + with _session_for_write() as session: + try: + session.add(sensor_analog) + session.flush() + except db_exc.DBDuplicateEntry: + exception.SensorAlreadyExists(uuid=values['uuid']) + return self._sensor_analog_get(values['uuid']) + + def _sensor_analog_get(self, sensorid, hostid=None): + query = model_query(models.SensorsAnalog) + + if hostid: + query = query.filter_by(host_id=hostid) + + query = add_sensor_filter(query, sensorid) + + try: + result = query.one() + except NoResultFound: + raise exception.SensorNotFound(sensor=sensorid) + + return result + + def _sensor_analog_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.SensorsAnalog, limit, marker, + sort_key, sort_dir) + + def _sensor_analog_get_all(self, hostid=None, sensorgroupid=None): + query = model_query(models.SensorsAnalog, read_deleted="no") + if hostid: + query = query.filter_by(host_id=hostid) + if sensorgroupid: + query = query.filter_by(sensorgroup_id=hostid) + return query.all() + + def _sensor_analog_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorsAnalog) + query = add_port_filter_by_host(query, host) + return _paginate_query(models.SensorsAnalog, limit, marker, + sort_key, sort_dir, query) + + def _sensor_analog_get_by_sensorgroup(self, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.SensorsAnalog) + query = add_sensor_filter_by_sensorgroup(query, sensorgroup) + return _paginate_query(models.SensorsAnalog, limit, marker, + sort_key, sort_dir, query) + + def _sensor_analog_get_by_host_sensorgroup(self, host, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorsAnalog) + query = add_sensor_filter_by_host_sensorgroup(query, + host, + sensorgroup) + return _paginate_query(models.SensorsAnalog, limit, marker, + sort_key, sort_dir, query) + + def _sensor_analog_update(self, sensorid, values, hostid=None): + with _session_for_write(): + # May need to reserve in multi controller system; ref sysinv + query = model_query(models.SensorsAnalog, read_deleted="no") + + if hostid: + query = query.filter_by(host_id=hostid) + + try: + query = add_sensor_filter(query, sensorid) + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for port %s" % sensorid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for port %s" % sensorid) + + return query.one() + + def _sensor_analog_destroy(self, sensorid): + with _session_for_write(): + # Delete port which should cascade to delete SensorsAnalog + if uuidutils.is_uuid_like(sensorid): + model_query(models.Sensors, read_deleted="no").\ + filter_by(uuid=sensorid).\ + delete() + else: + model_query(models.Sensors, read_deleted="no").\ + filter_by(id=sensorid).\ + delete() + + def sensor_analog_create(self, hostid, values): + return self._sensor_analog_create(hostid, values) + + def sensor_analog_get(self, sensorid, hostid=None): + return self._sensor_analog_get(sensorid, hostid) + + def sensor_analog_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_analog_get_list(limit, marker, sort_key, sort_dir) + + def sensor_analog_get_all(self, hostid=None, sensorgroupid=None): + return self._sensor_analog_get_all(hostid, sensorgroupid) + + def sensor_analog_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_analog_get_by_host(host, limit, marker, + sort_key, sort_dir) + + def sensor_analog_get_by_sensorgroup(self, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_analog_get_by_sensorgroup(sensorgroup, + limit, marker, + sort_key, sort_dir) + + def sensor_analog_get_by_host_sensorgroup(self, host, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_analog_get_by_host_sensorgroup(host, sensorgroup, + limit, marker, + sort_key, sort_dir) + + def sensor_analog_update(self, sensorid, values, hostid=None): + return self._sensor_analog_update(sensorid, values, hostid) + + def sensor_analog_destroy(self, sensorid): + return self._sensor_analog_destroy(sensorid) + + def _sensor_discrete_create(self, hostid, values): + if utils.is_int_like(hostid): + host = self.host_get(int(hostid)) + elif utils.is_uuid_like(hostid): + host = self.host_get(hostid.strip()) + elif isinstance(hostid, models.Hosts): + host = hostid + else: + raise exception.HostNotFound(host=hostid) + + values['host_id'] = host['id'] + + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + sensor_discrete = models.SensorsDiscrete() + sensor_discrete.update(values) + with _session_for_write() as session: + try: + session.add(sensor_discrete) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.SensorAlreadyExists(uuid=values['uuid']) + return self._sensor_discrete_get(values['uuid']) + + def _sensor_discrete_get(self, sensorid, hostid=None): + query = model_query(models.SensorsDiscrete) + + if hostid: + query = query.filter_by(host_id=hostid) + + query = add_sensor_filter(query, sensorid) + + try: + result = query.one() + except NoResultFound: + raise exception.SensorNotFound(sensor=sensorid) + + return result + + def _sensor_discrete_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(models.SensorsDiscrete, limit, marker, + sort_key, sort_dir) + + def _sensor_discrete_get_all(self, hostid=None, sensorgroupid=None): + query = model_query(models.SensorsDiscrete, read_deleted="no") + if hostid: + query = query.filter_by(host_id=hostid) + if sensorgroupid: + query = query.filter_by(sensorgroup_id=hostid) + return query.all() + + def _sensor_discrete_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorsDiscrete) + query = add_port_filter_by_host(query, host) + return _paginate_query(models.SensorsDiscrete, limit, marker, + sort_key, sort_dir, query) + + def _sensor_discrete_get_by_sensorgroup(self, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(models.SensorsDiscrete) + query = add_sensor_filter_by_sensorgroup(query, sensorgroup) + return _paginate_query(models.SensorsDiscrete, limit, marker, + sort_key, sort_dir, query) + + def _sensor_discrete_get_by_host_sensorgroup(self, host, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorsDiscrete) + query = add_sensor_filter_by_host_sensorgroup(query, + host, + sensorgroup) + return _paginate_query(models.SensorsDiscrete, limit, marker, + sort_key, sort_dir, query) + + def _sensor_discrete_update(self, sensorid, values, hostid=None): + with _session_for_write(): + # May need to reserve in multi controller system; ref sysinv + query = model_query(models.SensorsDiscrete, read_deleted="no") + + if hostid: + query = query.filter_by(host_id=hostid) + + try: + query = add_sensor_filter(query, sensorid) + result = query.one() + for k, v in values.items(): + setattr(result, k, v) + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for port %s" % sensorid) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for port %s" % sensorid) + + return query.one() + + def _sensor_discrete_destroy(self, sensorid): + with _session_for_write(): + # Delete port which should cascade to delete SensorsDiscrete + if uuidutils.is_uuid_like(sensorid): + model_query(models.Sensors, read_deleted="no").\ + filter_by(uuid=sensorid).\ + delete() + else: + model_query(models.Sensors, read_deleted="no").\ + filter_by(id=sensorid).\ + delete() + + def sensor_discrete_create(self, hostid, values): + return self._sensor_discrete_create(hostid, values) + + def sensor_discrete_get(self, sensorid, hostid=None): + return self._sensor_discrete_get(sensorid, hostid) + + def sensor_discrete_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_discrete_get_list( + limit, marker, sort_key, sort_dir) + + def sensor_discrete_get_all(self, hostid=None, sensorgroupid=None): + return self._sensor_discrete_get_all(hostid, sensorgroupid) + + def sensor_discrete_get_by_host(self, host, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_discrete_get_by_host(host, limit, marker, + sort_key, sort_dir) + + def sensor_discrete_get_by_sensorgroup(self, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_discrete_get_by_sensorgroup( + sensorgroup, limit, marker, sort_key, sort_dir) + + def sensor_discrete_get_by_host_sensorgroup(self, host, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensor_discrete_get_by_host_sensorgroup( + host, sensorgroup, limit, marker, sort_key, sort_dir) + + def sensor_discrete_update(self, sensorid, values, hostid=None): + return self._sensor_discrete_update(sensorid, values, hostid) + + def sensor_discrete_destroy(self, sensorid): + return self._sensor_discrete_destroy(sensorid) + + def _sensor_get(self, cls, sensor_id, ihost=None, obj=None): + session = None + if obj: + session = inspect(obj).session + query = model_query(cls, session=session) + query = add_sensor_filter(query, sensor_id) + if ihost: + query = add_sensor_filter_by_host(query, ihost) + + try: + result = query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensor %s" % sensor_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensor %s" % sensor_id) + + return result + + def _sensor_create(self, obj, host_id, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + values['host_id'] = int(host_id) + + if 'sensor_profile' in values: + values.pop('sensor_profile') + + # The id is null for ae sensors with more than one member + # sensor + temp_id = obj.id + obj.update(values) + if obj.id is None: + obj.id = temp_id + + with _session_for_write() as session: + try: + session.add(obj) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add sensor %s (uuid: %s), an sensor " + "with name %s already exists on host %s" % + (values['sensorname'], + values['uuid'], + values['sensorname'], + values['host_id'])) + raise exception.SensorAlreadyExists(uuid=values['uuid']) + return self._sensor_get(type(obj), values['uuid']) + + def sensor_create(self, hostid, values): + if values['datatype'] == 'discrete': + sensor = models.SensorsDiscrete() + elif values['datatype'] == 'analog': + sensor = models.SensorsAnalog() + else: + sensor = models.SensorsAnalog() + LOG.error("default SensorsAnalog due to datatype=%s" % + values['datatype']) + + return self._sensor_create(sensor, hostid, values) + + def sensor_get(self, sensorid, hostid=None): + return self._sensor_get(models.Sensors, sensorid, hostid) + + def sensor_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + model_query(models.Sensors) + return _paginate_query(models.Sensors, limit, marker, + sort_key, sort_dir) + + def sensor_get_all(self, host_id=None, sensorgroupid=None): + query = model_query(models.Sensors, read_deleted="no") + + if host_id: + query = query.filter_by(host_id=host_id) + if sensorgroupid: + query = query.filter_by(sensorgroup_id=sensorgroupid) + return query.all() + + def sensor_get_by_host(self, ihost, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Sensors) + query = add_sensor_filter_by_host(query, ihost) + return _paginate_query(models.Sensors, limit, marker, + sort_key, sort_dir, query) + + def _sensor_get_by_sensorgroup(self, cls, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(cls) + query = add_sensor_filter_by_sensorgroup(query, sensorgroup) + return _paginate_query(cls, limit, marker, sort_key, sort_dir, query) + + def sensor_get_by_sensorgroup(self, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Sensors) + query = add_sensor_filter_by_sensorgroup(query, sensorgroup) + return _paginate_query(models.Sensors, limit, marker, + sort_key, sort_dir, query) + + def sensor_get_by_host_sensorgroup(self, ihost, sensorgroup, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.Sensors) + query = add_sensor_filter_by_host(query, ihost) + query = add_sensor_filter_by_sensorgroup(query, sensorgroup) + return _paginate_query(models.Sensors, limit, marker, + sort_key, sort_dir, query) + + def _sensor_update(self, cls, sensor_id, values): + with _session_for_write(): + query = model_query(models.Sensors) + query = add_sensor_filter(query, sensor_id) + try: + result = query.one() + # obj = self._sensor_get(models.Sensors, sensor_id) + for k, v in values.items(): + if v == 'none': + v = None + setattr(result, k, v) + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensor %s" % sensor_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensor %s" % sensor_id) + + return query.one() + + def sensor_update(self, sensor_id, values): + with _session_for_write(): + query = model_query(models.Sensors, read_deleted="no") + query = add_sensor_filter(query, sensor_id) + try: + result = query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensor %s" % sensor_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensor %s" % sensor_id) + + if result.datatype == 'discrete': + return self._sensor_update(models.SensorsDiscrete, + sensor_id, values) + elif result.datatype == 'analog': + return self._sensor_update(models.SensorsAnalog, + sensor_id, values) + else: + return self._sensor_update(models.SensorsAnalog, + sensor_id, values) + + def _sensor_destroy(self, cls, sensor_id): + with _session_for_write(): + # Delete sensor which should cascade to delete derived sensors + if uuidutils.is_uuid_like(sensor_id): + model_query(cls, read_deleted="no").\ + filter_by(uuid=sensor_id).\ + delete() + else: + model_query(cls, read_deleted="no").\ + filter_by(id=sensor_id).\ + delete() + + def sensor_destroy(self, sensor_id): + return self._sensor_destroy(models.Sensors, sensor_id) + + # SENSOR GROUPS + def sensorgroup_create(self, host_id, values): + if values['datatype'] == 'discrete': + sensorgroup = models.SensorGroupsDiscrete() + elif values['datatype'] == 'analog': + sensorgroup = models.SensorGroupsAnalog() + else: + LOG.error("default SensorsAnalog due to datatype=%s" % + values['datatype']) + + sensorgroup = models.SensorGroupsAnalog + return self._sensorgroup_create(sensorgroup, host_id, values) + + def _sensorgroup_get(self, cls, sensorgroup_id, ihost=None, obj=None): + query = model_query(cls) + query = add_sensorgroup_filter(query, sensorgroup_id) + if ihost: + query = add_sensorgroup_filter_by_host(query, ihost) + + try: + result = query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensorgroup %s" % sensorgroup_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensorgroup %s" % + sensorgroup_id) + + return result + + def sensorgroup_get(self, sensorgroup_id, ihost=None): + return self._sensorgroup_get(models.SensorGroups, + sensorgroup_id, + ihost) + + def sensorgroup_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorGroups) + return _paginate_query(models.SensorGroupsAnalog, limit, marker, + sort_key, sort_dir, query) + + def sensorgroup_get_by_host_sensor(self, ihost, sensor, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorGroups) + query = add_sensorgroup_filter_by_host(query, ihost) + query = add_sensorgroup_filter_by_sensor(query, sensor) + try: + result = query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for host %s port %s" % (ihost, sensor)) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for host %s port %s" % + (ihost, sensor)) + + return result + + def sensorgroup_get_by_host(self, ihost, + limit=None, marker=None, + sort_key=None, sort_dir=None): + query = model_query(models.SensorGroups) + query = add_sensorgroup_filter_by_host(query, ihost) + return _paginate_query(models.SensorGroups, limit, marker, + sort_key, sort_dir, query) + + def sensorgroup_update(self, sensorgroup_id, values): + with _session_for_write(): + query = model_query(models.SensorGroups, read_deleted="no") + query = add_sensorgroup_filter(query, sensorgroup_id) + try: + result = query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensorgroup %s" % sensorgroup_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensorgroup %s" % + sensorgroup_id) + + if result.datatype == 'discrete': + return self._sensorgroup_update(models.SensorGroupsDiscrete, + sensorgroup_id, + values) + elif result.datatype == 'analog': + return self._sensorgroup_update(models.SensorGroupsAnalog, + sensorgroup_id, + values) + else: + return self._sensorgroup_update(models.SensorGroupsAnalog, + sensorgroup_id, + values) + + def sensorgroup_propagate(self, sensorgroup_id, values): + query = model_query(models.SensorGroups, read_deleted="no") + query = add_sensorgroup_filter(query, sensorgroup_id) + try: + result = query.one() + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensorgroup %s" % sensorgroup_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensorgroup %s" % + sensorgroup_id) + + sensors = self._sensor_get_by_sensorgroup(models.Sensors, + result.uuid) + for sensor in sensors: + LOG.info("sensorgroup update propagate sensor=%s val=%s" % + (sensor.sensorname, values)) + self._sensor_update(models.Sensors, sensor.uuid, values) + + def _sensorgroup_create(self, obj, host_id, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + values['host_id'] = int(host_id) + + if 'sensorgroup_profile' in values: + values.pop('sensorgroup_profile') + + temp_id = obj.id + obj.update(values) + if obj.id is None: + obj.id = temp_id + with _session_for_write() as session: + try: + session.add(obj) + session.flush() + except db_exc.DBDuplicateEntry: + LOG.error("Failed to add sensorgroup %s (uuid: %s), a " + "sensorgroup with name %s already exists on host %s" + % (values['sensorgroupname'], + values['uuid'], + values['sensorgroupname'], + values['host_id'])) + raise exception.SensorGroupAlreadyExists(uuid=values['uuid']) + return self._sensorgroup_get(type(obj), values['uuid']) + + def _sensorgroup_get_all(self, cls, host_id=None): + query = model_query(cls, read_deleted="no") + if utils.is_int_like(host_id): + query = query.filter_by(host_id=host_id) + return query.all() + + def _sensorgroup_get_list(self, cls, limit=None, marker=None, + sort_key=None, sort_dir=None): + return _paginate_query(cls, limit, marker, sort_key, sort_dir) + + def _sensorgroup_get_by_host_sensor(self, cls, ihost, sensor, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(cls).join(models.Sensors) + query = add_sensorgroup_filter_by_host(query, ihost) + query = add_sensorgroup_filter_by_sensor(query, sensor) + return _paginate_query(cls, limit, marker, sort_key, sort_dir, query) + + def _sensorgroup_get_by_host(self, cls, ihost, + limit=None, marker=None, + sort_key=None, sort_dir=None): + + query = model_query(cls) + query = add_sensorgroup_filter_by_host(query, ihost) + return _paginate_query(cls, limit, marker, sort_key, sort_dir, query) + + def _sensorgroup_update(self, cls, sensorgroup_id, values): + with _session_for_write() as session: + # query = model_query(models.SensorGroups, read_deleted="no") + query = model_query(cls, read_deleted="no") + try: + query = add_sensorgroup_filter(query, sensorgroup_id) + result = query.one() + + # obj = self._sensorgroup_get(models.SensorGroups, + obj = self._sensorgroup_get(cls, sensorgroup_id) + + for k, v in values.items(): + if k == 'algorithm' and v == 'none': + v = None + if k == 'actions_critical_choices' and v == 'none': + v = None + if k == 'actions_major_choices' and v == 'none': + v = None + if k == 'actions_minor_choices' and v == 'none': + v = None + setattr(result, k, v) + + except NoResultFound: + raise exception.InvalidParameterValue( + err="No entry found for sensorgroup %s" % sensorgroup_id) + except MultipleResultsFound: + raise exception.InvalidParameterValue( + err="Multiple entries found for sensorgroup %s" % + sensorgroup_id) + try: + session.add(obj) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.SensorGroupAlreadyExists(uuid=values['uuid']) + return query.one() + + def _sensorgroup_destroy(self, cls, sensorgroup_id): + with _session_for_write(): + # Delete sensorgroup which should cascade to + # delete derived sensorgroups + if uuidutils.is_uuid_like(sensorgroup_id): + model_query(cls, read_deleted="no").\ + filter_by(uuid=sensorgroup_id).\ + delete() + else: + model_query(cls, read_deleted="no").\ + filter_by(id=sensorgroup_id).\ + delete() + + def sensorgroup_destroy(self, sensorgroup_id): + return self._sensorgroup_destroy(models.SensorGroups, sensorgroup_id) + + def sensorgroup_analog_create(self, host_id, values): + sensorgroup = models.SensorGroupsAnalog() + return self._sensorgroup_create(sensorgroup, host_id, values) + + def sensorgroup_analog_get_all(self, host_id=None): + return self._sensorgroup_get_all(models.SensorGroupsAnalog, host_id) + + def sensorgroup_analog_get(self, sensorgroup_id): + return self._sensorgroup_get(models.SensorGroupsAnalog, + sensorgroup_id) + + def sensorgroup_analog_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensorgroup_get_list(models.SensorGroupsAnalog, + limit, marker, + sort_key, sort_dir) + + def sensorgroup_analog_get_by_host(self, ihost, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensorgroup_get_by_host(models.SensorGroupsAnalog, + ihost, + limit, marker, + sort_key, sort_dir) + + def sensorgroup_analog_update(self, sensorgroup_id, values): + return self._sensorgroup_update(models.SensorGroupsAnalog, + sensorgroup_id, + values) + + def sensorgroup_analog_destroy(self, sensorgroup_id): + return self._sensorgroup_destroy(models.SensorGroupsAnalog, + sensorgroup_id) + + def sensorgroup_discrete_create(self, host_id, values): + sensorgroup = models.SensorGroupsDiscrete() + return self._sensorgroup_create(sensorgroup, host_id, values) + + def sensorgroup_discrete_get_all(self, host_id=None): + return self._sensorgroup_get_all(models.SensorGroupsDiscrete, host_id) + + def sensorgroup_discrete_get(self, sensorgroup_id): + return self._sensorgroup_get(models.SensorGroupsDiscrete, + sensorgroup_id) + + def sensorgroup_discrete_get_list(self, limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensorgroup_get_list(models.SensorGroupsDiscrete, + limit, marker, + sort_key, sort_dir) + + def sensorgroup_discrete_get_by_host(self, ihost, + limit=None, marker=None, + sort_key=None, sort_dir=None): + return self._sensorgroup_get_by_host(models.SensorGroupsDiscrete, + ihost, + limit, marker, sort_key, sort_dir) + + def sensorgroup_discrete_update(self, sensorgroup_id, values): + return self._sensorgroup_update(models.SensorGroupsDiscrete, + sensorgroup_id, values) + + def sensorgroup_discrete_destroy(self, sensorgroup_id): + return self._sensorgroup_destroy(models.SensorGroupsDiscrete, + sensorgroup_id) diff --git a/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/README b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/README new file mode 100644 index 00000000..54745cf4 --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at: + https://github.com/openstack/sqlalchemy-migrate diff --git a/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/__init__.py b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/manage.py b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/manage.py new file mode 100644 index 00000000..1d50f84d --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/manage.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from migrate.versioning.shell import main + + +if __name__ == '__main__': + main(debug='False', repository='.') diff --git a/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/migrate.cfg b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/migrate.cfg new file mode 100644 index 00000000..2790b4be --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/migrate.cfg @@ -0,0 +1,21 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=inventory + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] + diff --git a/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/001_init.py b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/001_init.py new file mode 100644 index 00000000..51c998d5 --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/001_init.py @@ -0,0 +1,605 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sqlalchemy import Column, MetaData, String, Table, UniqueConstraint +from sqlalchemy import Boolean, Integer, Enum, Text, ForeignKey, DateTime + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + systems = Table( + 'systems', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True, index=True), + + Column('system_type', String(255)), + Column('system_mode', String(255)), + + Column('name', String(255), unique=True), + Column('contact', String(255)), + Column('location', String(255)), + + Column('description', String(255), unique=True), + Column('timezone', String(255)), + Column('region_name', Text), + Column('services', Integer, default=72), + Column('service_project_name', Text), + Column('distributed_cloud_role', String(255)), + Column('security_profile', String(255)), + Column('security_feature', String(255)), + Column('software_version', String(255)), + Column('capabilities', Text), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + # Hosts Enum definitions + recordtype_enum = Enum('standard', + 'reserve1', + 'reserve2', + name='recordtype_enum') + + personality_enum = Enum('controller', + 'compute', + 'storage', + 'reserve1', + 'reserve2', + name='personality_enum') + + admin_enum = Enum('locked', + 'unlocked', + 'reserve1', + 'reserve2', + name='admin_enum') + + operational_enum = Enum('disabled', + 'enabled', + 'reserve1', + 'reserve2', + name='operational_enum') + + availability_enum = Enum('available', + 'intest', + 'degraded', + 'failed', + 'power-off', + 'offline', + 'offduty', + 'online', + 'dependency', + 'not-installed', + 'reserve1', + 'reserve2', + name='availability_enum') + + action_enum = Enum('none', + 'lock', + 'force-lock', + 'unlock', + 'reset', + 'swact', + 'force-swact', + 'reboot', + 'power-on', + 'power-off', + 'reinstall', + 'reserve1', + 'reserve2', + name='action_enum') + + hosts = Table( + 'hosts', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + + Column('hostname', String(255), unique=True, index=True), + Column('recordtype', recordtype_enum, default="standard"), + Column('reserved', Boolean), + + Column('invprovision', String(64), default="unprovisioned"), + + Column('mgmt_mac', String(255), unique=True), + Column('mgmt_ip', String(255), unique=True), + + # Board Management database members + Column('bm_ip', String(255)), + Column('bm_mac', String(255)), + Column('bm_type', String(255)), + Column('bm_username', String(255)), + + Column('personality', personality_enum), + Column('subfunctions', String(255)), + Column('subfunction_oper', String(255)), + Column('subfunction_avail', String(255)), + + Column('serialid', String(255)), + Column('location', Text), + Column('administrative', admin_enum, default="locked"), + Column('operational', operational_enum, default="disabled"), + Column('availability', availability_enum, default="offline"), + Column('action', action_enum, default="none"), + Column('host_action', String(255)), + Column('action_state', String(255)), + Column('mtce_info', String(255)), + Column('install_state', String(255)), + Column('install_state_info', String(255)), + Column('vim_progress_status', String(255)), + Column('task', String(64)), + Column('uptime', Integer), + Column('capabilities', Text), + + Column('boot_device', String(255)), + Column('rootfs_device', String(255)), + Column('install_output', String(255)), + Column('console', String(255)), + Column('tboot', String(64)), + Column('ttys_dcd', Boolean), + Column('iscsi_initiator_name', String(64)), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + nodes = Table( + 'nodes', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + + # numaNode from /sys/devices/system/node/nodeX/cpulist or cpumap + Column('numa_node', Integer), + Column('capabilities', Text), + + Column('host_id', Integer, + ForeignKey('hosts.id', ondelete='CASCADE')), + UniqueConstraint('numa_node', 'host_id', name='u_hostnuma'), + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + cpus = Table( + 'cpus', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + Column('cpu', Integer), + Column('core', Integer), + Column('thread', Integer), + Column('cpu_family', String(255)), + Column('cpu_model', String(255)), + Column('capabilities', Text), + + Column('host_id', Integer, + ForeignKey('hosts.id', ondelete='CASCADE')), + Column('node_id', Integer, + ForeignKey('nodes.id', ondelete='CASCADE')), + UniqueConstraint('cpu', 'host_id', name='u_hostcpu'), + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + memorys = Table( + 'memorys', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, + primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + + # per NUMA: /sys/devices/system/node/node/meminfo + Column('memtotal_mib', Integer), + Column('memavail_mib', Integer), + Column('platform_reserved_mib', Integer), + + Column('hugepages_configured', Boolean), # if hugepages_configured + + Column('vswitch_hugepages_size_mib', Integer), + Column('vswitch_hugepages_reqd', Integer), + Column('vswitch_hugepages_nr', Integer), + Column('vswitch_hugepages_avail', Integer), + + Column('vm_hugepages_nr_2M', Integer), + Column('vm_hugepages_nr_1G', Integer), + Column('vm_hugepages_use_1G', Boolean), + Column('vm_hugepages_possible_2M', Integer), + Column('vm_hugepages_possible_1G', Integer), + + Column('vm_hugepages_nr_2M_pending', Integer), # To be removed + Column('vm_hugepages_nr_1G_pending', Integer), # To be removed + Column('vm_hugepages_avail_2M', Integer), + Column('vm_hugepages_avail_1G', Integer), + + Column('vm_hugepages_nr_4K', Integer), + + Column('node_memtotal_mib', Integer), + + Column('capabilities', Text), + + # psql requires unique FK + Column('host_id', Integer, + ForeignKey('hosts.id', ondelete='CASCADE')), + Column('node_id', Integer, ForeignKey('nodes.id')), + UniqueConstraint('host_id', 'node_id', name='u_hostnode'), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + ports = Table( + 'ports', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + + Column('host_id', Integer, ForeignKey('hosts.id', + ondelete='CASCADE')), + Column('node_id', Integer, ForeignKey('nodes.id', + ondelete='SET NULL')), + + Column('type', String(255)), + Column('name', String(255)), + Column('namedisplay', String(255)), + Column('pciaddr', String(255)), + Column('dev_id', Integer), + Column('sriov_totalvfs', Integer), + Column('sriov_numvfs', Integer), + Column('sriov_vfs_pci_address', String(1020)), + Column('driver', String(255)), + + Column('pclass', String(255)), + Column('pvendor', String(255)), + Column('pdevice', String(255)), + Column('psvendor', String(255)), + Column('psdevice', String(255)), + Column('dpdksupport', Boolean, default=False), + Column('numa_node', Integer), + Column('capabilities', Text), + + UniqueConstraint('pciaddr', 'dev_id', 'host_id', + name='u_pciaddr_dev_host_id'), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + ethernet_ports = Table( + 'ethernet_ports', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, ForeignKey('ports.id', ondelete="CASCADE"), + primary_key=True, nullable=False), + + Column('mac', String(255)), + Column('mtu', Integer), + Column('speed', Integer), + Column('link_mode', String(255)), + Column('duplex', String(255)), + Column('autoneg', String(255)), + Column('bootp', String(255)), + Column('capabilities', Text), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + pci_devices = Table( + 'pci_devices', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(255), unique=True, index=True), + Column('host_id', Integer, ForeignKey('hosts.id', + ondelete='CASCADE')), + Column('name', String(255)), + Column('pciaddr', String(255)), + Column('pclass_id', String(6)), + Column('pvendor_id', String(4)), + Column('pdevice_id', String(4)), + Column('pclass', String(255)), + Column('pvendor', String(255)), + Column('pdevice', String(255)), + Column('psvendor', String(255)), + Column('psdevice', String(255)), + Column('numa_node', Integer), + Column('driver', String(255)), + Column('sriov_totalvfs', Integer), + Column('sriov_numvfs', Integer), + Column('sriov_vfs_pci_address', String(1020)), + Column('enabled', Boolean), + Column('extra_info', Text), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + lldp_agents = Table( + 'lldp_agents', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + Column('host_id', Integer, ForeignKey('hosts.id', + ondelete='CASCADE')), + Column('port_id', Integer, ForeignKey('ports.id', + ondelete='CASCADE')), + Column('status', String(255)), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + lldp_neighbours = Table( + 'lldp_neighbours', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + Column('host_id', Integer, ForeignKey('hosts.id', + ondelete='CASCADE')), + Column('port_id', Integer, ForeignKey('ports.id', + ondelete='CASCADE')), + + Column('msap', String(511), nullable=False), + + UniqueConstraint('msap', 'port_id', + name='u_msap_port_id'), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + lldp_tlvs = Table( + 'lldp_tlvs', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('agent_id', Integer, + ForeignKey('lldp_agents.id', ondelete="CASCADE"), + nullable=True), + Column('neighbour_id', Integer, + ForeignKey('lldp_neighbours.id', ondelete="CASCADE"), + nullable=True), + Column('type', String(255)), + Column('value', String(255)), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + sensorgroups = Table( + 'sensorgroups', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + + Column('uuid', String(36), unique=True), + Column('host_id', Integer, + ForeignKey('hosts.id', ondelete='CASCADE')), + + Column('sensorgroupname', String(255)), + Column('path', String(255)), + Column('datatype', String(255)), # polymorphic 'analog'/'discrete + Column('sensortype', String(255)), + Column('description', String(255)), + Column('state', String(255)), # enabled or disabled + Column('possible_states', String(255)), + Column('audit_interval_group', Integer), + Column('record_ttl', Integer), + + Column('algorithm', String(255)), + Column('actions_critical_choices', String(255)), + Column('actions_major_choices', String(255)), + Column('actions_minor_choices', String(255)), + Column('actions_minor_group', String(255)), + Column('actions_major_group', String(255)), + Column('actions_critical_group', String(255)), + + Column('suppress', Boolean), # True, disables the action + + Column('capabilities', Text), + + UniqueConstraint('sensorgroupname', 'path', 'host_id', + name='u_sensorgroupname_path_hostid'), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + # polymorphic on datatype 'discrete' + sensorgroups_discrete = Table( + 'sensorgroups_discrete', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, + ForeignKey('sensorgroups.id', ondelete="CASCADE"), + primary_key=True, nullable=False), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + # polymorphic on datatype 'analog' + sensorgroups_analog = Table( + 'sensorgroups_analog', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, + ForeignKey('sensorgroups.id', ondelete="CASCADE"), + primary_key=True, nullable=False), + + Column('unit_base_group', String(255)), # revolutions + Column('unit_modifier_group', String(255)), # 100 + Column('unit_rate_group', String(255)), # minute + + Column('t_minor_lower_group', String(255)), + Column('t_minor_upper_group', String(255)), + Column('t_major_lower_group', String(255)), + Column('t_major_upper_group', String(255)), + Column('t_critical_lower_group', String(255)), + Column('t_critical_upper_group', String(255)), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + sensors = Table( + 'sensors', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, primary_key=True, nullable=False), + Column('uuid', String(36), unique=True), + + Column('host_id', Integer, + ForeignKey('hosts.id', ondelete='CASCADE')), + + Column('sensorgroup_id', Integer, + ForeignKey('sensorgroups.id', ondelete='SET NULL')), + + Column('sensorname', String(255)), + Column('path', String(255)), + + Column('datatype', String(255)), # polymorphic on datatype + Column('sensortype', String(255)), + + Column('status', String(255)), # ok, minor, major, critical, disabled + Column('state', String(255)), # enabled, disabled + Column('state_requested', String(255)), + + Column('sensor_action_requested', String(255)), + + Column('audit_interval', Integer), + Column('algorithm', String(255)), + Column('actions_minor', String(255)), + Column('actions_major', String(255)), + Column('actions_critical', String(255)), + + Column('suppress', Boolean), # True, disables the action + + Column('capabilities', Text), + + UniqueConstraint('sensorname', 'path', 'host_id', + name='u_sensorname_path_host_id'), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + # discrete sensor + sensors_discrete = Table( + 'sensors_discrete', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, + ForeignKey('sensors.id', ondelete="CASCADE"), + primary_key=True, nullable=False), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + # analog sensor + sensors_analog = Table( + 'sensors_analog', + meta, + Column('created_at', DateTime), + Column('updated_at', DateTime), + Column('deleted_at', DateTime), + Column('id', Integer, + ForeignKey('sensors.id', ondelete="CASCADE"), + primary_key=True, nullable=False), + + Column('unit_base', String(255)), # revolutions + Column('unit_modifier', String(255)), # 10^2 + Column('unit_rate', String(255)), # minute + + Column('t_minor_lower', String(255)), + Column('t_minor_upper', String(255)), + Column('t_major_lower', String(255)), + Column('t_major_upper', String(255)), + Column('t_critical_lower', String(255)), + Column('t_critical_upper', String(255)), + + mysql_engine=ENGINE, + mysql_charset=CHARSET, + ) + + # TODO(sc) disks + tables = ( + systems, + hosts, + nodes, + cpus, + memorys, + pci_devices, + ports, + ethernet_ports, + lldp_agents, + lldp_neighbours, + lldp_tlvs, + sensorgroups, + sensorgroups_discrete, + sensorgroups_analog, + sensors, + sensors_discrete, + sensors_analog, + ) + + for index, table in enumerate(tables): + try: + table.create() + except Exception: + # If an error occurs, drop all tables created so far to return + # to the previously existing state. + meta.drop_all(tables=tables[:index]) + raise diff --git a/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/__init__.py b/inventory/inventory/inventory/db/sqlalchemy/migrate_repo/versions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/db/sqlalchemy/migration.py b/inventory/inventory/inventory/db/sqlalchemy/migration.py new file mode 100644 index 00000000..24e8292f --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/migration.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import os +import sqlalchemy + +from migrate import exceptions as versioning_exceptions +from migrate.versioning import api as versioning_api +from migrate.versioning.repository import Repository +from oslo_db.sqlalchemy import enginefacade + +from inventory.common import exception +from inventory.common.i18n import _ +from inventory.db import migration + +_REPOSITORY = None + +get_engine = enginefacade.get_legacy_facade().get_engine + + +def db_sync(version=None): + if version is not None: + try: + version = int(version) + except ValueError: + raise exception.ApiError(_("version should be an integer")) + + current_version = db_version() + repository = _find_migrate_repo() + if version is None or version > current_version: + return versioning_api.upgrade(get_engine(), repository, version) + else: + return versioning_api.downgrade(get_engine(), repository, + version) + + +def db_version(): + repository = _find_migrate_repo() + try: + return versioning_api.db_version(get_engine(), repository) + except versioning_exceptions.DatabaseNotControlledError: + meta = sqlalchemy.MetaData() + engine = get_engine() + meta.reflect(bind=engine) + tables = meta.tables + if len(tables) == 0: + db_version_control(migration.INIT_VERSION) + return versioning_api.db_version(get_engine(), repository) + + +def db_version_control(version=None): + repository = _find_migrate_repo() + versioning_api.version_control(get_engine(), repository, version) + return version + + +def _find_migrate_repo(): + """Get the path for the migrate repository.""" + global _REPOSITORY + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'migrate_repo') + assert os.path.exists(path) + if _REPOSITORY is None: + _REPOSITORY = Repository(path) + return _REPOSITORY diff --git a/inventory/inventory/inventory/db/sqlalchemy/models.py b/inventory/inventory/inventory/db/sqlalchemy/models.py new file mode 100644 index 00000000..ee748d78 --- /dev/null +++ b/inventory/inventory/inventory/db/sqlalchemy/models.py @@ -0,0 +1,589 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# -*- encoding: utf-8 -*- +# +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import json +import urlparse + +from oslo_config import cfg +from oslo_db.sqlalchemy import models +from sqlalchemy import Column, Enum, ForeignKey, Integer, Boolean +from sqlalchemy import UniqueConstraint, String, Text +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy.types import TypeDecorator, VARCHAR +from sqlalchemy.orm import relationship, backref + + +def table_args(): + engine_name = urlparse.urlparse(cfg.CONF.database_connection).scheme + if engine_name == 'mysql': + return {'mysql_engine': 'InnoDB', + 'mysql_charset': "utf8"} + return None + + +class JSONEncodedDict(TypeDecorator): + """Represents an immutable structure as a json-encoded string.""" + + impl = VARCHAR + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + +class InventoryBase(models.TimestampMixin, models.ModelBase): + + metadata = None + + def as_dict(self): + d = {} + for c in self.__table__.columns: + d[c.name] = self[c.name] + return d + + +Base = declarative_base(cls=InventoryBase) + + +class Systems(Base): + __tablename__ = 'systems' + + # The reference for system is from systemconfig + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36), unique=True) + + name = Column(String(255), unique=True) + system_type = Column(String(255)) + system_mode = Column(String(255)) + description = Column(String(255)) + capabilities = Column(JSONEncodedDict) + contact = Column(String(255)) + location = Column(String(255)) + services = Column(Integer, default=72) + software_version = Column(String(255)) + timezone = Column(String(255)) + security_profile = Column(String(255)) + region_name = Column(Text) + service_project_name = Column(Text) + distributed_cloud_role = Column(String(255)) + security_feature = Column(String(255)) + + +class Hosts(Base): + recordTypeEnum = Enum('standard', + 'profile', + 'sprofile', + 'reserve1', + 'reserve2', + name='recordtypeEnum') + + adminEnum = Enum('locked', + 'unlocked', + 'reserve1', + 'reserve2', + name='administrativeEnum') + + operEnum = Enum('disabled', + 'enabled', + 'reserve1', + 'reserve2', + name='operationalEnum') + + availEnum = Enum('available', + 'intest', + 'degraded', + 'failed', + 'power-off', + 'offline', + 'offduty', + 'online', + 'dependency', + 'not-installed', + 'reserv1', + 'reserve2', + name='availabilityEnum') + + actionEnum = Enum('none', + 'lock', + 'force-lock', + 'unlock', + 'reset', + 'swact', + 'force-swact', + 'reboot', + 'power-on', + 'power-off', + 'reinstall', + 'reserve1', + 'reserve2', + name='actionEnum') + + __tablename__ = 'hosts' + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36), unique=True) + + hostname = Column(String(255), unique=True, index=True) + recordtype = Column(recordTypeEnum, default="standard") + reserved = Column(Boolean, default=False) + + invprovision = Column(String(64), default="unprovisioned") + + mgmt_mac = Column(String(255), unique=True) + mgmt_ip = Column(String(255)) + + # board management IP address, MAC, type and username + bm_ip = Column(String(255)) + bm_mac = Column(String(255)) + bm_type = Column(String(255)) + bm_username = Column(String(255)) + + personality = Column(String(255)) + subfunctions = Column(String(255)) + subfunction_oper = Column(operEnum, default="disabled") + subfunction_avail = Column(availEnum, default="not-installed") + serialid = Column(String(255)) + location = Column(JSONEncodedDict) + administrative = Column(adminEnum, default="locked") + operational = Column(operEnum, default="disabled") + availability = Column(availEnum, default="offline") + action = Column(actionEnum, default="none") + host_action = Column(String(255)) + action_state = Column(String(255)) + mtce_info = Column(String(255)) + install_state = Column(String(255)) + install_state_info = Column(String(255)) + vim_progress_status = Column(String(255)) + task = Column(String(64)) + uptime = Column(Integer, default=0) + capabilities = Column(JSONEncodedDict) + + boot_device = Column(String(255), default="sda") + rootfs_device = Column(String(255), default="sda") + install_output = Column(String(255), default="text") + console = Column(String(255), default="ttyS0,115200") + tboot = Column(String(64), default="") + ttys_dcd = Column(Boolean) + iscsi_initiator_name = Column(String(64)) + + +class Nodes(Base): + __tablename__ = 'nodes' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36), unique=True) + + numa_node = Column(Integer) + capabilities = Column(JSONEncodedDict) + + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + host = relationship("Hosts", + backref="nodes", lazy="joined", join_depth=1) + + UniqueConstraint('numa_node', 'host_id', name='u_hostnuma') + + +class Cpus(Base): + __tablename__ = 'cpus' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36), unique=True) + + cpu = Column(Integer) + core = Column(Integer) + thread = Column(Integer) + cpu_family = Column(String(255)) + cpu_model = Column(String(255)) + # allocated_function = Column(String(255)) # systemconfig allocates + capabilities = Column(JSONEncodedDict) + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + node_id = Column(Integer, ForeignKey('nodes.id', ondelete='CASCADE')) + + host = relationship("Hosts", + backref="cpus", lazy="joined", join_depth=1) + node = relationship("Nodes", + backref="cpus", lazy="joined", join_depth=1) + + UniqueConstraint('cpu', 'host_id', name='u_hostcpu') + + +class Memorys(Base): + __tablename__ = 'memorys' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36), unique=True) + + memtotal_mib = Column(Integer) + memavail_mib = Column(Integer) + platform_reserved_mib = Column(Integer) + node_memtotal_mib = Column(Integer) + + hugepages_configured = Column(Boolean, default=False) + + vswitch_hugepages_size_mib = Column(Integer) + vswitch_hugepages_reqd = Column(Integer) + vswitch_hugepages_nr = Column(Integer) + vswitch_hugepages_avail = Column(Integer) + + vm_hugepages_nr_2M_pending = Column(Integer) + vm_hugepages_nr_1G_pending = Column(Integer) + vm_hugepages_nr_2M = Column(Integer) + vm_hugepages_nr_1G = Column(Integer) + vm_hugepages_nr_4K = Column(Integer) + vm_hugepages_avail_2M = Column(Integer) + vm_hugepages_avail_1G = Column(Integer) + + vm_hugepages_use_1G = Column(Boolean, default=False) + vm_hugepages_possible_2M = Column(Integer) + vm_hugepages_possible_1G = Column(Integer) + capabilities = Column(JSONEncodedDict) + + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + node_id = Column(Integer, ForeignKey('nodes.id')) + + host = relationship("Hosts", backref="memory", lazy="joined", join_depth=1) + node = relationship("Nodes", backref="memory", lazy="joined", join_depth=1) + + UniqueConstraint('host_id', 'node_id', name='u_hostnode') + + +class Ports(Base): + __tablename__ = 'ports' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36)) + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + node_id = Column(Integer, ForeignKey('nodes.id')) + + type = Column(String(255)) + + name = Column(String(255)) + namedisplay = Column(String(255)) + pciaddr = Column(String(255)) + pclass = Column(String(255)) + pvendor = Column(String(255)) + pdevice = Column(String(255)) + psvendor = Column(String(255)) + psdevice = Column(String(255)) + dpdksupport = Column(Boolean, default=False) + numa_node = Column(Integer) + dev_id = Column(Integer) + sriov_totalvfs = Column(Integer) + sriov_numvfs = Column(Integer) + # Each PCI Address is 12 char, 1020 char is enough for 64 devices + sriov_vfs_pci_address = Column(String(1020)) + driver = Column(String(255)) + capabilities = Column(JSONEncodedDict) + + node = relationship("Nodes", backref="ports", lazy="joined", join_depth=1) + host = relationship("Hosts", backref="ports", lazy="joined", join_depth=1) + + UniqueConstraint('pciaddr', 'dev_id', 'host_id', name='u_pciaddrdevhost') + + __mapper_args__ = { + 'polymorphic_identity': 'port', + 'polymorphic_on': type + } + + +class EthernetPorts(Ports): + __tablename__ = 'ethernet_ports' + + id = Column(Integer, + ForeignKey('ports.id'), primary_key=True, nullable=False) + + mac = Column(String(255), unique=True) + mtu = Column(Integer) + speed = Column(Integer) + link_mode = Column(String(255)) + duplex = Column(String(255)) + autoneg = Column(String(255)) + bootp = Column(String(255)) + + UniqueConstraint('mac', name='u_mac') + + __mapper_args__ = { + 'polymorphic_identity': 'ethernet' + } + + +class LldpAgents(Base): + __tablename__ = 'lldp_agents' + + id = Column('id', Integer, primary_key=True, nullable=False) + uuid = Column('uuid', String(36)) + host_id = Column('host_id', Integer, ForeignKey('hosts.id', + ondelete='CASCADE')) + port_id = Column('port_id', Integer, ForeignKey('ports.id', + ondelete='CASCADE')) + status = Column('status', String(255)) + + lldp_tlvs = relationship("LldpTlvs", + backref=backref("lldpagents", lazy="subquery"), + cascade="all") + + host = relationship("Hosts", lazy="joined", join_depth=1) + port = relationship("Ports", lazy="joined", join_depth=1) + + +class PciDevices(Base): + __tablename__ = 'pci_devices' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36)) + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + name = Column(String(255)) + pciaddr = Column(String(255)) + pclass_id = Column(String(6)) + pvendor_id = Column(String(4)) + pdevice_id = Column(String(4)) + pclass = Column(String(255)) + pvendor = Column(String(255)) + pdevice = Column(String(255)) + psvendor = Column(String(255)) + psdevice = Column(String(255)) + numa_node = Column(Integer) + sriov_totalvfs = Column(Integer) + sriov_numvfs = Column(Integer) + sriov_vfs_pci_address = Column(String(1020)) + driver = Column(String(255)) + enabled = Column(Boolean) + extra_info = Column(Text) + + host = relationship("Hosts", lazy="joined", join_depth=1) + + UniqueConstraint('pciaddr', 'host_id', name='u_pciaddrhost') + + +class LldpNeighbours(Base): + __tablename__ = 'lldp_neighbours' + + id = Column('id', Integer, primary_key=True, nullable=False) + uuid = Column('uuid', String(36)) + host_id = Column('host_id', Integer, ForeignKey('hosts.id', + ondelete='CASCADE')) + port_id = Column('port_id', Integer, ForeignKey('ports.id', + ondelete='CASCADE')) + msap = Column('msap', String(511)) + + lldp_tlvs = relationship( + "LldpTlvs", + backref=backref("lldpneighbours", lazy="subquery"), + cascade="all") + + host = relationship("Hosts", lazy="joined", join_depth=1) + port = relationship("Ports", lazy="joined", join_depth=1) + + UniqueConstraint('msap', 'port_id', name='u_msap_port_id') + + +class LldpTlvs(Base): + __tablename__ = 'lldp_tlvs' + + id = Column('id', Integer, primary_key=True, nullable=False) + agent_id = Column('agent_id', Integer, ForeignKey('lldp_agents.id', + ondelete='CASCADE'), nullable=True) + neighbour_id = Column('neighbour_id', Integer, + ForeignKey('lldp_neighbours.id', ondelete='CASCADE'), + nullable=True) + type = Column('type', String(255)) + value = Column('value', String(255)) + + lldp_agent = relationship("LldpAgents", + backref=backref("lldptlvs", lazy="subquery"), + cascade="all", + lazy="joined") + + lldp_neighbour = relationship( + "LldpNeighbours", + backref=backref("lldptlvs", lazy="subquery"), + cascade="all", + lazy="joined") + + UniqueConstraint('type', 'agent_id', + name='u_type@agent') + + UniqueConstraint('type', 'neighbour_id', + name='u_type@neighbour') + + +class SensorGroups(Base): + __tablename__ = 'sensorgroups' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36)) + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + + sensortype = Column(String(255)) + datatype = Column(String(255)) # polymorphic + sensorgroupname = Column(String(255)) + path = Column(String(255)) + description = Column(String(255)) + + state = Column(String(255)) + possible_states = Column(String(255)) + algorithm = Column(String(255)) + audit_interval_group = Column(Integer) + record_ttl = Column(Integer) + + actions_minor_group = Column(String(255)) + actions_major_group = Column(String(255)) + actions_critical_group = Column(String(255)) + + suppress = Column(Boolean, default=False) + + capabilities = Column(JSONEncodedDict) + + actions_critical_choices = Column(String(255)) + actions_major_choices = Column(String(255)) + actions_minor_choices = Column(String(255)) + + host = relationship("Hosts", lazy="joined", join_depth=1) + + UniqueConstraint('sensorgroupname', 'path', 'host_id', + name='u_sensorgroupname_path_host_id') + + __mapper_args__ = { + 'polymorphic_identity': 'sensorgroup', + 'polymorphic_on': datatype + } + + +class SensorGroupsCommon(object): + @declared_attr + def id(cls): + return Column(Integer, + ForeignKey('sensorgroups.id', ondelete="CASCADE"), + primary_key=True, nullable=False) + + +class SensorGroupsDiscrete(SensorGroupsCommon, SensorGroups): + __tablename__ = 'sensorgroups_discrete' + + __mapper_args__ = { + 'polymorphic_identity': 'discrete', + } + + +class SensorGroupsAnalog(SensorGroupsCommon, SensorGroups): + __tablename__ = 'sensorgroups_analog' + + unit_base_group = Column(String(255)) + unit_modifier_group = Column(String(255)) + unit_rate_group = Column(String(255)) + + t_minor_lower_group = Column(String(255)) + t_minor_upper_group = Column(String(255)) + t_major_lower_group = Column(String(255)) + t_major_upper_group = Column(String(255)) + t_critical_lower_group = Column(String(255)) + t_critical_upper_group = Column(String(255)) + + __mapper_args__ = { + 'polymorphic_identity': 'analog', + } + + +class Sensors(Base): + __tablename__ = 'sensors' + + id = Column(Integer, primary_key=True, nullable=False) + uuid = Column(String(36)) + host_id = Column(Integer, ForeignKey('hosts.id', ondelete='CASCADE')) + + sensorgroup_id = Column(Integer, + ForeignKey('sensorgroups.id', + ondelete='SET NULL')) + sensortype = Column(String(255)) # "watchdog", "temperature". + datatype = Column(String(255)) # "discrete" or "analog" + + sensorname = Column(String(255)) + path = Column(String(255)) + + status = Column(String(255)) + state = Column(String(255)) + state_requested = Column(String(255)) + + sensor_action_requested = Column(String(255)) + + audit_interval = Column(Integer) + algorithm = Column(String(255)) + actions_minor = Column(String(255)) + actions_major = Column(String(255)) + actions_critical = Column(String(255)) + + suppress = Column(Boolean, default=False) + + capabilities = Column(JSONEncodedDict) + + host = relationship("Hosts", lazy="joined", join_depth=1) + sensorgroup = relationship("SensorGroups", lazy="joined", join_depth=1) + + UniqueConstraint('sensorname', 'path', 'host_id', + name='u_sensorname_path_host_id') + + __mapper_args__ = { + 'polymorphic_identity': 'sensor', + 'polymorphic_on': datatype + # with_polymorphic is only supported in sqlalchemy.orm >= 0.8 + # 'with_polymorphic': '*' + } + + +class SensorsDiscrete(Sensors): + __tablename__ = 'sensors_discrete' + + id = Column(Integer, ForeignKey('sensors.id'), + primary_key=True, nullable=False) + + __mapper_args__ = { + 'polymorphic_identity': 'discrete' + } + + +class SensorsAnalog(Sensors): + __tablename__ = 'sensors_analog' + + id = Column(Integer, ForeignKey('sensors.id'), + primary_key=True, nullable=False) + + unit_base = Column(String(255)) + unit_modifier = Column(String(255)) + unit_rate = Column(String(255)) + + t_minor_lower = Column(String(255)) + t_minor_upper = Column(String(255)) + t_major_lower = Column(String(255)) + t_major_upper = Column(String(255)) + t_critical_lower = Column(String(255)) + t_critical_upper = Column(String(255)) + + __mapper_args__ = { + 'polymorphic_identity': 'analog' + } diff --git a/inventory/inventory/inventory/db/utils.py b/inventory/inventory/inventory/db/utils.py new file mode 100644 index 00000000..d8f18d42 --- /dev/null +++ b/inventory/inventory/inventory/db/utils.py @@ -0,0 +1,48 @@ +# Copyright (c) 2015 Ericsson AB. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class LazyPluggable(object): + """A pluggable backend loaded lazily based on some value.""" + + def __init__(self, pivot, **backends): + self.__backends = backends + self.__pivot = pivot + self.__backend = None + + def __get_backend(self): + if not self.__backend: + backend_name = 'sqlalchemy' + backend = self.__backends[backend_name] + if isinstance(backend, tuple): + name = backend[0] + fromlist = backend[1] + else: + name = backend + fromlist = backend + + self.__backend = __import__(name, None, None, fromlist) + return self.__backend + + def __getattr__(self, key): + backend = self.__get_backend() + return getattr(backend, key) + + +IMPL = LazyPluggable('backend', sqlalchemy='inventory.db.sqlalchemy.api') + + +def purge_deleted(age, granularity='days'): + IMPL.purge_deleted(age, granularity) diff --git a/inventory/inventory/inventory/objects/__init__.py b/inventory/inventory/inventory/objects/__init__.py new file mode 100644 index 00000000..213fa432 --- /dev/null +++ b/inventory/inventory/inventory/objects/__init__.py @@ -0,0 +1,43 @@ +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(comstud): You may scratch your head as you see code that imports +# this module and then accesses attributes for objects such as Node, +# etc, yet you do not see these attributes in here. Never fear, there is +# a little bit of magic. When objects are registered, an attribute is set +# on this module automatically, pointing to the newest/latest version of +# the object. + + +def register_all(): + # NOTE(danms): You must make sure your object gets imported in this + # function in order for it to be registered by services that may + # need to receive it via RPC. + __import__('inventory.objects.cpu') + __import__('inventory.objects.host') + __import__('inventory.objects.lldp_agent') + __import__('inventory.objects.lldp_neighbour') + __import__('inventory.objects.lldp_tlv') + __import__('inventory.objects.memory') + __import__('inventory.objects.node') + __import__('inventory.objects.pci_device') + __import__('inventory.objects.port_ethernet') + __import__('inventory.objects.port') + __import__('inventory.objects.sensor_analog') + __import__('inventory.objects.sensor_discrete') + __import__('inventory.objects.sensorgroup_analog') + __import__('inventory.objects.sensorgroup_discrete') + __import__('inventory.objects.sensorgroup') + __import__('inventory.objects.sensor') + __import__('inventory.objects.system') diff --git a/inventory/inventory/inventory/objects/base.py b/inventory/inventory/inventory/objects/base.py new file mode 100644 index 00000000..173f0a3e --- /dev/null +++ b/inventory/inventory/inventory/objects/base.py @@ -0,0 +1,345 @@ +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventory import objects +from oslo_log import log +from oslo_utils import versionutils +from oslo_versionedobjects import base as object_base +# from oslo_versionedobjects import exception as ovo_exception + +from oslo_versionedobjects import fields as object_fields +LOG = log.getLogger(__name__) + + +class InventoryObjectRegistry(object_base.VersionedObjectRegistry): + def registration_hook(self, cls, index): + # NOTE(jroll): blatantly stolen from nova + # NOTE(danms): This is called when an object is registered, + # and is responsible for maintaining inventory.objects.$OBJECT + # as the highest-versioned implementation of a given object. + version = versionutils.convert_version_to_tuple(cls.VERSION) + if not hasattr(objects, cls.obj_name()): # noqa + setattr(objects, cls.obj_name(), cls) # noqa + else: + cur_version = versionutils.convert_version_to_tuple( + getattr(objects, cls.obj_name()).VERSION) # noqa + if version >= cur_version: + setattr(objects, cls.obj_name(), cls) # noqa + + +class InventoryObject(object_base.VersionedObject): + """Base class and object factory. + + This forms the base of all objects that can be remoted or instantiated + via RPC. Simply defining a class that inherits from this base class + will make it remotely instantiatable. Objects should implement the + necessary "get" classmethod routines as well as "save" object methods + as appropriate. + """ + + OBJ_SERIAL_NAMESPACE = 'inventory_object' + OBJ_PROJECT_NAMESPACE = 'inventory' + + fields = { + 'created_at': object_fields.DateTimeField(nullable=True), + 'updated_at': object_fields.DateTimeField(nullable=True), + } + + _foreign_fields = {} + _optional_fields = [] + + def _get_foreign_field(self, field, db_object): + """Retrieve data from a foreign relationship on a DB entry. + + Depending on how the field was described in _foreign_fields the data + may be retrieved by calling a function to do the work, or by accessing + the specified remote field name if specified as a string. + """ + accessor = self._foreign_fields[field] + if callable(accessor): + return accessor(field, db_object) + + # Split as "local object reference:remote field name" + local, remote = accessor.split(':') + try: + local_object = db_object[local] + if local_object: + return local_object[remote] + except KeyError: + pass # foreign relationships are not always available + return None + + def __getitem__(self, name): + return getattr(self, name) + + def __setitem__(self, name, value): + setattr(self, name, value) + + def as_dict(self): + return dict((k, getattr(self, k)) + for k in self.fields + if hasattr(self, k)) + + @classmethod + def get_defaults(cls): + """Return a dict of its fields with their default value.""" + return dict((k, v(None)) + for k, v in cls.fields.iteritems() + if k != "id" and callable(v)) + + def get(self, key, value=None): + """For backwards-compatibility with dict-based objects. + + NOTE(danms): May be removed in the future. + """ + return self[key] + + def _set_from_db_object(self, context, cls_object, db_object, fields): + """Sets object fields. + + :param context: security context + :param db_object: A DB entity of the object + :param fields: list of fields to set on obj from values from db_object. + """ + + for field in cls_object.fields: + if field in cls_object._optional_fields: + if not hasattr(db_object, field): + continue + + if field in cls_object._foreign_fields: + setattr(self, field, + cls_object._get_foreign_field(field, db_object)) + continue + + setattr(self, field, db_object[field]) + # cls_object[field] = db_object[field] + + @staticmethod + def _from_db_object(context, obj, db_object, fields=None): + """Converts a database entity to a formal object. + + This always converts the database entity to the latest version + of the object. Note that the latest version is available at + object.__class__.VERSION. object.VERSION is the version of this + particular object instance; it is possible that it is not the latest + version. + + :param context: security context + :param obj: An object of the class. + :param db_object: A DB entity of the object + :param fields: list of fields to set on obj from values from db_object. + :return: The object of the class with the database entity added + :raises: ovo_exception.IncompatibleObjectVersion + """ + # objname = obj.obj_name() + # db_version = db_object['version'] + + # if not versionutils.is_compatible(db_version, obj.__class__.VERSION): + # raise ovo_exception.IncompatibleObjectVersion( + # objname=objname, objver=db_version, + # supported=obj.__class__.VERSION) + + obj._set_from_db_object(context, obj, db_object, fields) + + obj._context = context + + # NOTE(rloo). We now have obj, a versioned object that corresponds to + # its DB representation. A versioned object has an internal attribute + # ._changed_fields; this is a list of changed fields -- used, e.g., + # when saving the object to the DB (only those changed fields are + # saved to the DB). The obj.obj_reset_changes() clears this list + # since we didn't actually make any modifications to the object that + # we want saved later. + obj.obj_reset_changes() + + # if db_version != obj.__class__.VERSION: + # # convert to the latest version + # obj.VERSION = db_version + # obj.convert_to_version(obj.__class__.VERSION, + # remove_unavailable_fields=False) + return obj + + def _convert_to_version(self, target_version, + remove_unavailable_fields=True): + """Convert to the target version. + + Subclasses should redefine this method, to do the conversion of the + object to the target version. + + Convert the object to the target version. The target version may be + the same, older, or newer than the version of the object. This is + used for DB interactions as well as for serialization/deserialization. + + The remove_unavailable_fields flag is used to distinguish these two + cases: + + 1) For serialization/deserialization, we need to remove the unavailable + fields, because the service receiving the object may not know about + these fields. remove_unavailable_fields is set to True in this case. + + 2) For DB interactions, we need to set the unavailable fields to their + appropriate values so that these fields are saved in the DB. (If + they are not set, the VersionedObject magic will not know to + save/update them to the DB.) remove_unavailable_fields is set to + False in this case. + + :param target_version: the desired version of the object + :param remove_unavailable_fields: True to remove fields that are + unavailable in the target version; set this to True when + (de)serializing. False to set the unavailable fields to appropriate + values; set this to False for DB interactions. + """ + pass + + def convert_to_version(self, target_version, + remove_unavailable_fields=True): + """Convert this object to the target version. + + Convert the object to the target version. The target version may be + the same, older, or newer than the version of the object. This is + used for DB interactions as well as for serialization/deserialization. + + The remove_unavailable_fields flag is used to distinguish these two + cases: + + 1) For serialization/deserialization, we need to remove the unavailable + fields, because the service receiving the object may not know about + these fields. remove_unavailable_fields is set to True in this case. + + 2) For DB interactions, we need to set the unavailable fields to their + appropriate values so that these fields are saved in the DB. (If + they are not set, the VersionedObject magic will not know to + save/update them to the DB.) remove_unavailable_fields is set to + False in this case. + + _convert_to_version() does the actual work. + + :param target_version: the desired version of the object + :param remove_unavailable_fields: True to remove fields that are + unavailable in the target version; set this to True when + (de)serializing. False to set the unavailable fields to appropriate + values; set this to False for DB interactions. + """ + if self.VERSION != target_version: + self._convert_to_version( + target_version, + remove_unavailable_fields=remove_unavailable_fields) + if remove_unavailable_fields: + # NOTE(rloo): We changed the object, but don't keep track of + # any of these changes, since it is inaccurate anyway (because + # it doesn't keep track of any 'changed' unavailable fields). + self.obj_reset_changes() + + # NOTE(rloo): self.__class__.VERSION is the latest version that + # is supported by this service. self.VERSION is the version of + # this object instance -- it may get set via e.g. the + # serialization or deserialization process, or here. + if (self.__class__.VERSION != target_version or + self.VERSION != self.__class__.VERSION): + self.VERSION = target_version + + @classmethod + def get_target_version(cls): + return cls.VERSION + + def do_version_changes_for_db(self): + """Change the object to the version needed for the database. + + If needed, this changes the object (modifies object fields) to be in + the correct version for saving to the database. + + The version used to save the object in the DB is determined as follows: + + * If the object is pinned, we save the object in the pinned version. + Since it is pinned, we must not save in a newer version, in case + a rolling upgrade is happening and some services are still using the + older version of inventory, with no knowledge of this newer version. + * If the object isn't pinned, we save the object in the latest version. + + Because the object may be converted to a different object version, this + method must only be called just before saving the object to the DB. + + :returns: a dictionary of changed fields and their new values + (could be an empty dictionary). These are the fields/values + of the object that would be saved to the DB. + """ + target_version = self.get_target_version() + + if target_version != self.VERSION: + # Convert the object so we can save it in the target version. + self.convert_to_version(target_version, + remove_unavailable_fields=False) + + changes = self.obj_get_changes() + # NOTE(rloo): Since this object doesn't keep track of the version that + # is saved in the DB and we don't want to make a DB call + # just to find out, we always update 'version' in the DB. + changes['version'] = self.VERSION + + return changes + + @classmethod + def _from_db_object_list(cls, context, db_objects): + """Returns objects corresponding to database entities. + + Returns a list of formal objects of this class that correspond to + the list of database entities. + + :param cls: the VersionedObject class of the desired object + :param context: security context + :param db_objects: A list of DB models of the object + :returns: A list of objects corresponding to the database entities + """ + return [cls._from_db_object(context, cls(), db_obj) + for db_obj in db_objects] + + def save(self, context=None): + updates = {} + changes = self.do_version_changes_for_db() + + for field in changes: + if field == 'version': + continue + updates[field] = self[field] + + self.save_changes(context, updates) + self.obj_reset_changes() + + +class InventoryObjectSerializer(object_base.VersionedObjectSerializer): + # Base class to use for object hydration + OBJ_BASE_CLASS = InventoryObject + + +def obj_to_primitive(obj): + """Recursively turn an object into a python primitive. + + An InventoryObject becomes a dict, and anything that implements + ObjectListBase becomes a list. + """ + if isinstance(obj, object_base.ObjectListBase): + return [obj_to_primitive(x) for x in obj] + elif isinstance(obj, InventoryObject): + result = {} + for key, value in obj.iteritems(): + result[key] = obj_to_primitive(value) + return result + else: + return obj diff --git a/inventory/inventory/inventory/objects/cpu.py b/inventory/inventory/inventory/objects/cpu.py new file mode 100644 index 00000000..bf187bf9 --- /dev/null +++ b/inventory/inventory/inventory/objects/cpu.py @@ -0,0 +1,119 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class CPU(base.InventoryObject, object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(), + 'uuid': object_fields.StringField(nullable=True), + 'host_id': object_fields.IntegerField(), + 'host_uuid': object_fields.StringField(nullable=True), + 'node_id': object_fields.IntegerField(nullable=True), + 'node_uuid': object_fields.StringField(nullable=True), + 'numa_node': object_fields.IntegerField(nullable=True), + 'cpu': object_fields.IntegerField(), + 'core': object_fields.IntegerField(nullable=True), + 'thread': object_fields.IntegerField(nullable=True), + 'cpu_family': object_fields.StringField(nullable=True), + 'cpu_model': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + # part of config + # 'allocated_function': object_fields.StringField(nullable=True), + } + + _foreign_fields = {'host_uuid': 'host:uuid', + 'node_uuid': 'node:uuid', + 'numa_node': 'node:numa_node'} + + @classmethod + def get_by_uuid(cls, context, uuid): + db_cpu = cls.dbapi.cpu_get(uuid) + return cls._from_db_object(context, cls(), db_cpu) + + def save_changes(self, context, updates): + self.dbapi.cpu_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of CPU objects. + + :param cls: the :class:`CPU` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`CPU` object. + + """ + db_cpus = cls.dbapi.cpu_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_cpus) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_cpus = cls.dbapi.cpu_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_cpus) + + @classmethod + def get_by_node(cls, context, node_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_cpus = cls.dbapi.cpu_get_by_node( + node_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_cpus) + + @classmethod + def get_by_host_node(cls, context, host_uuid, node_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_cpus = cls.dbapi.cpu_get_by_host_node( + host_uuid, + node_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_cpus) + + def create(self, context=None): + """Create a CPU record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + """ + values = self.do_version_changes_for_db() + db_cpu = self.dbapi.cpu_create(values) + return self._from_db_object(self._context, self, db_cpu) diff --git a/inventory/inventory/inventory/objects/fields.py b/inventory/inventory/inventory/objects/fields.py new file mode 100644 index 00000000..d85c5cae --- /dev/null +++ b/inventory/inventory/inventory/objects/fields.py @@ -0,0 +1,160 @@ +# Copyright 2015 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import ast +import hashlib +import inspect + +from oslo_versionedobjects import fields as object_fields +import six + +from inventory.common import utils + + +class IntegerField(object_fields.IntegerField): + pass + + +class UUIDField(object_fields.UUIDField): + pass + + +class StringField(object_fields.StringField): + pass + + +class StringAcceptsCallable(object_fields.String): + @staticmethod + def coerce(obj, attr, value): + if callable(value): + value = value() + return super(StringAcceptsCallable, StringAcceptsCallable).coerce( + obj, attr, value) + + +class StringFieldThatAcceptsCallable(object_fields.StringField): + """Custom StringField object that allows for functions as default + + In some cases we need to allow for dynamic defaults based on configuration + options, this StringField object allows for a function to be passed as a + default, and will only process it at the point the field is coerced + """ + + AUTO_TYPE = StringAcceptsCallable() + + def __repr__(self): + default = self._default + if (self._default != object_fields.UnspecifiedDefault and + callable(self._default)): + default = "%s-%s" % ( + self._default.__name__, + hashlib.md5(inspect.getsource( + self._default).encode()).hexdigest()) + return '%s(default=%s,nullable=%s)' % (self._type.__class__.__name__, + default, self._nullable) + + +class DateTimeField(object_fields.DateTimeField): + pass + + +class BooleanField(object_fields.BooleanField): + pass + + +class ListOfStringsField(object_fields.ListOfStringsField): + pass + + +class ObjectField(object_fields.ObjectField): + pass + + +class ListOfObjectsField(object_fields.ListOfObjectsField): + pass + + +class FlexibleDict(object_fields.FieldType): + @staticmethod + def coerce(obj, attr, value): + if isinstance(value, six.string_types): + value = ast.literal_eval(value) + return dict(value) + + +class FlexibleDictField(object_fields.AutoTypedField): + AUTO_TYPE = FlexibleDict() + + # TODO(lucasagomes): In our code we've always translated None to {}, + # this method makes this field to work like this. But probably won't + # be accepted as-is in the oslo_versionedobjects library + def _null(self, obj, attr): + if self.nullable: + return {} + super(FlexibleDictField, self)._null(obj, attr) + + +class EnumField(object_fields.EnumField): + pass + + +class NotificationLevel(object_fields.Enum): + DEBUG = 'debug' + INFO = 'info' + WARNING = 'warning' + ERROR = 'error' + CRITICAL = 'critical' + + ALL = (DEBUG, INFO, WARNING, ERROR, CRITICAL) + + def __init__(self): + super(NotificationLevel, self).__init__( + valid_values=NotificationLevel.ALL) + + +class NotificationLevelField(object_fields.BaseEnumField): + AUTO_TYPE = NotificationLevel() + + +class NotificationStatus(object_fields.Enum): + START = 'start' + END = 'end' + ERROR = 'error' + SUCCESS = 'success' + + ALL = (START, END, ERROR, SUCCESS) + + def __init__(self): + super(NotificationStatus, self).__init__( + valid_values=NotificationStatus.ALL) + + +class NotificationStatusField(object_fields.BaseEnumField): + AUTO_TYPE = NotificationStatus() + + +class MACAddress(object_fields.FieldType): + @staticmethod + def coerce(obj, attr, value): + return utils.validate_and_normalize_mac(value) + + +class MACAddressField(object_fields.AutoTypedField): + AUTO_TYPE = MACAddress() diff --git a/inventory/inventory/inventory/objects/host.py b/inventory/inventory/inventory/objects/host.py new file mode 100644 index 00000000..e107dff7 --- /dev/null +++ b/inventory/inventory/inventory/objects/host.py @@ -0,0 +1,118 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class Host(base.InventoryObject, object_base.VersionedObjectDictCompat): + + VERSION = '1.0' + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + + 'recordtype': object_fields.StringField(nullable=True), + 'hostname': object_fields.StringField(nullable=True), + + 'personality': object_fields.StringField(nullable=True), + 'subfunctions': object_fields.StringField(nullable=True), + 'subfunction_oper': object_fields.StringField(nullable=True), + 'subfunction_avail': object_fields.StringField(nullable=True), + 'reserved': object_fields.StringField(nullable=True), + + 'invprovision': object_fields.StringField(nullable=True), + 'mgmt_mac': object_fields.StringField(nullable=True), + 'mgmt_ip': object_fields.StringField(nullable=True), + + # Board management members + 'bm_ip': object_fields.StringField(nullable=True), + 'bm_mac': object_fields.StringField(nullable=True), + 'bm_type': object_fields.StringField(nullable=True), + 'bm_username': object_fields.StringField(nullable=True), + + 'location': object_fields.FlexibleDictField(nullable=True), + 'serialid': object_fields.StringField(nullable=True), + 'administrative': object_fields.StringField(nullable=True), + 'operational': object_fields.StringField(nullable=True), + 'availability': object_fields.StringField(nullable=True), + 'host_action': object_fields.StringField(nullable=True), + 'action_state': object_fields.StringField(nullable=True), + 'mtce_info': object_fields.StringField(nullable=True), + 'vim_progress_status': object_fields.StringField(nullable=True), + 'action': object_fields.StringField(nullable=True), + 'task': object_fields.StringField(nullable=True), + 'uptime': object_fields.IntegerField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + + 'boot_device': object_fields.StringField(nullable=True), + 'rootfs_device': object_fields.StringField(nullable=True), + 'install_output': object_fields.StringField(nullable=True), + 'console': object_fields.StringField(nullable=True), + 'tboot': object_fields.StringField(nullable=True), + 'ttys_dcd': object_fields.StringField(nullable=True), + 'install_state': object_fields.StringField(nullable=True), + 'install_state_info': object_fields.StringField(nullable=True), + 'iscsi_initiator_name': object_fields.StringField(nullable=True), + } + + @classmethod + def get_by_uuid(cls, context, uuid): + db_host = cls.dbapi.host_get(uuid) + return cls._from_db_object(context, cls(), db_host) + + @classmethod + def get_by_filters_one(cls, context, filters): + db_host = cls.dbapi.host_get_by_filters_one(filters) + return cls._from_db_object(context, cls(), db_host) + + def save_changes(self, context, updates): + self.dbapi.host_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of Host objects. + + :param cls: the :class:`Host` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`Host` object. + + """ + db_hosts = cls.dbapi.host_get_list(filters=filters, limit=limit, + marker=marker, sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_hosts) + + def create(self, context=None): + """Create a Host record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: Host(context) + :raises: InvalidParameterValue if some property values are invalid. + """ + values = self.do_version_changes_for_db() + # self._validate_property_values(values.get('properties')) + db_host = self.dbapi.host_create(values) + return self._from_db_object(self._context, self, db_host) diff --git a/inventory/inventory/inventory/objects/lldp_agent.py b/inventory/inventory/inventory/objects/lldp_agent.py new file mode 100644 index 00000000..5735da4e --- /dev/null +++ b/inventory/inventory/inventory/objects/lldp_agent.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from oslo_versionedobjects import base as object_base + +from inventory.common import k_lldp +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields + + +def get_lldp_tlvs(field, db_object): + if hasattr(db_object, field): + return db_object[field] + if hasattr(db_object, 'lldptlvs'): + tlv_object = db_object['lldptlvs'] + if tlv_object: + for tlv in tlv_object: + if tlv['type'] == field: + return tlv['value'] + return None + + +@base.InventoryObjectRegistry.register +class LLDPAgent(base.InventoryObject, object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = {'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'status': object_fields.StringField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.StringField(nullable=True), + 'port_id': object_fields.IntegerField(nullable=True), + 'port_uuid': object_fields.UUIDField(nullable=True), + 'port_name': object_fields.StringField(nullable=True), + 'port_namedisplay': object_fields.StringField(nullable=True)} + + _foreign_fields = { + 'host_uuid': 'host:uuid', + 'port_uuid': 'port:uuid', + 'port_name': 'port:name', + 'port_namedisplay': 'port:namedisplay', + } + + for tlv in k_lldp.LLDP_TLV_VALID_LIST: + fields.update({tlv: object_fields.StringField(nullable=True)}) + _foreign_fields.update({tlv: get_lldp_tlvs}) + + @classmethod + def get_by_uuid(cls, context, uuid): + db_lldp_agent = cls.dbapi.lldp_agent_get(uuid) + return cls._from_db_object(context, cls(), db_lldp_agent) + + def save_changes(self, context, updates): + self.dbapi.lldp_agent_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of LLDPAgent objects. + + :param cls: the :class:`LLDPAgent` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`LLDPAgent` object. + + """ + db_lldp_agents = cls.dbapi.lldp_agent_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_agents) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_lldp_agents = cls.dbapi.lldp_agent_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_agents) + + @classmethod + def get_by_port(cls, context, port_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_lldp_agents = cls.dbapi.lldp_agent_get_by_port( + port_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_agents) + + def create(self, context, portid, hostid, values): + """Create a LLDPAgent record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + :param portid: port id + :param hostid: host id + :param values: dictionary of values + """ + values = self.do_version_changes_for_db() + db_lldp_agent = self.dbapi.lldp_agent_create(portid, hostid, values) + return self._from_db_object(self._context, self, db_lldp_agent) diff --git a/inventory/inventory/inventory/objects/lldp_neighbour.py b/inventory/inventory/inventory/objects/lldp_neighbour.py new file mode 100644 index 00000000..5df7f0a8 --- /dev/null +++ b/inventory/inventory/inventory/objects/lldp_neighbour.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from oslo_versionedobjects import base as object_base + +from inventory.common import k_lldp +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields + + +def get_lldp_tlvs(field, db_object): + if hasattr(db_object, field): + return db_object[field] + if hasattr(db_object, 'lldptlvs'): + tlv_object = db_object['lldptlvs'] + if tlv_object: + for tlv in tlv_object: + if tlv['type'] == field: + return tlv['value'] + return None + + +@base.InventoryObjectRegistry.register +class LLDPNeighbour(base.InventoryObject, + object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = {'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'msap': object_fields.StringField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + 'port_id': object_fields.IntegerField(nullable=True), + 'port_uuid': object_fields.UUIDField(nullable=True), + 'port_name': object_fields.StringField(nullable=True), + 'port_namedisplay': object_fields.StringField(nullable=True)} + + _foreign_fields = { + 'host_uuid': 'host:uuid', + 'port_uuid': 'port:uuid', + 'port_name': 'port:name', + 'port_namedisplay': 'port:namedisplay', + } + + for tlv in k_lldp.LLDP_TLV_VALID_LIST: + fields.update({tlv: object_fields.StringField(nullable=True)}) + _foreign_fields.update({tlv: get_lldp_tlvs}) + + @classmethod + def get_by_uuid(cls, context, uuid): + db_lldp_neighbour = cls.dbapi.lldp_neighbour_get(uuid) + return cls._from_db_object(context, cls(), db_lldp_neighbour) + + def save_changes(self, context, updates): + self.dbapi.lldp_neighbour_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of LLDPNeighbour objects. + + :param cls: the :class:`LLDPNeighbour` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`LLDPNeighbour` object. + + """ + db_lldp_neighbours = cls.dbapi.lldp_neighbour_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_neighbours) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_lldp_neighbours = cls.dbapi.lldp_neighbour_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_neighbours) + + @classmethod + def get_by_port(cls, context, port_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_lldp_neighbours = cls.dbapi.lldp_neighbour_get_by_port( + port_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_neighbours) + + def create(self, context, portid, hostid, values): + """Create a LLDPAgent record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + :param portid: port id + :param hostid: host id + :param values: dictionary of values + """ + values = self.do_version_changes_for_db() + db_lldp_neighbour = self.dbapi.lldp_neighbour_create( + portid, hostid, values) + return self._from_db_object(self._context, self, db_lldp_neighbour) diff --git a/inventory/inventory/inventory/objects/lldp_tlv.py b/inventory/inventory/inventory/objects/lldp_tlv.py new file mode 100644 index 00000000..a6f1b21f --- /dev/null +++ b/inventory/inventory/inventory/objects/lldp_tlv.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class LLDPTLV(base.InventoryObject, object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = {'id': object_fields.IntegerField(nullable=True), + 'agent_id': object_fields.IntegerField(nullable=True), + 'agent_uuid': object_fields.UUIDField(nullable=True), + 'neighbour_id': object_fields.IntegerField(nullable=True), + 'neighbour_uuid': object_fields.UUIDField(nullable=True), + 'type': object_fields.StringField(nullable=True), + 'value': object_fields.StringField(nullable=True)} + + _foreign_fields = { + 'agent_uuid': 'lldp_agent:uuid', + 'neighbour_uuid': 'lldp_neighbour:uuid', + } + + @classmethod + def get_by_id(cls, context, id): + db_lldp_tlv = cls.dbapi.lldp_tlv_get_by_id(id) + return cls._from_db_object(context, cls(), db_lldp_tlv) + + def save_changes(self, context, updates): + self.dbapi.lldp_tlv_update(self.id, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of LLDPTLV objects. + + :param cls: the :class:`LLDPTLV` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`LLDPTLV` object. + + """ + db_lldp_tlvs = cls.dbapi.lldp_tlv_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_tlvs) + + @classmethod + def get_by_neighbour(cls, context, neighbour_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_lldp_tlvs = cls.dbapi.lldp_tlv_get_by_neighbour( + neighbour_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_tlvs) + + @classmethod + def get_by_agent(cls, context, agent_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_lldp_tlvs = cls.dbapi.lldp_tlv_get_by_agent( + agent_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_lldp_tlvs) + + def create(self, values, context=None, agentid=None, neighbourid=None): + """Create a LLDPTLV record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + :param agentid: agent id + :param neighbourid: neighbour id + :param values: dictionary of values + """ + values = self.do_version_changes_for_db() + db_lldp_tlv = self.dbapi.lldp_tlv_create( + values, agentid, neighbourid) + return self._from_db_object(self._context, self, db_lldp_tlv) + + def destroy(self, context=None): + """Delete the LLDPTLV from the DB. + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: Node(context) + """ + self.dbapi.lldp_tlv_destroy(self.id) + self.obj_reset_changes() diff --git a/inventory/inventory/inventory/objects/memory.py b/inventory/inventory/inventory/objects/memory.py new file mode 100644 index 00000000..f4adf86a --- /dev/null +++ b/inventory/inventory/inventory/objects/memory.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class Memory(base.InventoryObject, object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'node_id': object_fields.IntegerField(nullable=True), + 'node_uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + 'numa_node': object_fields.IntegerField(nullable=True), + + 'memtotal_mib': object_fields.IntegerField(nullable=True), + 'memavail_mib': object_fields.IntegerField(nullable=True), + 'platform_reserved_mib': object_fields.IntegerField(nullable=True), + 'node_memtotal_mib': object_fields.IntegerField(nullable=True), + + 'hugepages_configured': object_fields.StringField(nullable=True), + + 'vswitch_hugepages_size_mib': + object_fields.IntegerField(nullable=True), + 'vswitch_hugepages_reqd': object_fields.IntegerField(nullable=True), + 'vswitch_hugepages_nr': object_fields.IntegerField(nullable=True), + 'vswitch_hugepages_avail': object_fields.IntegerField(nullable=True), + + 'vm_hugepages_nr_2M_pending': + object_fields.IntegerField(nullable=True), + 'vm_hugepages_nr_1G_pending': + object_fields.IntegerField(nullable=True), + 'vm_hugepages_nr_2M': object_fields.IntegerField(nullable=True), + 'vm_hugepages_avail_2M': object_fields.IntegerField(nullable=True), + 'vm_hugepages_nr_1G': object_fields.IntegerField(nullable=True), + 'vm_hugepages_avail_1G': object_fields.IntegerField(nullable=True), + 'vm_hugepages_nr_4K': object_fields.IntegerField(nullable=True), + + + 'vm_hugepages_use_1G': object_fields.StringField(nullable=True), + 'vm_hugepages_possible_2M': object_fields.IntegerField(nullable=True), + 'vm_hugepages_possible_1G': object_fields.IntegerField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + } + + _foreign_fields = {'host_uuid': 'host:uuid', + 'node_uuid': 'node:uuid', + 'numa_node': 'node:numa_node'} + + @classmethod + def get_by_uuid(cls, context, uuid): + db_memory = cls.dbapi.memory_get(uuid) + return cls._from_db_object(context, cls(), db_memory) + + def save_changes(self, context, updates): + self.dbapi.memory_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of Memory objects. + + :param cls: the :class:`Memory` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`Memory` object. + + """ + db_memorys = cls.dbapi.memory_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_memorys) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_memorys = cls.dbapi.memory_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_memorys) + + @classmethod + def get_by_host_node(cls, context, host_uuid, node_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_memorys = cls.dbapi.memory_get_by_host_node( + host_uuid, + node_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_memorys) + + @classmethod + def get_by_node(cls, context, node_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_memorys = cls.dbapi.memory_get_by_node( + node_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_memorys) + + def create(self, context=None): + """Create a Memory record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + """ + values = self.do_version_changes_for_db() + db_memory = self.dbapi.memory_create(values) + return self._from_db_object(self._context, self, db_memory) diff --git a/inventory/inventory/inventory/objects/node.py b/inventory/inventory/inventory/objects/node.py new file mode 100644 index 00000000..aa147683 --- /dev/null +++ b/inventory/inventory/inventory/objects/node.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class Node(base.InventoryObject, + object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=False), + 'host_uuid': object_fields.StringField(nullable=True), + + 'numa_node': object_fields.IntegerField(nullable=False), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + } + + _foreign_fields = {'host_uuid': 'host:uuid'} + + @classmethod + def get_by_uuid(cls, context, uuid): + db_node = cls.dbapi.node_get(uuid) + return cls._from_db_object(context, cls(), db_node) + + def save_changes(self, context, updates): + self.dbapi.node_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of Memory objects. + + :param cls: the :class:`Memory` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`Memory` object. + + """ + db_nodes = cls.dbapi.node_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_nodes) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_nodes = cls.dbapi.node_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_nodes) diff --git a/inventory/inventory/inventory/objects/pci_device.py b/inventory/inventory/inventory/objects/pci_device.py new file mode 100644 index 00000000..ccd1226e --- /dev/null +++ b/inventory/inventory/inventory/objects/pci_device.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class PCIDevice(base.InventoryObject, + object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + 'name': object_fields.StringField(nullable=True), + 'pciaddr': object_fields.StringField(nullable=True), + 'pclass_id': object_fields.StringField(nullable=True), + 'pvendor_id': object_fields.StringField(nullable=True), + 'pdevice_id': object_fields.StringField(nullable=True), + 'pclass': object_fields.StringField(nullable=True), + 'pvendor': object_fields.StringField(nullable=True), + 'pdevice': object_fields.StringField(nullable=True), + 'psvendor': object_fields.StringField(nullable=True), + 'psdevice': object_fields.StringField(nullable=True), + 'numa_node': object_fields.IntegerField(nullable=True), + 'sriov_totalvfs': object_fields.IntegerField(nullable=True), + 'sriov_numvfs': object_fields.IntegerField(nullable=True), + 'sriov_vfs_pci_address': object_fields.StringField(nullable=True), + 'driver': object_fields.StringField(nullable=True), + 'enabled': object_fields.BooleanField(nullable=True), + 'extra_info': object_fields.StringField(nullable=True), + } + + _foreign_fields = { + 'host_uuid': 'host:uuid' + } + + @classmethod + def get_by_uuid(cls, context, uuid): + db_pci_device = cls.dbapi.pci_device_get(uuid) + return cls._from_db_object(context, cls(), db_pci_device) + + def save_changes(self, context, updates): + self.dbapi.pci_device_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None): + """Return a list of CPU objects. + + :param cls: the :class:`CPU` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :returns: a list of :class:`CPU` object. + + """ + db_pci_devices = cls.dbapi.pci_device_get_list( + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_pci_devices) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_pci_devices = cls.dbapi.pci_device_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_pci_devices) + + def create(self, context=None): + """Create a CPU record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + """ + values = self.do_version_changes_for_db() + db_pci_device = self.dbapi.pci_device_create(values) + return self._from_db_object(self._context, self, db_pci_device) diff --git a/inventory/inventory/inventory/objects/port.py b/inventory/inventory/inventory/objects/port.py new file mode 100644 index 00000000..4cd7e8fb --- /dev/null +++ b/inventory/inventory/inventory/objects/port.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class Port(base.InventoryObject, object_base.VersionedObjectDictCompat): + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + 'node_id': object_fields.IntegerField(nullable=True), + 'node_uuid': object_fields.UUIDField(nullable=True), + + 'type': object_fields.StringField(nullable=True), + 'name': object_fields.StringField(nullable=True), + 'namedisplay': object_fields.StringField(nullable=True), + 'pciaddr': object_fields.StringField(nullable=True), + 'dev_id': object_fields.IntegerField(nullable=True), + 'pclass': object_fields.StringField(nullable=True), + 'pvendor': object_fields.StringField(nullable=True), + 'pdevice': object_fields.StringField(nullable=True), + 'psvendor': object_fields.StringField(nullable=True), + 'dpdksupport': object_fields.BooleanField(nullable=True), + 'psdevice': object_fields.StringField(nullable=True), + 'numa_node': object_fields.IntegerField(nullable=True), + 'sriov_totalvfs': object_fields.IntegerField(nullable=True), + 'sriov_numvfs': object_fields.IntegerField(nullable=True), + 'sriov_vfs_pci_address': object_fields.StringField(nullable=True), + 'driver': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + } + + # interface_uuid is in systemconfig + _foreign_fields = {'host_uuid': 'host:uuid', + 'node_uuid': 'node:uuid', + } + + @classmethod + def get_by_uuid(cls, context, uuid): + db_port = cls.dbapi.port_get(uuid) + return cls._from_db_object(context, cls(), db_port) + + def save_changes(self, context, updates): + self.dbapi.port_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of Port objects. + + :param cls: the :class:`Port` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`Port` object. + + """ + db_ports = cls.dbapi.port_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ports) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_ports = cls.dbapi.port_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ports) + + @classmethod + def get_by_numa_node(cls, context, node_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_ports = cls.dbapi.port_get_by_numa_node( + node_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ports) + + def create(self, context=None): + """Create a EthernetPort record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + :raises: InvalidParameterValue if some property values are invalid. + """ + values = self.do_version_changes_for_db() + db_port = self.dbapi.port_create(values) + return self._from_db_object(self._context, self, db_port) diff --git a/inventory/inventory/inventory/objects/port_ethernet.py b/inventory/inventory/inventory/objects/port_ethernet.py new file mode 100644 index 00000000..c1284472 --- /dev/null +++ b/inventory/inventory/inventory/objects/port_ethernet.py @@ -0,0 +1,93 @@ +# +# Copyright (c) 2013-2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.objects import base +from inventory.objects import fields as object_fields +from inventory.objects import port +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class EthernetPort(port.Port, object_base.VersionedObjectDictCompat): + + fields = dict({ + 'mac': object_fields.StringField(nullable=True), + 'mtu': object_fields.IntegerField(nullable=True), + 'speed': object_fields.IntegerField(nullable=True), + 'link_mode': object_fields.StringField(nullable=True), + 'duplex': object_fields.IntegerField(nullable=True), + 'autoneg': object_fields.StringField(nullable=True), + 'bootp': object_fields.StringField(nullable=True)}, + **port.Port.fields) + + @classmethod + def get_by_uuid(cls, context, uuid): + db_ethernet_port = cls.dbapi.ethernet_port_get(uuid) + return cls._from_db_object(context, cls(), db_ethernet_port) + + def save_changes(self, context, updates): + self.dbapi.ethernet_port_update(self.uuid, updates) + + @classmethod + def list(cls, context, limit=None, marker=None, sort_key=None, + sort_dir=None, filters=None): + """Return a list of EthernetPort objects. + + :param cls: the :class:`EthernetPort` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: Filters to apply. + :returns: a list of :class:`EthernetPort` object. + + """ + db_ethernet_ports = cls.dbapi.ethernet_port_get_list( + filters=filters, + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ethernet_ports) + + @classmethod + def get_by_host(cls, context, host_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_ethernet_ports = cls.dbapi.ethernet_port_get_by_host( + host_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ethernet_ports) + + @classmethod + def get_by_numa_node(cls, context, node_uuid, + limit=None, marker=None, + sort_key=None, sort_dir=None): + db_ethernet_ports = cls.dbapi.ethernet_port_get_by_numa_node( + node_uuid, + limit=limit, + marker=marker, + sort_key=sort_key, + sort_dir=sort_dir) + return cls._from_db_object_list(context, db_ethernet_ports) + + def create(self, context=None): + """Create a EthernetPort record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. + :raises: InvalidParameterValue if some property values are invalid. + """ + values = self.do_version_changes_for_db() + db_ethernet_port = self.dbapi.ethernet_port_create(values) + return self._from_db_object(self._context, self, db_ethernet_port) diff --git a/inventory/inventory/inventory/objects/sensor.py b/inventory/inventory/inventory/objects/sensor.py new file mode 100644 index 00000000..a6928dc8 --- /dev/null +++ b/inventory/inventory/inventory/objects/sensor.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class Sensor(base.InventoryObject, object_base.VersionedObjectDictCompat): + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + 'sensorgroup_id': object_fields.IntegerField(nullable=True), + 'sensorgroup_uuid': object_fields.UUIDField(nullable=True), + + 'sensorname': object_fields.StringField(nullable=True), + 'path': object_fields.StringField(nullable=True), + 'datatype': object_fields.StringField(nullable=True), + 'sensortype': object_fields.StringField(nullable=True), + + 'status': object_fields.StringField(nullable=True), + 'state': object_fields.StringField(nullable=True), + 'state_requested': object_fields.IntegerField(nullable=True), + 'audit_interval': object_fields.IntegerField(nullable=True), + 'algorithm': object_fields.StringField(nullable=True), + 'sensor_action_requested': object_fields.StringField(nullable=True), + 'actions_minor': object_fields.StringField(nullable=True), + 'actions_major': object_fields.StringField(nullable=True), + 'actions_critical': object_fields.StringField(nullable=True), + + 'unit_base': object_fields.StringField(nullable=True), + 'unit_modifier': object_fields.StringField(nullable=True), + 'unit_rate': object_fields.StringField(nullable=True), + + 't_minor_lower': object_fields.StringField(nullable=True), + 't_minor_upper': object_fields.StringField(nullable=True), + 't_major_lower': object_fields.StringField(nullable=True), + 't_major_upper': object_fields.StringField(nullable=True), + 't_critical_lower': object_fields.StringField(nullable=True), + 't_critical_upper': object_fields.StringField(nullable=True), + + 'suppress': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True) + } + + _foreign_fields = { + 'host_uuid': 'host:uuid', + 'sensorgroup_uuid': 'sensorgroup:uuid', + } + + _optional_fields = [ + 'unit_base', + 'unit_modifier', + 'unit_rate', + + 't_minor_lower', + 't_minor_upper', + 't_major_lower', + 't_major_upper', + 't_critical_lower', + 't_critical_upper', + ] + + @object_base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.isensor_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.isensor_update(self.uuid, updates) diff --git a/inventory/inventory/inventory/objects/sensor_analog.py b/inventory/inventory/inventory/objects/sensor_analog.py new file mode 100644 index 00000000..3206593c --- /dev/null +++ b/inventory/inventory/inventory/objects/sensor_analog.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class SensorAnalog(base.InventoryObject, + object_base.VersionedObjectDictCompat): + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + + 'sensorgroup_id': object_fields.IntegerField(nullable=True), + 'sensorgroup_uuid': object_fields.UUIDField(nullable=True), + + 'sensorname': object_fields.StringField(nullable=True), + 'path': object_fields.StringField(nullable=True), + 'datatype': object_fields.StringField(nullable=True), + 'sensortype': object_fields.StringField(nullable=True), + + 'status': object_fields.StringField(nullable=True), + 'state': object_fields.StringField(nullable=True), + 'state_requested': object_fields.IntegerField(nullable=True), + 'sensor_action_requested': object_fields.StringField(nullable=True), + 'audit_interval': object_fields.IntegerField(nullable=True), + 'algorithm': object_fields.StringField(nullable=True), + 'actions_minor': object_fields.StringField(nullable=True), + 'actions_major': object_fields.StringField(nullable=True), + 'actions_critical': object_fields.StringField(nullable=True), + + 'unit_base': object_fields.StringField(nullable=True), + 'unit_modifier': object_fields.StringField(nullable=True), + 'unit_rate': object_fields.StringField(nullable=True), + + 't_minor_lower': object_fields.StringField(nullable=True), + 't_minor_upper': object_fields.StringField(nullable=True), + 't_major_lower': object_fields.StringField(nullable=True), + 't_major_upper': object_fields.StringField(nullable=True), + 't_critical_lower': object_fields.StringField(nullable=True), + 't_critical_upper': object_fields.StringField(nullable=True), + + 'suppress': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + } + + _foreign_fields = { + 'host_uuid': 'host:uuid', + 'sensorgroup_uuid': 'sensorgroup:uuid', + } + + @object_base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.isensor_analog_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.isensor_analog_update(self.uuid, updates) diff --git a/inventory/inventory/inventory/objects/sensor_discrete.py b/inventory/inventory/inventory/objects/sensor_discrete.py new file mode 100644 index 00000000..42e3dfef --- /dev/null +++ b/inventory/inventory/inventory/objects/sensor_discrete.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class SensorDiscrete(base.InventoryObject, + object_base.VersionedObjectDictCompat): + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + + 'sensorgroup_id': object_fields.IntegerField(nullable=True), + 'sensorgroup_uuid': object_fields.UUIDField(nullable=True), + + 'sensorname': object_fields.StringField(nullable=True), + 'path': object_fields.StringField(nullable=True), + 'datatype': object_fields.StringField(nullable=True), + 'sensortype': object_fields.StringField(nullable=True), + + 'status': object_fields.StringField(nullable=True), + 'state': object_fields.StringField(nullable=True), + 'state_requested': object_fields.IntegerField(nullable=True), + 'audit_interval': object_fields.IntegerField(nullable=True), + 'algorithm': object_fields.StringField(nullable=True), + 'sensor_action_requested': object_fields.StringField(nullable=True), + 'actions_minor': object_fields.StringField(nullable=True), + 'actions_major': object_fields.StringField(nullable=True), + 'actions_critical': object_fields.StringField(nullable=True), + + 'suppress': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True) + } + + _foreign_fields = { + 'host_uuid': 'host:uuid', + 'sensorgroup_uuid': 'sensorgroup:uuid', + } + + @object_base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.isensor_discrete_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.isensor_discrete_update(self.uuid, updates) diff --git a/inventory/inventory/inventory/objects/sensorgroup.py b/inventory/inventory/inventory/objects/sensorgroup.py new file mode 100644 index 00000000..5955c44a --- /dev/null +++ b/inventory/inventory/inventory/objects/sensorgroup.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class SensorGroup(base.InventoryObject, + object_base.VersionedObjectDictCompat): + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + 'host_uuid': object_fields.UUIDField(nullable=True), + + 'sensorgroupname': object_fields.StringField(nullable=True), + 'path': object_fields.StringField(nullable=True), + + 'datatype': object_fields.StringField(nullable=True), + 'sensortype': object_fields.StringField(nullable=True), + 'description': object_fields.StringField(nullable=True), + + 'state': object_fields.StringField(nullable=True), + 'possible_states': object_fields.StringField(nullable=True), + 'audit_interval_group': object_fields.IntegerField(nullable=True), + 'record_ttl': object_fields.StringField(nullable=True), + + 'algorithm': object_fields.StringField(nullable=True), + 'actions_minor_group': object_fields.StringField(nullable=True), + 'actions_major_group': object_fields.StringField(nullable=True), + 'actions_critical_group': object_fields.StringField(nullable=True), + + 'unit_base_group': object_fields.StringField(nullable=True), + 'unit_modifier_group': object_fields.StringField(nullable=True), + 'unit_rate_group': object_fields.StringField(nullable=True), + + 't_minor_lower_group': object_fields.StringField(nullable=True), + 't_minor_upper_group': object_fields.StringField(nullable=True), + 't_major_lower_group': object_fields.StringField(nullable=True), + 't_major_upper_group': object_fields.StringField(nullable=True), + 't_critical_lower_group': object_fields.StringField(nullable=True), + 't_critical_upper_group': object_fields.StringField(nullable=True), + + 'suppress': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + + 'actions_critical_choices': object_fields.StringField(nullable=True), + 'actions_major_choices': object_fields.StringField(nullable=True), + 'actions_minor_choices': object_fields.StringField(nullable=True) + } + + _foreign_fields = { + 'host_uuid': 'host:uuid' + } + + _optional_fields = [ + 'unit_base_group', + 'unit_modifier_group', + 'unit_rate_group', + + 't_minor_lower_group', + 't_minor_upper_group', + 't_major_lower_group', + 't_major_upper_group', + 't_critical_lower_group', + 't_critical_upper_group', + ] + + @object_base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.isensorgroup_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.isensorgroup_update(self.uuid, updates) diff --git a/inventory/inventory/inventory/objects/sensorgroup_analog.py b/inventory/inventory/inventory/objects/sensorgroup_analog.py new file mode 100644 index 00000000..1de49aaf --- /dev/null +++ b/inventory/inventory/inventory/objects/sensorgroup_analog.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from oslo_versionedobjects import base as object_base + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields + + +@base.InventoryObjectRegistry.register +class SensorGroupAnalog(base.InventoryObject, + object_base.VersionedObjectDictCompat): + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + + 'sensorgroupname': object_fields.StringField(nullable=True), + 'path': object_fields.StringField(nullable=True), + + 'sensortype': object_fields.StringField(nullable=True), + 'datatype': object_fields.StringField(nullable=True), + 'description': object_fields.StringField(nullable=True), + + 'state': object_fields.StringField(nullable=True), + 'possible_states': object_fields.StringField(nullable=True), + 'audit_interval_group': object_fields.IntegerField(nullable=True), + 'record_ttl': object_fields.StringField(nullable=True), + + 'algorithm': object_fields.StringField(nullable=True), + 'actions_critical_choices': object_fields.StringField(nullable=True), + 'actions_major_choices': object_fields.StringField(nullable=True), + 'actions_minor_choices': object_fields.StringField(nullable=True), + 'actions_minor_group': object_fields.StringField(nullable=True), + 'actions_major_group': object_fields.StringField(nullable=True), + 'actions_critical_group': object_fields.StringField(nullable=True), + + 'unit_base_group': object_fields.StringField(nullable=True), + 'unit_modifier_group': object_fields.StringField(nullable=True), + 'unit_rate_group': object_fields.StringField(nullable=True), + + 't_minor_lower_group': object_fields.StringField(nullable=True), + 't_minor_upper_group': object_fields.StringField(nullable=True), + 't_major_lower_group': object_fields.StringField(nullable=True), + 't_major_upper_group': object_fields.StringField(nullable=True), + 't_critical_lower_group': object_fields.StringField(nullable=True), + 't_critical_upper_group': object_fields.StringField(nullable=True), + + 'suppress': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True) + } + + @object_base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.isensorgroup_analog_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.isensorgroup_analog_update(self.uuid, updates) diff --git a/inventory/inventory/inventory/objects/sensorgroup_discrete.py b/inventory/inventory/inventory/objects/sensorgroup_discrete.py new file mode 100644 index 00000000..53df768e --- /dev/null +++ b/inventory/inventory/inventory/objects/sensorgroup_discrete.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 +# + +from oslo_versionedobjects import base as object_base + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields + + +@base.InventoryObjectRegistry.register +class SensorGroupDiscrete(base.InventoryObject, + object_base.VersionedObjectDictCompat): + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'host_id': object_fields.IntegerField(nullable=True), + + 'sensorgroupname': object_fields.StringField(nullable=True), + 'path': object_fields.StringField(nullable=True), + + 'datatype': object_fields.StringField(nullable=True), + 'sensortype': object_fields.StringField(nullable=True), + 'description': object_fields.StringField(nullable=True), + + 'state': object_fields.StringField(nullable=True), + 'possible_states': object_fields.StringField(nullable=True), + 'audit_interval_group': object_fields.IntegerField(nullable=True), + 'record_ttl': object_fields.StringField(nullable=True), + + 'algorithm': object_fields.StringField(nullable=True), + 'actions_critical_choices': object_fields.StringField(nullable=True), + 'actions_major_choices': object_fields.StringField(nullable=True), + 'actions_minor_choices': object_fields.StringField(nullable=True), + 'actions_minor_group': object_fields.StringField(nullable=True), + 'actions_major_group': object_fields.StringField(nullable=True), + 'actions_critical_group': object_fields.StringField(nullable=True), + + 'suppress': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True) + + } + + @object_base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + return cls.dbapi.isensorgroup_discrete_get(uuid) + + def save_changes(self, context, updates): + self.dbapi.isensorgroup_discrete_update(self.uuid, updates) diff --git a/inventory/inventory/inventory/objects/system.py b/inventory/inventory/inventory/objects/system.py new file mode 100644 index 00000000..dea0e8d6 --- /dev/null +++ b/inventory/inventory/inventory/objects/system.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventory.db import api as db_api +from inventory.objects import base +from inventory.objects import fields as object_fields +from oslo_versionedobjects import base as object_base + + +@base.InventoryObjectRegistry.register +class System(base.InventoryObject, object_base.VersionedObjectDictCompat): + + VERSION = '1.0' + + dbapi = db_api.get_instance() + + fields = { + 'id': object_fields.IntegerField(nullable=True), + 'uuid': object_fields.UUIDField(nullable=True), + 'name': object_fields.StringField(nullable=True), + 'system_type': object_fields.StringField(nullable=True), + 'system_mode': object_fields.StringField(nullable=True), + 'description': object_fields.StringField(nullable=True), + 'capabilities': object_fields.FlexibleDictField(nullable=True), + 'contact': object_fields.StringField(nullable=True), + 'location': object_fields.StringField(nullable=True), + 'services': object_fields.IntegerField(nullable=True), + 'software_version': object_fields.StringField(nullable=True), + 'timezone': object_fields.StringField(nullable=True), + 'security_profile': object_fields.StringField(nullable=True), + 'region_name': object_fields.StringField(nullable=True), + 'service_project_name': object_fields.StringField(nullable=True), + 'distributed_cloud_role': object_fields.StringField(nullable=True), + 'security_feature': object_fields.StringField(nullable=True), + } + + @classmethod + def get_by_uuid(cls, context, uuid): + db_system = cls.dbapi.system_get(uuid) + return cls._from_db_object(context, cls(), db_system) + + @classmethod + def get_one(cls, context): + db_system = cls.dbapi.system_get_one() + system = cls._from_db_object(context, cls(), db_system) + return system + + @classmethod + def list(cls, context, + limit=None, marker=None, sort_key=None, sort_dir=None): + """Return a list of System objects. + + :param cls: the :class:`System` + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :returns: a list of :class:`System` object. + + """ + db_systems = cls.dbapi.system_get_list( + limit=limit, marker=marker, sort_key=sort_key, sort_dir=sort_dir) + return cls._from_db_object_list(context, db_systems) + + def save_changes(self, context, updates): + self.dbapi.system_update(self.uuid, updates) + + def create(self, context=None): + """Create a System record in the DB. + + Column-wise updates will be made based on the result of + self.what_changed(). + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: System(context) + """ + values = self.do_version_changes_for_db() + db_system = self.dbapi.system_create(values) + return self._from_db_object(self._context, self, db_system) diff --git a/inventory/inventory/inventory/systemconfig/__init__.py b/inventory/inventory/inventory/systemconfig/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/systemconfig/config.py b/inventory/inventory/inventory/systemconfig/config.py new file mode 100644 index 00000000..a3108c1a --- /dev/null +++ b/inventory/inventory/inventory/systemconfig/config.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from oslo_config import cfg +from oslo_utils._i18n import _ + +INVENTORY_CONFIG_OPTS = [ + cfg.ListOpt('drivers', + default=['systemconfig'], + help=_("SystemConfig driver " + "entrypoints to be loaded from the " + "inventory.systemconfig namespace.")), +] + +cfg.CONF.register_opts(INVENTORY_CONFIG_OPTS, group="configuration") diff --git a/inventory/inventory/inventory/systemconfig/drivers/__init__.py b/inventory/inventory/inventory/systemconfig/drivers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/systemconfig/drivers/base.py b/inventory/inventory/inventory/systemconfig/drivers/base.py new file mode 100644 index 00000000..5d1dabef --- /dev/null +++ b/inventory/inventory/inventory/systemconfig/drivers/base.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class SystemConfigDriverBase(object): + """SystemConfig Driver Base Class.""" + + @abc.abstractmethod + def system_get_one(self): + pass + + @abc.abstractmethod + def network_get_by_type(self, network_type): + pass + + @abc.abstractmethod + def address_get_by_name(self, name): + pass + + @abc.abstractmethod + def host_configure_check(self, host_uuid): + pass + + @abc.abstractmethod + def host_configure(self, host_uuid, do_compute_apply=False): + pass + + @abc.abstractmethod + def host_unconfigure(self, host_uuid): + pass diff --git a/inventory/inventory/inventory/systemconfig/drivers/sysinv/__init__.py b/inventory/inventory/inventory/systemconfig/drivers/sysinv/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/systemconfig/drivers/sysinv/driver.py b/inventory/inventory/inventory/systemconfig/drivers/sysinv/driver.py new file mode 100644 index 00000000..47a9f0f0 --- /dev/null +++ b/inventory/inventory/inventory/systemconfig/drivers/sysinv/driver.py @@ -0,0 +1,235 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventory.systemconfig.drivers import base +from inventory.systemconfig import plugin + +import cgtsclient as sysinv_client +from keystoneauth1.access import service_catalog as k_service_catalog +from keystoneclient.auth.identity import v3 +from keystoneclient import session +from oslo_config import cfg +from oslo_log import log + +CONF = cfg.CONF + +LOG = log.getLogger(__name__) + +sysinv_group = cfg.OptGroup( + 'sysinv', + title='SysInv Options', + help="Configuration options for the sysinv service") + +sysinv_opts = [ + cfg.StrOpt('catalog_info', + default='platform:sysinv:internalURL', + help="Service catalog Look up info."), + cfg.StrOpt('os_region_name', + default='RegionOne', + help="Region name of this node. It is used for catalog lookup") +] + +CONF.register_group(sysinv_group) +CONF.register_opts(sysinv_opts, group=sysinv_group) + + +def _get_keystone_session(auth_url): + auth = v3.Password(auth_url=auth_url, + username=cfg.CONF.KEYSTONE_AUTHTOKEN.username, + password=cfg.CONF.KEYSTONE_AUTHTOKEN.password, + user_domain_name=cfg.CONF.KEYSTONE_AUTHTOKEN. + user_domain_name, + project_name=cfg.CONF.KEYSTONE_AUTHTOKEN. + project_name, + project_domain_name=cfg.CONF.KEYSTONE_AUTHTOKEN. + project_domain_name) + keystone_session = session.Session(auth=auth) + return keystone_session + + +def sysinvclient(context, version=1, endpoint=None): + """Constructs a sysinv client object for making API requests. + + :param context: The request context for auth. + :param version: API endpoint version. + :param endpoint: Optional If the endpoint is not available, + it will be retrieved from context + """ + + region_name = CONF.sysinv.os_region_name + if not context.service_catalog: + # Obtain client via keystone session + auth_url = CONF.KEYSTONE_AUTHTOKEN.auth_url + "/v3" + session = _get_keystone_session(auth_url) + LOG.debug("sysinvclient auth_url=%s region_name=%s session=%s" % + (auth_url, region_name, session)) + + return sysinv_client.Client( + session=session, + version=version, + auth_url=auth_url, + endpoint_type='internalURL', + region_name=region_name) + + auth_token = context.auth_token + if endpoint is None: + sc = k_service_catalog.ServiceCatalogV2(context.service_catalog) + service_type, service_name, interface = \ + CONF.sysinv.catalog_info.split(':') + + service_parameters = {'service_type': service_type, + 'service_name': service_name, + 'interface': interface, + 'region_name': region_name} + endpoint = sc.url_for(**service_parameters) + + return sysinv_client.Client(version=version, + endpoint=endpoint, + auth_token=auth_token) + + +class SysinvSystemConfigDriver(base.SystemConfigDriverBase): + """Class to encapsulate SystemConfig driver operations""" + def __init__(self, **kwargs): + self.context = kwargs.get('context') + self.neighbours = [] + self.neighbour_audit_count = 0 + self._client = sysinvclient(self.context) + LOG.info("SysinvSystemConfigDriver kwargs=%s self.context=%s" % + (kwargs, self.context)) + + def initialize(self): + self.__init__() + + def system_get(self): + systems = self._client.isystem.list() + if not systems: + return None + return [plugin.System(n) for n in systems] + + def system_get_one(self): + systems = self._client.isystem.list() + if not systems: + return None + return [plugin.System(n) for n in systems][0] + + def host_interface_list(self, host_id): + interfaces = self._client.iinterface.list(host_id) + return [plugin.Interface(n) for n in interfaces] + + def host_interface_get(self, interface_id): + interface = self._client.iinterface.get(interface_id) + if not interface: + raise ValueError( + 'No match found for interface_id "%s".' % interface_id) + return plugin.Interface(interface) + + def host_configure_check(self, host_uuid): + LOG.info("host_configure_check %s" % host_uuid) + # host = self._client.ihost.get(host_uuid) + capabilities = [] + host = self._client.ihost.configure_check(host_uuid, capabilities) + LOG.info("host_configure_check host=%s" % host) + if host: + return True + else: + return False + + def host_configure(self, host_uuid, do_compute_apply=False): + LOG.info("simulate host_configure") + # host = self._client.ihost.get(host_uuid) + # TODO(sc) for host configuration + host = self._client.ihost.configure(host_uuid, do_compute_apply) + if host: + return plugin.Host(host) + else: + return None + + def host_unconfigure(self, host_uuid): + LOG.info("simulate host_unconfigure") + host = self._client.ihost.get(host_uuid) + if host: + return plugin.Host(host) + else: + return None + + host = self._client.ihost.unconfigure(host_uuid) + + return host + + def network_list(self): + networks = self._client.network.list() + return [plugin.Network(n) for n in networks] + + def network_get_by_type(self, network_type): + networks = self._client.network.list() + if networks: + return [plugin.Network(n) for n in networks + if n.type == network_type][0] + return [] + + def network_get(self, network_uuid): + network = self._client.network.get(network_uuid) + if not network: + raise ValueError( + 'No match found for network_uuid "%s".' % network_uuid) + return plugin.Network(network) + + def address_list_by_interface(self, interface_id): + addresses = self._client.address.list_by_interface(interface_id) + return [plugin.Address(n) for n in addresses] + + def address_list_by_field_value(self, field, value): + q = [{'field': field, + 'type': '', + 'value': value, + 'op': 'eq'}] + addresses = self._client.address.list(q) + return [plugin.Address(n) for n in addresses] + + def address_get(self, address_uuid): + address = self._client.address.get(address_uuid) + if not address: + raise ValueError( + 'No match found for address uuid "%s".' % address_uuid) + return plugin.Address(address) + + def address_pool_list(self): + pools = self._client.address_pool.list() + return [plugin.AddressPool(p) for p in pools] + + def address_pool_get(self, address_pool_uuid): + pool = self._client.address_pool.get(address_pool_uuid) + if not pool: + raise ValueError( + 'No match found for address pool uuid "%s".' % + address_pool_uuid) + return plugin.AddressPool(pool) + + def route_list_by_interface(self, interface_id): + routees = self._client.route.list_by_interface(interface_id) + return [plugin.Route(n) for n in routees] + + def route_get(self, route_uuid): + route = self._client.route.get(route_uuid) + if not route: + raise ValueError( + 'No match found for route uuid "%s".' % route_uuid) + return plugin.Route(route) + + def address_get_by_name(self, name): + field = 'name' + value = name + addresses = self.address_list_by_field_value(field, value) + if len(addresses) == 1: + address = addresses[0] + LOG.info("address_get_by_name via systemconfig " + "name=%s address=%s" % + (address.name, address.address)) + else: + LOG.error("Unexpected address_get_by_name %s %s" % + (name, addresses)) + return None diff --git a/inventory/inventory/inventory/systemconfig/manager.py b/inventory/inventory/inventory/systemconfig/manager.py new file mode 100644 index 00000000..6e7b3a3e --- /dev/null +++ b/inventory/inventory/inventory/systemconfig/manager.py @@ -0,0 +1,179 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventory.common import exception +from oslo_config import cfg +from oslo_log import log +from stevedore.named import NamedExtensionManager + +LOG = log.getLogger(__name__) +cfg.CONF.import_opt('drivers', + 'inventory.systemconfig.config', + group='configuration') + + +class SystemConfigDriverManager(NamedExtensionManager): + """Implementation of Sysinv SystemConfig drivers.""" + + def __init__(self, invoke_kwds={}, + namespace='inventory.systemconfig.drivers'): + + # Registered configuration drivers, keyed by name. + self.drivers = {} + + # Ordered list of inventory configuration drivers, defining + # the order in which the drivers are called. + self.ordered_drivers = [] + + names = cfg.CONF.configuration.drivers + LOG.info("Configured inventory configuration drivers: args %s " + "names=%s" % + (invoke_kwds, names)) + + super(SystemConfigDriverManager, self).__init__( + namespace, + names, + invoke_kwds=invoke_kwds, + invoke_on_load=True, + name_order=True) + + LOG.info("Loaded systemconfig drivers: %s" % self.names()) + self._register_drivers() + + def _register_drivers(self): + """Register all configuration drivers. + + This method should only be called once in the + SystemConfigDriverManager constructor. + """ + for ext in self: + self.drivers[ext.name] = ext + self.ordered_drivers.append(ext) + LOG.info("Registered systemconfig drivers: %s", + [driver.name for driver in self.ordered_drivers]) + + def _call_drivers_and_return_array(self, method_name, attr=None, + raise_orig_exc=False): + """Helper method for calling a method across all drivers. + + :param method_name: name of the method to call + :param attr: an optional attribute to provide to the drivers + :param raise_orig_exc: whether or not to raise the original + driver exception, or use a general one + """ + ret = [] + for driver in self.ordered_drivers: + try: + method = getattr(driver.obj, method_name) + if attr: + ret = ret + method(attr) + else: + ret = ret + method() + except Exception as e: + LOG.exception(e) + LOG.error( + "Inventory SystemConfig driver '%(name)s' " + "failed in %(method)s", + {'name': driver.name, 'method': method_name} + ) + if raise_orig_exc: + raise + else: + raise exception.SystemConfigDriverError( + method=method_name + ) + return list(set(ret)) + + def _call_drivers(self, method_name, + raise_orig_exc=False, + return_first=True, + **kwargs): + """Helper method for calling a method across all drivers. + + :param method_name: name of the method to call + :param attr: an optional attribute to provide to the drivers + :param raise_orig_exc: whether or not to raise the original + driver exception, or use a general one + """ + for driver in self.ordered_drivers: + try: + method = getattr(driver.obj, method_name) + LOG.info("_call_drivers_kwargs method_name=%s kwargs=%s" + % (method_name, kwargs)) + + ret = method(**kwargs) + if return_first: + return ret + + except Exception as e: + LOG.exception(e) + LOG.error( + "Inventory SystemConfig driver '%(name)s' " + "failed in %(method)s", + {'name': driver.name, 'method': method_name} + ) + if raise_orig_exc: + raise + else: + raise exception.SystemConfigDriverError( + method=method_name + ) + + def system_get_one(self): + try: + return self._call_drivers( + "system_get_one", + raise_orig_exc=True) + except Exception as e: + LOG.exception(e) + + def network_get_by_type(self, network_type): + try: + return self._call_drivers( + "network_get_by_type", + raise_orig_exc=True, + network_type=network_type) + except Exception as e: + LOG.exception(e) + + def address_get_by_name(self, name): + try: + return self._call_drivers( + "address_get_by_name", + raise_orig_exc=True, + name=name) + except Exception as e: + LOG.exception(e) + + def host_configure_check(self, host_uuid): + try: + return self._call_drivers("host_configure_check", + raise_orig_exc=True, + host_uuid=host_uuid) + except Exception as e: + LOG.exception(e) + + def host_configure(self, host_uuid, do_compute_apply=False): + try: + return self._call_drivers("host_configure", + raise_orig_exc=True, + host_uuid=host_uuid, + do_compute_apply=do_compute_apply) + except Exception as e: + LOG.exception(e) + + def host_unconfigure(self, host_uuid): + try: + return self._call_drivers("host_unconfigure", + raise_orig_exc=True, + host_uuid=host_uuid) + except Exception as e: + LOG.exception(e) diff --git a/inventory/inventory/inventory/systemconfig/plugin.py b/inventory/inventory/inventory/systemconfig/plugin.py new file mode 100644 index 00000000..f9ed912c --- /dev/null +++ b/inventory/inventory/inventory/systemconfig/plugin.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +from inventory.common import base +from inventory.common import exception +from inventory.systemconfig import manager +from oslo_log import log +from oslo_utils import excutils + +LOG = log.getLogger(__name__) + + +class SystemConfigPlugin(object): + """Implementation of the Plugin.""" + + def __init__(self, invoke_kwds): + self.manager = manager.SystemConfigDriverManager( + invoke_kwds=invoke_kwds) + + def system_get_one(self): + try: + system = self.manager.system_get_one() + except exception.SystemConfigDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("system_get failed") + + return system + + def network_get_by_type(self, network_type): + try: + network = self.manager.network_get_by_type( + network_type=network_type) + except exception.SystemConfigDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("network_get_by_type failed") + + return network + + def address_get_by_name(self, name): + try: + address = self.manager.address_get_by_name( + name=name) + except exception.SystemConfigDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("address_get_by_name failed") + + return address + + def host_configure_check(self, host_uuid): + try: + return self.manager.host_configure_check( + host_uuid=host_uuid) + except exception.SystemConfigDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("host_configure_check failed") + + def host_configure(self, host_uuid, do_compute_apply=False): + try: + host = self.manager.host_configure( + host_uuid=host_uuid, + do_compute_apply=do_compute_apply) + except exception.SystemConfigDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("host_configure failed") + + return host + + def host_unconfigure(self, host_uuid): + try: + host = self.manager.host_unconfigure( + host_uuid=host_uuid) + except exception.SystemConfigDriverError as e: + LOG.exception(e) + with excutils.save_and_reraise_exception(): + LOG.error("host_unconfigure failed") + + return host + + +class System(base.APIResourceWrapper): + """Wrapper for SystemConfig System""" + + _attrs = ['uuid', 'name', 'system_type', 'system_mode', 'description', + 'software_version', 'capabilities', 'region_name', + 'updated_at', 'created_at'] + + def __init__(self, apiresource): + super(System, self).__init__(apiresource) + + def get_short_software_version(self): + if self.software_version: + return self.software_version.split(" ")[0] + return None + + +class Host(base.APIResourceWrapper): + """Wrapper for Inventory Hosts""" + + _attrs = ['uuid', 'hostname', 'personality', + 'mgmt_mac', 'mgmt_ip', 'bm_ip', + 'subfunctions', + 'capabilities', + 'created_at', 'updated_at', + ] + + # Removed 'id', 'requires_reboot' + # Add this back to models, migrate_repo: peers + + def __init__(self, apiresource): + super(Host, self).__init__(apiresource) + self._personality = self.personality + self._capabilities = self.capabilities + + +class Interface(base.APIResourceWrapper): + """Wrapper for SystemConfig Interfaces""" + + _attrs = ['id', 'uuid', 'ifname', 'ifclass', 'iftype', + 'networktype', 'networks', 'vlan_id', + 'uses', 'used_by', 'ihost_uuid', + 'ipv4_mode', 'ipv6_mode', 'ipv4_pool', 'ipv6_pool', + 'sriov_numvfs', + # VLAN and virtual interfaces + 'imac', 'imtu', 'providernetworks', 'providernetworksdict', + # AE-only + 'aemode', 'txhashpolicy', 'schedpolicy', + ] + + def __init__(self, apiresource): + super(Interface, self).__init__(apiresource) + if not self.ifname: + self.ifname = '(' + str(self.uuid)[-8:] + ')' + + +class Network(base.APIResourceWrapper): + """Wrapper for SystemConfig Networks""" + _attrs = ['id', 'uuid', 'type', 'name', 'dynamic', 'pool_uuid'] + + def __init__(self, apiresource): + super(Network, self).__init__(apiresource) + + +class Address(base.APIResourceWrapper): + """Wrapper for SystemConfig Addresses""" + + _attrs = ['uuid', 'name', 'interface_uuid', + 'address', 'prefix', 'enable_dad'] + + def __init__(self, apiresource): + super(Address, self).__init__(apiresource) + + +class AddressPool(base.APIResourceWrapper): + """Wrapper for SystemConfig Address Pools""" + + _attrs = ['uuid', 'name', 'network', 'family', 'prefix', 'order', 'ranges'] + + def __init__(self, apiresource): + super(AddressPool, self).__init__(apiresource) + + +class Route(base.APIResourceWrapper): + """Wrapper for SystemConfig Routers""" + + _attrs = ['uuid', 'interface_uuid', 'network', + 'prefix', 'gateway', 'metric'] + + def __init__(self, apiresource): + super(Route, self).__init__(apiresource) diff --git a/inventory/inventory/inventory/tests/__init__.py b/inventory/inventory/inventory/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/inventory/tests/base.py b/inventory/inventory/inventory/tests/base.py new file mode 100644 index 00000000..1c30cdb5 --- /dev/null +++ b/inventory/inventory/inventory/tests/base.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2010-2011 OpenStack Foundation +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslotest import base + + +class TestCase(base.BaseTestCase): + + """Test case base class for all unit tests.""" diff --git a/inventory/inventory/inventory/tests/test_inventory.py b/inventory/inventory/inventory/tests/test_inventory.py new file mode 100644 index 00000000..8e77fbf1 --- /dev/null +++ b/inventory/inventory/inventory/tests/test_inventory.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_inventory +-------------- + +Tests for `inventory` module. +""" + +from inventory.tests import base + + +class TestInventory(base.TestCase): + + def test_something(self): + pass diff --git a/inventory/inventory/inventory/version.py b/inventory/inventory/inventory/version.py new file mode 100644 index 00000000..cade1b2b --- /dev/null +++ b/inventory/inventory/inventory/version.py @@ -0,0 +1,18 @@ +# Copyright 2011 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pbr.version + +version_info = pbr.version.VersionInfo('inventory') diff --git a/inventory/inventory/releasenotes/notes/.placeholder b/inventory/inventory/releasenotes/notes/.placeholder new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/releasenotes/source/_static/.placeholder b/inventory/inventory/releasenotes/source/_static/.placeholder new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/releasenotes/source/_templates/.placeholder b/inventory/inventory/releasenotes/source/_templates/.placeholder new file mode 100644 index 00000000..e69de29b diff --git a/inventory/inventory/releasenotes/source/conf.py b/inventory/inventory/releasenotes/source/conf.py new file mode 100644 index 00000000..7fe27dad --- /dev/null +++ b/inventory/inventory/releasenotes/source/conf.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'openstackdocstheme', + 'reno.sphinxext', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'inventory Release Notes' +copyright = u'2018, StarlingX' + +# openstackdocstheme options +repository_name = 'openstack/inventory' +bug_project = '22952' +bug_tag = '' +html_last_updated_fmt = '%Y-%m-%d %H:%M' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +# The full version, including alpha/beta/rc tags. +release = '' +# The short X.Y version. +version = '' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'openstackdocs' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'inventoryReleaseNotesdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'inventoryReleaseNotes.tex', + u'inventory Release Notes Documentation', + u'OpenStack Foundation', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'inventoryrereleasenotes', + u'inventory Release Notes Documentation', + [u'OpenStack Foundation'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'inventory ReleaseNotes', + u'inventory Release Notes Documentation', + u'OpenStack Foundation', 'inventoryReleaseNotes', + 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/inventory/inventory/releasenotes/source/index.rst b/inventory/inventory/releasenotes/source/index.rst new file mode 100644 index 00000000..5f58e0f6 --- /dev/null +++ b/inventory/inventory/releasenotes/source/index.rst @@ -0,0 +1,8 @@ +======================= +inventory Release Notes +======================= + +.. toctree:: + :maxdepth: 1 + + unreleased diff --git a/inventory/inventory/releasenotes/source/unreleased.rst b/inventory/inventory/releasenotes/source/unreleased.rst new file mode 100644 index 00000000..875030f9 --- /dev/null +++ b/inventory/inventory/releasenotes/source/unreleased.rst @@ -0,0 +1,5 @@ +============================ +Current Series Release Notes +============================ + +.. release-notes:: diff --git a/inventory/inventory/requirements.txt b/inventory/inventory/requirements.txt new file mode 100644 index 00000000..562e2a97 --- /dev/null +++ b/inventory/inventory/requirements.txt @@ -0,0 +1,47 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +pbr>=2.0 # Apache-2.0 +SQLAlchemy +amqplib>=0.6.1 +anyjson>=0.3.3 +argparse +eventlet==0.20.0 +futurist>=1.2.0 # Apache-2.0 +greenlet>=0.3.2 # MIT +kombu>=2.4.8 +lxml>=2.3 +WebOb>=1.7.1 # MIT +sqlalchemy-migrate>=0.7 +netaddr +iso8601>=0.1.4 +oslo.concurrency>=3.7.1 # Apache-2.0 +oslo.config!=4.3.0,!=4.4.0,>=4.0.0 # Apache-2.0 +oslo.context>=2.14.0 # Apache-2.0 +oslo.rootwrap>=5.0.0 # Apache-2.0 +oslo.i18n!=3.15.2,>=2.1.0 # Apache-2.0 +oslo.log>=3.22.0 # Apache-2.0 +oslo.middleware>=3.27.0 # Apache-2.0 +oslo.policy>=1.23.0 # Apache-2.0 +oslo.db>=4.1.0 # Apache-2.0 +oslo.serialization>=1.10.0,!=2.19.1 # Apache-2.0 +oslo.service>=1.10.0 # Apache-2.0 +oslo.utils>=3.5.0 # Apache-2.0 +osprofiler>=1.4.0 # Apache-2.0 +python-cinderclient>=3.1.0 # Apache-2.0 +python-keystoneclient>=3.8.0 # Apache-2.0 +keyring +keystonemiddleware>=4.12.0 # Apache-2.0 +oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0 +retrying!=1.3.0,>=1.2.3 # Apache-2.0 +oslo.versionedobjects>=1.17.0 # Apache-2.0 +stevedore>=0.10 +pecan>=1.0.0 +six>=1.9.0 # MIT +jsonpatch>=1.1 # BSD +WSME>=0.8 # MIT +PyYAML>=3.10 +python-magnumclient>=2.0.0 # Apache-2.0 +psutil +simplejson>=2.2.0 # MIT diff --git a/inventory/inventory/scripts/inventory-agent-initd b/inventory/inventory/scripts/inventory-agent-initd new file mode 100755 index 00000000..6d7fe73a --- /dev/null +++ b/inventory/inventory/scripts/inventory-agent-initd @@ -0,0 +1,204 @@ +#! /bin/sh +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# +# chkconfig: 2345 75 25 +# +### BEGIN INIT INFO +# Provides: inventory-agent +# Default-Start: 3 5 +# Default-Stop: 0 1 2 6 +# Short-Description: inventory-agent daemon +### END INIT INFO + +. /etc/init.d/functions +. /etc/build.info + + +PLATFORM_CONF="/etc/platform/platform.conf" +NODETYPE="" +DAEMON_NAME="inventory-agent" +INVENTORYAGENT="/usr/bin/${DAEMON_NAME}" +INVENTORY_CONF_DIR="/etc/inventory" +INVENTORY_CONF_FILE="${INVENTORY_CONF_DIR}/inventory.conf" +INVENTORY_CONF_DEFAULT_FILE="/opt/platform/inventory/${SW_VERSION}/inventory.conf.default" +INVENTORY_READY_FLAG=/var/run/.inventory_ready + +DELAY_SEC=20 + +daemon_pidfile="/var/run/${DAEMON_NAME}.pid" + +if [ -f ${PLATFORM_CONF} ] ; then + NODETYPE=`cat ${PLATFORM_CONF} | grep nodetype | cut -f2 -d'='` +else + logger "$0: ${PLATFORM_CONF} is missing" + exit 1 +fi + +if [ ! -e "${INVENTORYAGENT}" ] ; then + logger "$0: ${INVENTORYAGENT} is missing" + exit 1 +fi + +RETVAL=0 + +PATH=/sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin +export PATH + +mount_and_copy_config_file() +{ + echo "Mount /opt/platform" + logger "$0: Info: nfs-mount controller:/opt/platform/inventory/${SW_VERSION} /mnt/inventory" + mkdir /mnt/inventory + timeout 10s nfs-mount controller:/opt/platform/inventory/${SW_VERSION} /mnt/inventory &> /dev/null + RETVAL=$? + # 0 = true + if [ ${RETVAL} -ne 0 ] ; then + logger "$0: Warn: nfs-mount controller:/opt/platform/inventory/${SW_VERSION} /mnt/inventory" + else + mkdir -p $INVENTORY_CONF_DIR + cp /mnt/inventory/inventory.conf.default ${INVENTORY_CONF_FILE} + RETVAL=$? + if [ $? -ne 0 ] ; then + logger "$0: Warn: cp /mnt/inventory/inventory.conf.default ${INVENTORY_CONF_FILE}" + fi + timeout 5s umount /mnt/inventory + rmdir /mnt/inventory + fi + + return ${RETVAL} +} + + +case "$1" in + start) + # Check for installation failure + if [ -f /etc/platform/installation_failed ] ; then + logger "$0: /etc/platform/installation_failed flag is set. Aborting." + exit 1 + fi + + echo -n "Setting up config for inventory-agent: " + if [ -e ${INVENTORY_READY_FLAG} ] ; then + # clear it on every restart, so agent can update it + rm -f ${INVENTORY_READY_FLAG} + fi + + if [ -f ${INVENTORY_CONF_FILE} ] ; then + logger "$0: ${INVENTORY_CONF_FILE} already exists" + RETVAL=0 + else + # Avoid self-mount due to potential nfs issues + echo "Checking for controller-platform-nfs " + + # try for DELAY_SEC seconds to reach controller-platform-nfs + START=`date +%s` + FOUND=0 + while [ $(date +%s) -lt $(( ${START} + ${DELAY_SEC} )) ] ; do + ping -c 1 controller-platform-nfs > /dev/null 2>&1 || ping6 -c 1 controller-platform-nfs > /dev/null 2>&1 + if [ $? -eq 0 ] ; then + FOUND=1 + break + fi + sleep 1 + done + + CONF_COPIED=0 + if [ ${FOUND} -eq 0 ] ; then + # 'controller-platform-nfs' is not available; continue other setup + echo "controller-platform-nfs is not available" + else + # Only required if conf file does not already exist + if [ -f ${INVENTORY_CONF_DEFAULT_FILE} ] ; then + echo "Copying self inventory.conf without mount" + mkdir -p $INVENTORY_CONF_DIR + cp ${INVENTORY_CONF_DEFAULT_FILE} ${INVENTORY_CONF_FILE} + RETVAL=$? + if [ $? -ne 0 ] ; then + logger "$0: Warn: cp /mnt/inventory/inventory.conf.default ${INVENTORY_CONF_FILE} failed. Try mount." + else + CONF_COPIED=1 + fi + fi + if [ ${CONF_COPIED} -eq 0 ] ; then + CONF_COPY_COUNT=0 + while [ $CONF_COPY_COUNT -lt 3 ]; do + if mount_and_copy_config_file ; then + logger "$0: Info: Mount and copy config file PASSED. Attempt: ${CONF_COPY_COUNT}" + break + fi + let CONF_COPY_COUNT=CONF_COPY_COUNT+1 + logger "$0: Warn: Mount and copy config file failed. Attempt: ${CONF_COPY_COUNT}" + done + fi + fi + fi + + if [ -e ${daemon_pidfile} ] ; then + echo "Killing existing process before starting new" + pid=`cat ${daemon_pidfile}` + kill -TERM $pid + rm -f ${daemon_pidfile} + fi + + echo -n "Starting inventory-agent: " + /bin/sh -c "${INVENTORYAGENT}"' >> /dev/null 2>&1 & echo $!' > ${daemon_pidfile} + RETVAL=$? + if [ $RETVAL -eq 0 ] ; then + echo "OK" + touch /var/lock/subsys/${DAEMON_NAME} + else + echo "FAIL" + fi + ;; + + stop) + echo -n "Stopping inventory-agent: " + if [ -e ${daemon_pidfile} ] ; then + pid=`cat ${daemon_pidfile}` + kill -TERM $pid + rm -f ${daemon_pidfile} + rm -f /var/lock/subsys/${DAEMON_NAME} + echo "OK" + else + echo "FAIL" + fi + ;; + + restart) + $0 stop + sleep 1 + $0 start + ;; + + status) + if [ -e ${daemon_pidfile} ] ; then + pid=`cat ${daemon_pidfile}` + ps -p $pid | grep -v "PID TTY" >> /dev/null 2>&1 + if [ $? -eq 0 ] ; then + echo "inventory-agent is running" + RETVAL=0 + else + echo "inventory-agent is not running" + RETVAL=1 + fi + else + echo "inventory-agent is not running ; no pidfile" + RETVAL=1 + fi + ;; + + condrestart) + [ -f /var/lock/subsys/$DAEMON_NAME ] && $0 restart + ;; + + *) + echo "usage: $0 { start | stop | status | restart | condrestart | status }" + ;; +esac + +exit $RETVAL diff --git a/inventory/inventory/scripts/inventory-agent.service b/inventory/inventory/scripts/inventory-agent.service new file mode 100644 index 00000000..c24285fc --- /dev/null +++ b/inventory/inventory/scripts/inventory-agent.service @@ -0,0 +1,15 @@ +[Unit] +Description=Inventory Agent +After=nfscommon.service sw-patch.service +After=network-online.target systemd-udev-settle.service +Before=pmon.service + +[Service] +Type=forking +RemainAfterExit=yes +ExecStart=/etc/init.d/inventory-agent start +ExecStop=/etc/init.d/inventory-agent stop +PIDFile=/var/run/inventory-agent.pid + +[Install] +WantedBy=multi-user.target diff --git a/inventory/inventory/scripts/inventory-api b/inventory/inventory/scripts/inventory-api new file mode 100755 index 00000000..d1bde8eb --- /dev/null +++ b/inventory/inventory/scripts/inventory-api @@ -0,0 +1,409 @@ +#!/bin/sh +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Purpose: This resource agent manages +# +# .... the STX Inventory REST API Service +# +# +# OCF instance parameters: +# OCF_RESKEY_binary +# OCF_RESKEY_client_binary +# OCF_RESKEY_config +# OCF_RESKEY_os_username +# OCF_RESKEY_os_tenant_name +# OCF_RESKEY_os_auth_url +# OCF_RESKEY_os_password +# OCF_RESKEY_user +# OCF_RESKEY_pid +# OCF_RESKEY_additional_parameters +# +# RA Spec: +# +# http://www.opencf.org/cgi-bin/viewcvs.cgi/specs/ra/resource-agent-api.txt?rev=HEAD +# +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +process="inventory" +service="-api" +binname="${process}${service}" + +####################################################################### + +# Fill in some defaults if no values are specified +OCF_RESKEY_binary_default=${binname} +OCF_RESKEY_dbg_default="false" +OCF_RESKEY_user_default="inventory" +OCF_RESKEY_pid_default="/var/run/${binname}.pid" +OCF_RESKEY_config_default="/etc/inventory/inventory.conf" +OCF_RESKEY_client_binary_default="inventory" + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} +: ${OCF_RESKEY_dbg=${OCF_RESKEY_dbg_default}} +: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} +: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}} +: ${OCF_RESKEY_client_binary=${OCF_RESKEY_client_binary_default}} + +mydaemon="/usr/bin/${OCF_RESKEY_binary}" + +####################################################################### + +usage() { + cat < + + +1.0 + + +This 'inventory-api' is an OCF Compliant Resource Agent that manages start, stop +and in-service monitoring of the Inventory REST API Process + + + +Manages the Inventory REST API (inventory-api) process in the STX Platform. + + + + + + + +dbg = false ... info, warn and err logs sent to output stream (default) +dbg = true ... Additional debug logs are also sent to the output stream + +Service Debug Control Option + + + + + +User running Inventory API Service (inventory-api) + +Inventory API Service (inventory-api) user + + + + + + + + + + + + + + +END + return ${OCF_SUCCESS} +} + +inventory_api_validate() { + + local rc + + proc="${binname}:validate" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + check_binary ${OCF_RESKEY_binary} + + if [ ! -f ${OCF_RESKEY_config} ] ; then + ocf_log err "${OCF_RESKEY_binary} ini file missing (${OCF_RESKEY_config})" + return ${OCF_ERR_CONFIGURED} + fi + + getent passwd $OCF_RESKEY_user >/dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "User $OCF_RESKEY_user doesn't exist" + return ${OCF_ERR_CONFIGURED} + fi + + return ${OCF_SUCCESS} +} + +inventory_api_status() { + local pid + local rc + + proc="${binname}:status" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + if [ ! -f $OCF_RESKEY_pid ]; then + ocf_log info "${binname}:Inventory API (inventory-api) is not running" + return $OCF_NOT_RUNNING + else + pid=`cat $OCF_RESKEY_pid` + fi + + ocf_run -warn kill -s 0 $pid + rc=$? + if [ $rc -eq 0 ]; then + return $OCF_SUCCESS + else + ocf_log info "${binname}:Old PID file found, but Inventory API (inventory-api) is not running" + rm -f $OCF_RESKEY_pid + return $OCF_NOT_RUNNING + fi +} + +inventory_api_monitor () { + local rc + proc="${binname}:monitor" + + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + inventory_api_status + rc=$? + # If status returned anything but success, return that immediately + if [ $rc -ne $OCF_SUCCESS ]; then + return $rc + fi + return $OCF_SUCCESS + + if [ -n "$OCF_RESKEY_os_username" ] && [ -n "$OCF_RESKEY_os_tenant_name" ] && [ -n "$OCF_RESKEY_os_auth_url" ]; then + ocf_run -q $OCF_RESKEY_client_binary \ + --os_username "$OCF_RESKEY_os_username" \ + --os_project_name "$OCF_RESKEY_os_tenant_name" \ + --os_auth_url "$OCF_RESKEY_os_auth_url" \ + --os_region_name "$OCF_RESKEY_os_region_name" \ + --system_url "$OCF_RESKEY_system_url" \ + show > /dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "Failed to connect to the Inventory Service (inventory-api): $rc" + return $OCF_NOT_RUNNING + fi + fi + + ocf_log debug "Inventory Service (inventory-api) monitor succeeded" + + return $OCF_SUCCESS +} + +inventory_api_start () { + local rc + + proc="${binname}:start" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + # If running then issue a ping test + if [ -f ${OCF_RESKEY_pid} ] ; then + inventory_api_status + rc=$? + if [ $rc -ne ${OCF_SUCCESS} ] ; then + ocf_log err "${proc} ping test failed (rc=${rc})" + inventory_api_stop + else + return ${OCF_SUCCESS} + fi + fi + + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + RUN_OPT_DEBUG="--debug" + else + RUN_OPT_DEBUG="" + fi + + # switch to non-root user before starting service + su ${OCF_RESKEY_user} -g root -s /bin/sh -c "${OCF_RESKEY_binary} --config-file=${OCF_RESKEY_config} ${RUN_OPT_DEBUG}"' >> /dev/null 2>&1 & echo $!' > $OCF_RESKEY_pid + rc=$? + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + ocf_log err "${proc} failed ${mydaemon} daemon (rc=$rc)" + return ${OCF_ERR_GENERIC} + else + if [ -f ${OCF_RESKEY_pid} ] ; then + pid=`cat ${OCF_RESKEY_pid}` + ocf_log info "${proc} running with pid ${pid}" + else + ocf_log info "${proc} with no pid file" + fi + fi + + # Record success or failure and return status + if [ ${rc} -eq $OCF_SUCCESS ] ; then + ocf_log info "Inventory Service (${OCF_RESKEY_binary}) started (pid=${pid})" + else + ocf_log err "Inventory Service (${OCF_RESKEY_binary}) failed to start (rc=${rc})" + rc=${OCF_NOT_RUNNING} + fi + + return ${rc} +} + +inventory_api_confirm_stop() { + local my_bin + local my_processes + + my_binary=`which ${OCF_RESKEY_binary}` + my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"` + + if [ -n "${my_processes}" ] + then + ocf_log info "About to SIGKILL the following: ${my_processes}" + pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)" + fi +} + +inventory_api_stop () { + local rc + local pid + + proc="${binname}:stop" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + inventory_api_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + ocf_log info "${proc} Inventory API (inventory-api) already stopped" + inventory_api_confirm_stop + return ${OCF_SUCCESS} + fi + + # Try SIGTERM + pid=`cat $OCF_RESKEY_pid` + ocf_run kill -s TERM $pid + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "${proc} Inventory API (inventory-api) couldn't be stopped" + inventory_api_confirm_stop + exit $OCF_ERR_GENERIC + fi + + # stop waiting + shutdown_timeout=15 + if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then + shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5)) + fi + count=0 + while [ $count -lt $shutdown_timeout ]; do + inventory_api_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + break + fi + count=`expr $count + 1` + sleep 1 + ocf_log info "${proc} Inventory API (inventory-api) still hasn't stopped yet. Waiting ..." + done + + inventory_api_status + rc=$? + if [ $rc -ne $OCF_NOT_RUNNING ]; then + # SIGTERM didn't help either, try SIGKILL + ocf_log info "${proc} Inventory API (inventory-api) failed to stop after ${shutdown_timeout}s using SIGTERM. Trying SIGKILL ..." + ocf_run kill -s KILL $pid + fi + inventory_api_confirm_stop + + ocf_log info "${proc} Inventory API (inventory-api) stopped." + + rm -f $OCF_RESKEY_pid + + return $OCF_SUCCESS + +} + +inventory_api_reload () { + local rc + + proc="${binname}:reload" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + inventory_api_stop + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + #sleep 1 + inventory_api_start + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + ocf_log info "Inventory (${OCF_RESKEY_binary}) process restarted" + fi + fi + + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + ocf_log err "Inventory (${OCF_RESKEY_binary}) process failed to restart (rc=${rc})" + fi + + return ${rc} +} + +case ${__OCF_ACTION} in + meta-data) meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) usage + exit ${OCF_SUCCESS} + ;; +esac + +# Anything except meta-data and help must pass validation +inventory_api_validate || exit $? + +if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${binname}:${__OCF_ACTION} action" +fi + +case ${__OCF_ACTION} in + + start) inventory_api_start + ;; + stop) inventory_api_stop + ;; + status) inventory_api_status + ;; + reload) inventory_api_reload + ;; + monitor) inventory_api_monitor + ;; + validate-all) inventory_api_validate + ;; + *) usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac diff --git a/inventory/inventory/scripts/inventory-api.service b/inventory/inventory/scripts/inventory-api.service new file mode 100644 index 00000000..7f7db8b9 --- /dev/null +++ b/inventory/inventory/scripts/inventory-api.service @@ -0,0 +1,15 @@ +[Unit] +Description=Inventory API +After=network-online.target syslog-ng.service config.service inventory-conductor.service + +[Service] +Type=simple +RemainAfterExit=yes +User=root +Environment=OCF_ROOT=/usr/lib/ocf +ExecStart=/usr/lib/ocf/resource.d/platform/inventory-api start +ExecStop=/usr/lib/ocf/resource.d/platform/inventory-api stop +PIDFile=/var/run/inventory-api.pid + +[Install] +WantedBy=multi-user.target diff --git a/inventory/inventory/scripts/inventory-conductor b/inventory/inventory/scripts/inventory-conductor new file mode 100755 index 00000000..61c880fd --- /dev/null +++ b/inventory/inventory/scripts/inventory-conductor @@ -0,0 +1,357 @@ +#!/bin/sh +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# +# +# Support: www.windriver.com +# +# Purpose: This resource agent manages +# +# .... the Inventory Conductor Service +# +# RA Spec: +# +# http://www.opencf.org/cgi-bin/viewcvs.cgi/specs/ra/resource-agent-api.txt?rev=HEAD +# +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +process="inventory" +service="-conductor" +binname="${process}${service}" + +####################################################################### + +# Fill in some defaults if no values are specified +OCF_RESKEY_binary_default=${binname} +OCF_RESKEY_dbg_default="false" +OCF_RESKEY_pid_default="/var/run/${binname}.pid" +OCF_RESKEY_config_default="/etc/inventory/inventory.conf" + + +: ${OCF_RESKEY_binary=${OCF_RESKEY_binary_default}} +: ${OCF_RESKEY_dbg=${OCF_RESKEY_dbg_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} +: ${OCF_RESKEY_config=${OCF_RESKEY_config_default}} + +mydaemon="/usr/bin/${OCF_RESKEY_binary}" + +####################################################################### + +usage() { + cat < + + +1.0 + + +This 'inventory-conductor' is an OCF Compliant Resource Agent that manages start, stop and in-service monitoring of the Inventory Conductor + + + +Manages the Config (inventory-conductor) process. + + + + + + + +dbg = false ... info, warn and err logs sent to output stream (default) +dbg = true ... Additional debug logs are also sent to the output stream + +Service Debug Control Option + + + + + + + + + + + + + + +END + return ${OCF_SUCCESS} +} + +inventory_conductor_validate() { + + local rc + + proc="${binname}:validate" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + check_binary ${OCF_RESKEY_binary} + + if [ ! -f ${OCF_RESKEY_config} ] ; then + ocf_log err "${OCF_RESKEY_binary} ini file missing (${OCF_RESKEY_config})" + return ${OCF_ERR_CONFIGURED} + fi + + return ${OCF_SUCCESS} +} + +inventory_conductor_status() { + local pid + local rc + + proc="${binname}:status" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + if [ ! -f $OCF_RESKEY_pid ]; then + ocf_log info "${binname}:Inventory Conductor (inventory-conductor) is not running" + return $OCF_NOT_RUNNING + else + pid=`cat $OCF_RESKEY_pid` + fi + + ocf_run -warn kill -s 0 $pid + rc=$? + if [ $rc -eq 0 ]; then + return $OCF_SUCCESS + else + ocf_log info "${binname}:Old PID file found, but Inventory Conductor (inventory-conductor)is not running" + rm -f $OCF_RESKEY_pid + return $OCF_NOT_RUNNING + fi +} + +inventory_conductor_monitor () { + local rc + proc="${binname}:monitor" + + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + inventory_conductor_status + rc=$? + return ${rc} +} + +inventory_conductor_start () { + local rc + + proc="${binname}:start" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + # If running then issue a ping test + if [ -f ${OCF_RESKEY_pid} ] ; then + inventory_conductor_status + rc=$? + if [ $rc -ne ${OCF_SUCCESS} ] ; then + ocf_log err "${proc} ping test failed (rc=${rc})" + inventory_conductor_stop + else + return ${OCF_SUCCESS} + fi + fi + + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + RUN_OPT_DEBUG="--debug" + else + RUN_OPT_DEBUG="" + fi + + su ${OCF_RESKEY_user} -s /bin/sh -c "${OCF_RESKEY_binary} --config-file=${OCF_RESKEY_config} ${RUN_OPT_DEBUG}"' >> /dev/null 2>&1 & echo $!' > $OCF_RESKEY_pid + rc=$? + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + ocf_log err "${proc} failed ${mydaemon} daemon (rc=$rc)" + return ${OCF_ERR_GENERIC} + else + if [ -f ${OCF_RESKEY_pid} ] ; then + pid=`cat ${OCF_RESKEY_pid}` + ocf_log info "${proc} running with pid ${pid}" + else + ocf_log info "${proc} with no pid file" + fi + fi + + # Record success or failure and return status + if [ ${rc} -eq $OCF_SUCCESS ] ; then + ocf_log info "Inventory Conductor Service (${OCF_RESKEY_binary}) started (pid=${pid})" + else + ocf_log err "Config Service (${OCF_RESKEY_binary}) failed to start (rc=${rc})" + rc=${OCF_NOT_RUNNING} + fi + + return ${rc} +} + +inventory_conductor_confirm_stop() { + local my_bin + local my_processes + + my_binary=`which ${OCF_RESKEY_binary}` + my_processes=`pgrep -l -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)"` + + if [ -n "${my_processes}" ] + then + ocf_log info "About to SIGKILL the following: ${my_processes}" + pkill -KILL -f "^(python|/usr/bin/python|/usr/bin/python2) ${my_binary}([^\w-]|$)" + fi +} + +inventory_conductor_stop () { + local rc + local pid + + proc="${binname}:stop" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + inventory_conductor_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + ocf_log info "${proc} Inventory Conductor (inventory-conductor) already stopped" + inventory_conductor_confirm_stop + return ${OCF_SUCCESS} + fi + + # Try SIGTERM + pid=`cat $OCF_RESKEY_pid` + ocf_run kill -s TERM $pid + rc=$? + if [ $rc -ne 0 ]; then + ocf_log err "${proc} Inventory Conductor (inventory-conductor) couldn't be stopped" + inventory_conductor_confirm_stop + exit $OCF_ERR_GENERIC + fi + + # stop waiting + shutdown_timeout=15 + if [ -n "$OCF_RESKEY_CRM_meta_timeout" ]; then + shutdown_timeout=$((($OCF_RESKEY_CRM_meta_timeout/1000)-5)) + fi + count=0 + while [ $count -lt $shutdown_timeout ]; do + inventory_conductor_status + rc=$? + if [ $rc -eq $OCF_NOT_RUNNING ]; then + break + fi + count=`expr $count + 1` + sleep 1 + ocf_log info "${proc} Inventory Conductor (inventory-conductor) still hasn't stopped yet. Waiting ..." + done + + inventory_conductor_status + rc=$? + if [ $rc -ne $OCF_NOT_RUNNING ]; then + # SIGTERM didn't help either, try SIGKILL + ocf_log info "${proc} Inventory Conductor (inventory-conductor) failed to stop after ${shutdown_timeout}s \ + using SIGTERM. Trying SIGKILL ..." + ocf_run kill -s KILL $pid + fi + inventory_conductor_confirm_stop + + ocf_log info "${proc} Inventory Conductor (inventory-conductor) stopped." + + rm -f $OCF_RESKEY_pid + + return $OCF_SUCCESS + +} + +inventory_conductor_reload () { + local rc + + proc="${binname}:reload" + if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${proc}" + fi + + inventory_conductor_stop + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + #sleep 1 + inventory_conductor_start + rc=$? + if [ $rc -eq ${OCF_SUCCESS} ] ; then + ocf_log info "Inventory (${OCF_RESKEY_binary}) process restarted" + fi + fi + + if [ ${rc} -ne ${OCF_SUCCESS} ] ; then + ocf_log info "Inventory (${OCF_RESKEY_binary}) process failed to restart (rc=${rc})" + fi + + return ${rc} +} + +case ${__OCF_ACTION} in + meta-data) meta_data + exit ${OCF_SUCCESS} + ;; + usage|help) usage + exit ${OCF_SUCCESS} + ;; +esac + +# Anything except meta-data and help must pass validation +inventory_conductor_validate || exit $? + +if [ ${OCF_RESKEY_dbg} = "true" ] ; then + ocf_log info "${binname}:${__OCF_ACTION} action" +fi + +case ${__OCF_ACTION} in + + start) inventory_conductor_start + ;; + stop) inventory_conductor_stop + ;; + status) inventory_conductor_status + ;; + reload) inventory_conductor_reload + ;; + monitor) inventory_conductor_monitor + ;; + validate-all) inventory_conductor_validate + ;; + *) usage + exit ${OCF_ERR_UNIMPLEMENTED} + ;; +esac diff --git a/inventory/inventory/scripts/inventory-conductor.service b/inventory/inventory/scripts/inventory-conductor.service new file mode 100644 index 00000000..31a6f63d --- /dev/null +++ b/inventory/inventory/scripts/inventory-conductor.service @@ -0,0 +1,15 @@ +[Unit] +Description=Inventory Conductor +After=network-online.target syslog-ng.service config.service rabbitmq-server.service + +[Service] +Type=simple +RemainAfterExit=yes +User=root +Environment=OCF_ROOT=/usr/lib/ocf +ExecStart=/usr/lib/ocf/resource.d/platform/inventory-conductor start +ExecStop=/usr/lib/ocf/resource.d/platform/inventory-conductor stop +PIDFile=/var/run/inventory-conductor.pid + +[Install] +WantedBy=multi-user.target diff --git a/inventory/inventory/setup.cfg b/inventory/inventory/setup.cfg new file mode 100644 index 00000000..56d42c38 --- /dev/null +++ b/inventory/inventory/setup.cfg @@ -0,0 +1,57 @@ +[metadata] +name = inventory +summary = Inventory +description-file = + README.rst +author = StarlingX +author-email = starlingx-discuss@lists.starlingx.io +home-page = http://www.starlingx.io/ +classifier = + Environment :: StarlingX + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + +[files] +packages = + inventory + +[entry_points] +console_scripts = + inventory-api = inventory.cmd.api:main + inventory-agent = inventory.cmd.agent:main + inventory-conductor = inventory.cmd.conductor:main + inventory-dbsync = inventory.cmd.dbsync:main + inventory-dnsmasq-lease-update = inventory.cmd.dnsmasq_lease_update:main + +oslo.config.opts = + inventory.common.config = inventory.common.config:list_opts + inventory.common.api.api_config = inventory.api.api_config:list_opts + +inventory.database.migration_backend = + sqlalchemy = oslo_db.sqlalchemy.migration + +inventory.agent.lldp.drivers = + lldpd = inventory.agent.lldp.drivers.lldpd.driver:InventoryLldpdAgentDriver + ovs = inventory.agent.lldp.drivers.ovs.driver:InventoryOVSAgentDriver + +inventory.systemconfig.drivers = + systemconfig = inventory.systemconfig.drivers.sysinv.driver:SysinvSystemConfigDriver + +[compile_catalog] +directory = inventory/locale +domain = inventory + +[update_catalog] +domain = inventory +output_dir = inventory/locale +input_file = inventory/locale/inventory.pot + +[extract_messages] +keywords = _ gettext ngettext l_ lazy_gettext +mapping_file = babel.cfg +output_file = inventory/locale/inventory.pot diff --git a/inventory/inventory/setup.py b/inventory/inventory/setup.py new file mode 100644 index 00000000..056c16c2 --- /dev/null +++ b/inventory/inventory/setup.py @@ -0,0 +1,29 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/inventory/inventory/test-requirements.txt b/inventory/inventory/test-requirements.txt new file mode 100644 index 00000000..b77cd6bf --- /dev/null +++ b/inventory/inventory/test-requirements.txt @@ -0,0 +1,37 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +hacking>=0.12.0,<0.13 # Apache-2.0 + +coverage>=4.0,!=4.4 # Apache-2.0 +discover +fixtures>=0.3.14 +mock<1.1.0,>=1.0 +mox +MySQL-python +# passlib>=1.7.0 +psycopg2 +python-barbicanclient<3.1.0,>=3.0.1 +python-subunit>=0.0.18 # Apache-2.0/BSD +requests-mock>=0.6.0 # Apache-2.0 +sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 +oslosphinx<2.6.0,>=2.5.0 # Apache-2.0 +stestr>=1.0.0 # Apache-2.0 +testtools>=1.4.0 # MIT +oslotest>=1.10.0 # Apache-2.0 +os-testr>=0.8.0 # Apache-2.0 +testrepository>=0.0.18 +testtools!=1.2.0,>=0.9.36 +tempest-lib<0.5.0,>=0.4.0 +ipaddr +pytest +keyring +pyudev +libvirt-python>=1.2.5 +migrate +python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0 +python-cephclient +python-ldap>=2.4.22,<3.0.0 +markupsafe +Babel>=0.9.6 diff --git a/inventory/inventory/tox.ini b/inventory/inventory/tox.ini new file mode 100644 index 00000000..5e37d4d6 --- /dev/null +++ b/inventory/inventory/tox.ini @@ -0,0 +1,101 @@ +[tox] +minversion = 2.0 +# envlist = pep8 +envlist = py27,pep8 + +# tox does not work if the path to the workdir is too long, so move it to /tmp +toxworkdir = /tmp/{env:USER}_inventorytox +cgcsdir = {toxinidir}/../../.. +wrsdir = {toxinidir}/../../../../../../../.. +distshare={toxworkdir}/.tox/distshare + +[testenv] +# sitepackages = True +install_command = pip install -v -v -v -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/pike} {opts} {packages} +whitelist_externals = bash + find +setenv = + VIRTUAL_ENV={envdir} + PYTHONWARNINGS=default::DeprecationWarning + OS_STDOUT_CAPTURE=1 + OS_STDERR_CAPTURE=1 + OS_TEST_TIMEOUT=60 + INVENTORY_TEST_ENV=True + TOX_WORK_DIR={toxworkdir} + PYLINTHOME={toxworkdir} + +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt + -e{[tox]cgcsdir}/stx-update/tsconfig/tsconfig + -e{[tox]cgcsdir}/stx-config/configutilities/configutilities + -e{[tox]cgcsdir}/stx-config/sysinv/cgts-client/cgts-client + -e{[tox]cgcsdir}/stx-fault/fm-api + -e{[tox]cgcsdir}/stx-fault/python-fmclient/fmclient + -e{[tox]cgcsdir}/stx-config/controllerconfig/controllerconfig + -e{[tox]cgcsdir}/stx-update/cgcs-patch/cgcs-patch + -e{[tox]cgcsdir}/stx-integ/utilities/platform-util/platform-util + +commands = + find . -type f -name "*.pyc" -delete + find . -type f -name ".coverage\.*" -delete + coverage erase + +[testenv:py27] +basepython = python2.7 +commands = + {[testenv]commands} + stestr run {posargs} + stestr slowest + +[testenv:pep8] +commands = flake8 {posargs} + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +deps = {[testenv]deps} + -e{[tox]cgcsdir}/stx-update/tsconfig/tsconfig + -e{[tox]cgcsdir}/stx-config/configutilities/configutilities + -e{[tox]cgcsdir}/stx-fault/fm-api + -e{[tox]cgcsdir}/stx-fault/python-fmclient/fmclient + -e{[tox]cgcsdir}/stx-config/controllerconfig/controllerconfig + -e{[tox]cgcsdir}/stx-config/sysinv/cgts-client/cgts-client + -e{[tox]cgcsdir}/stx-update/cgcs-patch/cgcs-patch + -e{[tox]cgcsdir}/stx-integ/utilities/platform-util/platform-util +setenv = + VIRTUAL_ENV={envdir} + PYTHON=coverage run --source inventory --parallel-mode +commands = + find . -type f -name "*.pyc" -delete + find . -type f -name ".coverage\.*" -delete + stestr run {posargs} + coverage erase + coverage combine + coverage html -d cover + coverage xml -o cover/coverage.xml + +[testenv:docs] +deps = -r{toxinidir}/doc/requirements.txt +commands = sphinx-build -W -b html doc/source doc/build/html + +[testenv:releasenotes] +deps = {[testenv:docs]deps} +commands = + sphinx-build -a -E -W -d releasenotes/build/doctrees -b html releasenotes/source releasenotes/build/html + +[testenv:debug] +commands = oslo_debug_helper {posargs} + +[hacking] +import_exceptions = inventory.common.i18n + +[flake8] +# H102 Apache License format +# H233 Python 3.x incompatible use of print operator +# H404 multi line docstring should start without a leading new line +# H405 multi line docstring summary not separated with an empty line +ignore = H102,H233,H404,H405 +show-source = True +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build diff --git a/python-inventoryclient/PKG-INFO b/python-inventoryclient/PKG-INFO new file mode 100644 index 00000000..9c4108c4 --- /dev/null +++ b/python-inventoryclient/PKG-INFO @@ -0,0 +1,13 @@ +Metadata-Version: 1.1 +Name: python-inventoryclient +Version: 1.0 +Summary: A python client library for Inventory +Home-page: https://wiki.openstack.org/wiki/StarlingX +Author: StarlingX +Author-email: starlingx-discuss@lists.starlingx.io +License: Apache-2.0 + +A python client library for Inventory + + +Platform: UNKNOWN diff --git a/python-inventoryclient/centos/build_srpm.data b/python-inventoryclient/centos/build_srpm.data new file mode 100644 index 00000000..7add43bd --- /dev/null +++ b/python-inventoryclient/centos/build_srpm.data @@ -0,0 +1,2 @@ +SRC_DIR="inventoryclient" +TIS_PATCH_VER=1 diff --git a/python-inventoryclient/centos/python-inventoryclient.spec b/python-inventoryclient/centos/python-inventoryclient.spec new file mode 100644 index 00000000..9dc2a8f2 --- /dev/null +++ b/python-inventoryclient/centos/python-inventoryclient.spec @@ -0,0 +1,82 @@ +%global pypi_name inventoryclient + +Summary: A python client library for Inventory +Name: python-inventoryclient +Version: 1.0 +Release: %{tis_patch_ver}%{?_tis_dist} +License: Apache-2.0 +Group: base +Packager: Wind River +URL: unknown +Source0: %{name}-%{version}.tar.gz + +BuildRequires: git +BuildRequires: python-pbr >= 2.0.0 +BuildRequires: python-setuptools +BuildRequires: python2-pip + +Requires: python-keystoneauth1 >= 3.1.0 +Requires: python-pbr >= 2.0.0 +Requires: python-six >= 1.9.0 +Requires: python-oslo-i18n >= 2.1.0 +Requires: python-oslo-utils >= 3.20.0 +Requires: python-requests +Requires: bash-completion + +%description +A python client library for Inventory + +%define local_bindir /usr/bin/ +%define local_etc_bash_completiond /etc/bash_completion.d/ +%define pythonroot /usr/lib64/python2.7/site-packages + +%define debug_package %{nil} + +%package sdk +Summary: SDK files for %{name} + +%description sdk +Contains SDK files for %{name} package + +%prep +%autosetup -n %{name}-%{version} -S git + +# Remove bundled egg-info +rm -rf *.egg-info + +%build +echo "Start build" + +export PBR_VERSION=%{version} +%{__python} setup.py build + +%install +echo "Start install" +export PBR_VERSION=%{version} +%{__python} setup.py install --root=%{buildroot} \ + --install-lib=%{pythonroot} \ + --prefix=/usr \ + --install-data=/usr/share \ + --single-version-externally-managed + +install -d -m 755 %{buildroot}%{local_etc_bash_completiond} +install -p -D -m 664 tools/inventory.bash_completion %{buildroot}%{local_etc_bash_completiond}/inventory.bash_completion + +# prep SDK package +mkdir -p %{buildroot}/usr/share/remote-clients +tar zcf %{buildroot}/usr/share/remote-clients/%{name}-%{version}.tgz --exclude='.gitignore' --exclude='.gitreview' -C .. %{name}-%{version} + +%clean +echo "CLEAN CALLED" +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root,-) +%doc LICENSE +%{local_bindir}/* +%{local_etc_bash_completiond}/* +%{pythonroot}/%{pypi_name}/* +%{pythonroot}/%{pypi_name}-%{version}*.egg-info + +%files sdk +/usr/share/remote-clients/%{name}-%{version}.tgz diff --git a/python-inventoryclient/inventoryclient/.gitignore b/python-inventoryclient/inventoryclient/.gitignore new file mode 100644 index 00000000..78c457c6 --- /dev/null +++ b/python-inventoryclient/inventoryclient/.gitignore @@ -0,0 +1,35 @@ +# Compiled files +*.py[co] +*.a +*.o +*.so + +# Sphinx +_build +doc/source/api/ + +# Packages/installer info +*.egg +*.egg-info +dist +build +eggs +parts +var +sdist +develop-eggs +.installed.cfg + +# Other +*.DS_Store +.stestr +.testrepository +.tox +.venv +.*.swp +.coverage +bandit.xml +cover +AUTHORS +ChangeLog +*.sqlite diff --git a/python-inventoryclient/inventoryclient/.testr.conf b/python-inventoryclient/inventoryclient/.testr.conf new file mode 100644 index 00000000..d42e8fe4 --- /dev/null +++ b/python-inventoryclient/inventoryclient/.testr.conf @@ -0,0 +1,10 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ + ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./inventoryclient/tests} $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list +# group tests when running concurrently +# This regex groups by classname +#group_regex=([^\.]+\.)+ diff --git a/python-inventoryclient/inventoryclient/LICENSE b/python-inventoryclient/inventoryclient/LICENSE new file mode 100644 index 00000000..68c771a0 --- /dev/null +++ b/python-inventoryclient/inventoryclient/LICENSE @@ -0,0 +1,176 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + diff --git a/python-inventoryclient/inventoryclient/inventoryclient/__init__.py b/python-inventoryclient/inventoryclient/inventoryclient/__init__.py new file mode 100644 index 00000000..15ddbe1f --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/__init__.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +try: + import inventoryclient.client + Client = inventoryclient.client.get_client +except ImportError: + import warnings + warnings.warn("Could not import inventoryclient.client", ImportWarning) + +import pbr.version + +version_info = pbr.version.VersionInfo('inventoryclient') + +try: + __version__ = version_info.version_string() +except AttributeError: + __version__ = None diff --git a/python-inventoryclient/inventoryclient/inventoryclient/client.py b/python-inventoryclient/inventoryclient/inventoryclient/client.py new file mode 100644 index 00000000..9fec2f98 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/client.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventoryclient.common.i18n import _ +from inventoryclient import exc +from keystoneauth1 import loading +from oslo_utils import importutils + + +SERVICE_TYPE = 'configuration' # TODO(jkung) This needs to be inventory + + +def get_client(version, endpoint=None, session=None, auth_token=None, + inventory_url=None, username=None, password=None, auth_url=None, + project_id=None, project_name=None, + region_name=None, timeout=None, + user_domain_id=None, user_domain_name=None, + project_domain_id=None, project_domain_name=None, + service_type=SERVICE_TYPE, endpoint_type=None, + **ignored_kwargs): + """Get an authenticated client, based on the credentials.""" + kwargs = {} + interface = endpoint_type or 'publicURL' + endpoint = endpoint or inventory_url + if auth_token and endpoint: + kwargs.update({ + 'token': auth_token, + }) + if timeout: + kwargs.update({ + 'timeout': timeout, + }) + elif auth_url: + auth_kwargs = {} + auth_type = 'password' + auth_kwargs.update({ + 'auth_url': auth_url, + 'project_id': project_id, + 'project_name': project_name, + 'user_domain_id': user_domain_id, + 'user_domain_name': user_domain_name, + 'project_domain_id': project_domain_id, + 'project_domain_name': project_domain_name, + }) + if username and password: + auth_kwargs.update({ + 'username': username, + 'password': password + }) + elif auth_token: + auth_type = 'token' + auth_kwargs.update({ + 'token': auth_token, + }) + + # Create new session only if it was not passed in + if not session: + loader = loading.get_plugin_loader(auth_type) + auth_plugin = loader.load_from_options(**auth_kwargs) + session = loading.session.Session().load_from_options( + auth=auth_plugin, timeout=timeout) + + exception_msg = _('Must provide Keystone credentials or user-defined ' + 'endpoint and token') + if not endpoint: + if session: + try: + endpoint = session.get_endpoint( + service_type=service_type, + interface=interface, + region_name=region_name + ) + except Exception as e: + raise exc.AuthSystem( + _('%(message)s, error was: %(error)s') % + {'message': exception_msg, 'error': e}) + else: + # Neither session, nor valid auth parameters provided + raise exc.AuthSystem(exception_msg) + + kwargs['endpoint_override'] = endpoint + kwargs['service_type'] = service_type + kwargs['interface'] = interface + kwargs['version'] = version + + inventory_module = importutils.import_versioned_module( + 'inventoryclient', version, 'client') + client_class = getattr(inventory_module, 'Client') + return client_class(endpoint, session=session, **kwargs) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/__init__.py b/python-inventoryclient/inventoryclient/inventoryclient/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/base.py b/python-inventoryclient/inventoryclient/inventoryclient/common/base.py new file mode 100644 index 00000000..885f5ac7 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/base.py @@ -0,0 +1,164 @@ +# Copyright 2013 Wind River, Inc. +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Base utilities to build API operation managers and objects on top of. +""" + +import copy +from inventoryclient import exc + + +def getid(obj): + """Abstracts the common pattern of allowing both an object or an + object's ID (UUID) as a parameter when dealing with relationships. + """ + try: + return obj.id + except AttributeError: + return obj + + +class Resource(object): + """A resource represents a particular instance of an object (tenant, user, + etc). This is pretty much just a bag for attributes. + + :param manager: Manager object + :param info: dictionary representing resource attributes + :param loaded: prevent lazy-loading if set to True + """ + def __init__(self, manager, info, loaded=False): + self.manager = manager + self._info = info + self._add_details(info) + self._loaded = loaded + + def _add_details(self, info): + for (k, v) in info.iteritems(): + setattr(self, k, v) + + def __getattr__(self, k): + if k not in self.__dict__: + # NOTE(bcwaldon): disallow lazy-loading if already loaded once + if not self.is_loaded(): + self.get() + return self.__getattr__(k) + + raise AttributeError(k) + else: + return self.__dict__[k] + + def __repr__(self): + reprkeys = sorted(k for k in self.__dict__.keys() if k[0] != '_' and + k != 'manager') + info = ", ".join("%s=%s" % (k, getattr(self, k)) for k in reprkeys) + return "<%s %s>" % (self.__class__.__name__, info) + + def get(self): + # set_loaded() first ... so if we have to bail, we know we tried. + self.set_loaded(True) + if not hasattr(self.manager, 'get'): + return + + new = self.manager.get(self.id) + if new: + self._add_details(new._info) + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + if hasattr(self, 'id') and hasattr(other, 'id'): + return self.id == other.id + return self._info == other._info + + def is_loaded(self): + return self._loaded + + def set_loaded(self, val): + self._loaded = val + + def to_dict(self): + return copy.deepcopy(self._info) + + +class ResourceClassNone(Resource): + def __repr__(self): + return "" % self._info + + +class Manager(object): + """Managers interact with a particular type of API and provide CRUD + operations for them. + """ + resource_class = ResourceClassNone + + def __init__(self, api): + self.api = api + + def _create(self, url, body): + resp, body = self.api.post(url, data=body) + if body: + if callable(self.resource_class): + return self.resource_class(self, body) + else: + raise exc.InvalidAttribute(url) + + def _upload(self, url, body, data=None): + files = {'file': ("for_upload", + body, + )} + resp = self.api.post(url, files=files, data=data) + return resp + + def _json_get(self, url): + """send a GET request and return a json serialized object""" + resp, body = self.api.get(url) + return body + + def _format_body_data(self, body, response_key): + if response_key: + try: + data = body[response_key] + except KeyError: + return [] + else: + data = body + + if not isinstance(data, list): + data = [data] + + return data + + def _list(self, url, response_key=None, obj_class=None, body=None): + resp, body = self.api.get(url) + + if obj_class is None: + obj_class = self.resource_class + + data = self._format_body_data(body, response_key) + return [obj_class(self, res, loaded=True) for res in data if res] + + def _update(self, url, **kwargs): + resp, body = self.api.patch(url, **kwargs) + # PATCH/PUT requests may not return a body + if body: + if callable(self.resource_class): + return self.resource_class(self, body) + else: + raise exc.InvalidAttribute(url) + + def _delete(self, url): + self.api.delete(url) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/cli_no_wrap.py b/python-inventoryclient/inventoryclient/inventoryclient/common/cli_no_wrap.py new file mode 100644 index 00000000..861a08c9 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/cli_no_wrap.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +The sole purpose of this module is to manage access to the _no_wrap variable +used by the wrapping_formatters module +""" + +_no_wrap = [False] + + +def is_nowrap_set(no_wrap=None): + """ + returns True if no wrapping desired. + determines this by either the no_wrap parameter + or if the global no_wrap flag is set + :param no_wrap: + :return: + """ + global _no_wrap + if no_wrap is True: + return True + if no_wrap is False: + return False + no_wrap = _no_wrap[0] + return no_wrap + + +def set_no_wrap(no_wrap): + """ + Sets the global nowrap flag + then returns result of call to is_nowrap_set(..) + :param no_wrap: + :return: + """ + global _no_wrap + if no_wrap is not None: + _no_wrap[0] = no_wrap + return is_nowrap_set(no_wrap) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/exceptions.py b/python-inventoryclient/inventoryclient/inventoryclient/common/exceptions.py new file mode 100644 index 00000000..fefc6087 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/exceptions.py @@ -0,0 +1,226 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import inspect +from inventoryclient.common.i18n import _ +import json +import six +from six.moves import http_client +import sys + + +class ClientException(Exception): + """An error occurred.""" + def __init__(self, message=None): + self.message = message + + def __str__(self): + return self.message or self.__class__.__doc__ + + +class InvalidEndpoint(ClientException): + """The provided endpoint is invalid.""" + + +class EndpointException(ClientException): + """Something is rotten in Service Catalog.""" + + +class CommunicationError(ClientException): + """Unable to communicate with server.""" + + +class Conflict(ClientException): + """HTTP 409 - Conflict. + + Indicates that the request could not be processed because of conflict + in the request, such as an edit conflict. + """ + http_status = http_client.CONFLICT + message = _("Conflict") + + +# _code_map contains all the classes that have http_status attribute. +_code_map = dict( + (getattr(obj, 'http_status', None), obj) + for name, obj in vars(sys.modules[__name__]).items() + if inspect.isclass(obj) and getattr(obj, 'http_status', False) +) + + +class HttpError(ClientException): + """The base exception class for all HTTP exceptions.""" + http_status = 0 + message = _("HTTP Error") + + def __init__(self, message=None, details=None, + response=None, request_id=None, + url=None, method=None, http_status=None): + self.http_status = http_status or self.http_status + self.message = message or self.message + self.details = details + self.request_id = request_id + self.response = response + self.url = url + self.method = method + formatted_string = "%s (HTTP %s)" % (self.message, self.http_status) + if request_id: + formatted_string += " (Request-ID: %s)" % request_id + super(HttpError, self).__init__(formatted_string) + + +class HTTPRedirection(HttpError): + """HTTP Redirection.""" + message = _("HTTP Redirection") + + +def _extract_error_json(body): + error_json = {} + try: + body_json = json.loads(body) + if 'error_message' in body_json: + raw_msg = body_json['error_message'] + error_json = json.loads(raw_msg) + except ValueError: + return {} + + return error_json + + +class HTTPClientError(HttpError): + """Client-side HTTP error. + + Exception for cases in which the client seems to have erred. + """ + message = _("HTTP Client Error") + + def __init__(self, message=None, details=None, + response=None, request_id=None, + url=None, method=None, http_status=None): + if method: + error_json = _extract_error_json(method) + message = error_json.get('faultstring') + super(HTTPClientError, self).__init__( + message=message, + details=details, + response=response, + request_id=request_id, + url=url, + method=method, + http_status=http_status) + + +class NotFound(HTTPClientError): + """HTTP 404 - Not Found. + + The requested resource could not be found but may be available again + in the future. + """ + http_status = 404 + message = "Not Found" + + +class HttpServerError(HttpError): + """Server-side HTTP error. + + Exception for cases in which the server is aware that it has + erred or is incapable of performing the request. + """ + message = _("HTTP Server Error") + + def __init__(self, message=None, details=None, + response=None, request_id=None, + url=None, method=None, http_status=None): + if method: + error_json = _extract_error_json(method) + message = error_json.get('faultstring') + super(HttpServerError, self).__init__( + message=message, + details=details, + response=response, + request_id=request_id, + url=url, + method=method, + http_status=http_status) + + +class ServiceUnavailable(HttpServerError): + """HTTP 503 - Service Unavailable. + + The server is currently unavailable. + """ + http_status = http_client.SERVICE_UNAVAILABLE + message = _("Service Unavailable") + + +class GatewayTimeout(HttpServerError): + """HTTP 504 - Gateway Timeout. + + The server was acting as a gateway or proxy and did not receive a timely + response from the upstream server. + """ + http_status = http_client.GATEWAY_TIMEOUT + message = "Gateway Timeout" + + +class HttpVersionNotSupported(HttpServerError): + """HTTP 505 - HttpVersion Not Supported. + + The server does not support the HTTP protocol version used in the request. + """ + http_status = http_client.HTTP_VERSION_NOT_SUPPORTED + message = "HTTP Version Not Supported" + + +def from_response(response, method, url=None): + """Returns an instance of :class:`HttpError` or subclass based on response. + + :param response: instance of `requests.Response` class + :param method: HTTP method used for request + :param url: URL used for request + """ + + req_id = response.headers.get("x-openstack-request-id") + kwargs = { + "http_status": response.status_code, + "response": response, + "method": method, + "url": url, + "request_id": req_id, + } + if "retry-after" in response.headers: + kwargs["retry_after"] = response.headers["retry-after"] + + content_type = response.headers.get("Content-Type", "") + if content_type.startswith("application/json"): + try: + body = response.json() + except ValueError: + pass + else: + if isinstance(body, dict): + error = body.get(list(body)[0]) + if isinstance(error, dict): + kwargs["message"] = (error.get("message") or + error.get("faultstring")) + kwargs["details"] = (error.get("details") or + six.text_type(body)) + elif content_type.startswith("text/"): + kwargs["details"] = getattr(response, 'text', '') + + try: + cls = _code_map[response.status_code] + except KeyError: + if 500 <= response.status_code < 600: + cls = HttpServerError + elif 400 <= response.status_code < 500: + cls = HTTPClientError + elif 404 == response.status_code: + cls = NotFound + else: + cls = HttpError + return cls(**kwargs) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/http.py b/python-inventoryclient/inventoryclient/inventoryclient/common/http.py new file mode 100644 index 00000000..ab8a30ea --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/http.py @@ -0,0 +1,402 @@ +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import json +import logging +import six +import socket + +from keystoneauth1 import adapter +from keystoneauth1 import exceptions as ksa_exc + +import OpenSSL +from oslo_utils import encodeutils +from oslo_utils import importutils +from oslo_utils import netutils +import requests + + +from inventoryclient.common import exceptions as exc +from inventoryclient.common import utils + +osprofiler_web = importutils.try_import("osprofiler.web") + +LOG = logging.getLogger(__name__) + +DEFAULT_VERSION = '1' +USER_AGENT = 'python-inventoryclient' +CHUNKSIZE = 1024 * 64 # 64kB +REQ_ID_HEADER = 'X-OpenStack-Request-ID' + +API_VERSION = '/v1' +API_VERSION_SELECTED_STATES = ('user', 'negotiated', 'cached', 'default') + +SENSITIVE_HEADERS = ('X-Auth-Token',) + +SUPPORTED_ENDPOINT_SCHEME = ('http', 'https') + + +def encode_headers(headers): + """Encodes headers. + + Note: This should be used right before + sending anything out. + + :param headers: Headers to encode + :returns: Dictionary with encoded headers' + names and values + """ + return dict((encodeutils.safe_encode(h), encodeutils.safe_encode(v)) + for h, v in headers.items() if v is not None) + + +class _BaseHTTPClient(object): + + @staticmethod + def _chunk_body(body): + chunk = body + while chunk: + chunk = body.read(CHUNKSIZE) + if not chunk: + break + yield chunk + + def _set_common_request_kwargs(self, headers, kwargs, skip_dumps=False): + """Handle the common parameters used to send the request.""" + + # Default Content-Type is json + content_type = headers.get('Content-Type', 'application/json') + + # NOTE(jamielennox): remove this later. Managers should pass json= if + # they want to send json data. + data = kwargs.pop("data", None) + if data is not None and not isinstance(data, six.string_types): + try: + if not skip_dumps: + data = json.dumps(data) + content_type = 'application/json' + except TypeError: + # Here we assume it's + # a file-like object + # and we'll chunk it + data = self._chunk_body(data) + + if not skip_dumps: + headers['Content-Type'] = content_type + + return data + + def _handle_response(self, resp): + if not resp.ok: + LOG.error("Request returned failure status %s.", resp.status_code) + raise exc.from_response(resp, resp.content) + elif (resp.status_code == requests.codes.multiple_choices and + resp.request.path_url != '/versions'): + # NOTE(flaper87): Eventually, we'll remove the check on `versions` + # which is a bug (1491350) on the server. + raise exc.from_response(resp, resp.content) + + content_type = resp.headers.get('Content-Type') + + # Read body into string if it isn't obviously image data + if content_type == 'application/octet-stream': + # Do not read all response in memory when downloading an image. + body_iter = _close_after_stream(resp, CHUNKSIZE) + else: + content = resp.text + if content_type and content_type.startswith('application/json'): + # Let's use requests json method, it should take care of + # response encoding + body_iter = resp.json() + else: + body_iter = six.StringIO(content) + try: + body_iter = json.loads(''.join([c for c in body_iter])) + except ValueError: + body_iter = None + + return resp, body_iter + + def upload_request_with_data(self, url, auth_token, files, data): + headers = {"X-Auth-Token": auth_token} + req = requests.post(url, headers=headers, files=files, data=data) + return req.json() + + +class HTTPClient(_BaseHTTPClient): + + def __init__(self, endpoint, **kwargs): + self.endpoint = endpoint + self.identity_headers = kwargs.get('identity_headers') + self.auth_token = kwargs.get('token') + self.language_header = kwargs.get('language_header') + self.global_request_id = kwargs.get('global_request_id') + if self.identity_headers: + self.auth_token = self.identity_headers.pop('X-Auth-Token', + self.auth_token) + + self.session = requests.Session() + self.session.headers["User-Agent"] = USER_AGENT + + if self.language_header: + self.session.headers["Accept-Language"] = self.language_header + + self.timeout = float(kwargs.get('timeout', 600)) + + if self.endpoint.startswith("https"): + + if kwargs.get('insecure', False) is True: + self.session.verify = False + else: + if kwargs.get('cacert', None) is not '': + self.session.verify = kwargs.get('cacert', True) + + self.session.cert = (kwargs.get('cert_file'), + kwargs.get('key_file')) + + @staticmethod + def parse_endpoint(endpoint): + return netutils.urlsplit(endpoint) + + def log_curl_request(self, method, url, headers, data, kwargs): + curl = ['curl -g -i -X %s' % method] + + headers = copy.deepcopy(headers) + headers.update(self.session.headers) + + for (key, value) in headers.items(): + header = '-H \'%s: %s\'' % utils.safe_header(key, value) + curl.append(header) + + if not self.session.verify: + curl.append('-k') + else: + if isinstance(self.session.verify, six.string_types): + curl.append(' --cacert %s' % self.session.verify) + + if self.session.cert: + curl.append(' --cert %s --key %s' % self.session.cert) + + if data and isinstance(data, six.string_types): + curl.append('-d \'%s\'' % data) + + curl.append(url) + + msg = ' '.join([encodeutils.safe_decode(item, errors='ignore') + for item in curl]) + LOG.debug(msg) + + @staticmethod + def log_http_response(resp): + status = (resp.raw.version / 10.0, resp.status_code, resp.reason) + dump = ['\nHTTP/%.1f %s %s' % status] + headers = resp.headers.items() + dump.extend(['%s: %s' % utils.safe_header(k, v) for k, v in headers]) + dump.append('') + content_type = resp.headers.get('Content-Type') + + if content_type != 'application/octet-stream': + dump.extend([resp.text, '']) + LOG.debug('\n'.join([encodeutils.safe_decode(x, errors='ignore') + for x in dump])) + + def _request(self, method, url, **kwargs): + """Send an http request with the specified characteristics. + + Wrapper around httplib.HTTP(S)Connection.request to handle tasks such + as setting headers and error handling. + """ + # Copy the kwargs so we can reuse the original in case of redirects + headers = copy.deepcopy(kwargs.pop('headers', {})) + + if self.identity_headers: + for k, v in self.identity_headers.items(): + headers.setdefault(k, v) + + data = self._set_common_request_kwargs(headers, kwargs) + + # add identity header to the request + if not headers.get('X-Auth-Token'): + headers['X-Auth-Token'] = self.auth_token + + if self.global_request_id: + headers.setdefault(REQ_ID_HEADER, self.global_request_id) + + if osprofiler_web: + headers.update(osprofiler_web.get_trace_id_headers()) + + # Note(flaper87): Before letting headers / url fly, + # they should be encoded otherwise httplib will + # complain. + headers = encode_headers(headers) + + # Since some packages send sysinv endpoint with 'v1' and some don't, + # the postprocessing for both options will be done here + # Instead of doing a fix in each of these packages + if 'v1' in self.endpoint and 'v1' in url: + # remove the '/v1' from endpoint + self.endpoint = self.endpoint.replace('/v1', '', 1) + elif 'v1' not in self.endpoint and 'v1' not in url: + self.endpoint = self.endpoint.rstrip('/') + '/v1' + + if self.endpoint.endswith("/") or url.startswith("/"): + conn_url = "%s%s" % (self.endpoint, url) + else: + conn_url = "%s/%s" % (self.endpoint, url) + self.log_curl_request(method, conn_url, headers, data, kwargs) + + try: + resp = self.session.request(method, + conn_url, + data=data, + headers=headers, + **kwargs) + except requests.exceptions.Timeout as e: + message = ("Error communicating with %(url)s: %(e)s" % + dict(url=conn_url, e=e)) + raise exc.InvalidEndpoint(message=message) + except requests.exceptions.ConnectionError as e: + message = ("Error finding address for %(url)s: %(e)s" % + dict(url=conn_url, e=e)) + raise exc.CommunicationError(message=message) + except socket.gaierror as e: + message = "Error finding address for %s %s: %s" % ( + self.endpoint, conn_url, e) + raise exc.InvalidEndpoint(message=message) + except (socket.error, socket.timeout, IOError) as e: + endpoint = self.endpoint + message = ("Error communicating with %(endpoint)s %(e)s" % + {'endpoint': endpoint, 'e': e}) + raise exc.CommunicationError(message=message) + except OpenSSL.SSL.Error as e: + message = ("SSL Error communicating with %(url)s: %(e)s" % + {'url': conn_url, 'e': e}) + raise exc.CommunicationError(message=message) + + # log request-id for each api call + request_id = resp.headers.get('x-openstack-request-id') + if request_id: + LOG.debug('%(method)s call to image for ' + '%(url)s used request id ' + '%(response_request_id)s', + {'method': resp.request.method, + 'url': resp.url, + 'response_request_id': request_id}) + + resp, body_iter = self._handle_response(resp) + self.log_http_response(resp) + return resp, body_iter + + def head(self, url, **kwargs): + return self._request('HEAD', url, **kwargs) + + def get(self, url, **kwargs): + return self._request('GET', url, **kwargs) + + def post(self, url, **kwargs): + return self._request('POST', url, **kwargs) + + def put(self, url, **kwargs): + return self._request('PUT', url, **kwargs) + + def patch(self, url, **kwargs): + return self._request('PATCH', url, **kwargs) + + def delete(self, url, **kwargs): + return self._request('DELETE', url, **kwargs) + + +def _close_after_stream(response, chunk_size): + """Iterate over the content and ensure the response is closed after.""" + # Yield each chunk in the response body + for chunk in response.iter_content(chunk_size=chunk_size): + yield chunk + # Once we're done streaming the body, ensure everything is closed. + # This will return the connection to the HTTPConnectionPool in urllib3 + # and ideally reduce the number of HTTPConnectionPool full warnings. + response.close() + + +class SessionClient(adapter.Adapter, _BaseHTTPClient): + + def __init__(self, session, **kwargs): + kwargs.setdefault('user_agent', USER_AGENT) + self.global_request_id = kwargs.pop('global_request_id', None) + super(SessionClient, self).__init__(session, **kwargs) + + def request(self, url, method, **kwargs): + headers = kwargs.pop('headers', {}) + + if 'v1' in url: + # remove the '/v1' from endpoint + # TODO(jkung) Remove when service catalog is updated + url = url.replace('/v1', '', 1) + + if self.global_request_id: + headers.setdefault(REQ_ID_HEADER, self.global_request_id) + + kwargs['raise_exc'] = False + file_to_upload = kwargs.get('files') + if file_to_upload: + skip_dumps = True + else: + skip_dumps = False + + data = self._set_common_request_kwargs(headers, kwargs, + skip_dumps=skip_dumps) + try: + if file_to_upload: + auth_token = super(SessionClient, self).get_token() + endpoint = super(SessionClient, + self).get_endpoint() + url = endpoint + url + return self.upload_request_with_data(url, + auth_token, + file_to_upload, + data=data) + + # NOTE(pumaranikar): To avoid bug #1641239, no modification of + # headers should be allowed after encode_headers() is called. + resp = super(SessionClient, + self).request(url, + method, + headers=encode_headers(headers), + data=data, + **kwargs) + except ksa_exc.ConnectTimeout as e: + conn_url = self.get_endpoint(auth=kwargs.get('auth')) + conn_url = "%s/%s" % (conn_url.rstrip('/'), url.lstrip('/')) + message = ("Error communicating with %(url)s %(e)s" % + dict(url=conn_url, e=e)) + raise exc.InvalidEndpoint(message=message) + except ksa_exc.ConnectFailure as e: + conn_url = self.get_endpoint(auth=kwargs.get('auth')) + conn_url = "%s/%s" % (conn_url.rstrip('/'), url.lstrip('/')) + message = ("Error finding address for %(url)s: %(e)s" % + dict(url=conn_url, e=e)) + raise exc.CommunicationError(message=message) + + return self._handle_response(resp) + + +def get_http_client(endpoint=None, session=None, **kwargs): + if session: + return SessionClient(session, **kwargs) + elif endpoint: + return HTTPClient(endpoint, **kwargs) + else: + raise AttributeError('Constructing a client must contain either an ' + 'endpoint or a session') diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/i18n.py b/python-inventoryclient/inventoryclient/inventoryclient/common/i18n.py new file mode 100644 index 00000000..1f0f53b2 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/i18n.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import oslo_i18n + +_translators = oslo_i18n.TranslatorFactory(domain='inventoryclient') + +# The primary translation function using the well-known name "_" +_ = _translators.primary diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/options.py b/python-inventoryclient/inventoryclient/inventoryclient/common/options.py new file mode 100644 index 00000000..cea260bf --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/options.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import re + +from six.moves import urllib + +OP_LOOKUP = {'!=': 'ne', + '>=': 'ge', + '<=': 'le', + '>': 'gt', + '<': 'lt', + '=': 'eq'} + +OP_LOOKUP_KEYS = '|'.join(sorted(OP_LOOKUP.keys(), key=len, reverse=True)) +OP_SPLIT_RE = re.compile(r'(%s)' % OP_LOOKUP_KEYS) + +DATA_TYPE_RE = re.compile(r'^(string|integer|float|datetime|boolean)(::)(.+)$') + + +def build_url(path, q, params=None): + """Convert list of dicts and a list of params to query url format. + + This will convert the following: + "[{field=this,op=le,value=34}, + {field=that,op=eq,value=foo,type=string}], + ['foo=bar','sna=fu']" + to: + "?q.field=this&q.field=that& + q.op=le&q.op=eq& + q.type=&q.type=string& + q.value=34&q.value=foo& + foo=bar&sna=fu" + """ + if q: + query_params = {'q.field': [], + 'q.value': [], + 'q.op': [], + 'q.type': []} + + for query in q: + for name in ['field', 'op', 'value', 'type']: + query_params['q.%s' % name].append(query.get(name, '')) + + # Transform the dict to a sequence of two-element tuples in fixed + # order, then the encoded string will be consistent in Python 2&3. + new_qparams = sorted(query_params.items(), key=lambda x: x[0]) + path += "?" + urllib.parse.urlencode(new_qparams, doseq=True) + + if params: + for p in params: + path += '&%s' % p + elif params: + path += '?%s' % params[0] + for p in params[1:]: + path += '&%s' % p + return path + + +def cli_to_array(cli_query): + """Convert CLI list of queries to the Python API format. + + This will convert the following: + "this<=34;that=string::foo" + to + "[{field=this,op=le,value=34,type=''}, + {field=that,op=eq,value=foo,type=string}]" + + """ + + if cli_query is None: + return None + + def split_by_op(query): + """Split a single query string to field, operator, value.""" + + def _value_error(message): + raise ValueError('invalid query %(query)s: missing %(message)s' % + {'query': query, 'message': message}) + + try: + field, operator, value = OP_SPLIT_RE.split(query, maxsplit=1) + except ValueError: + _value_error('operator') + + if not len(field): + _value_error('field') + + if not len(value): + _value_error('value') + + return field.strip(), operator, value.strip() + + def split_by_data_type(query_value): + frags = DATA_TYPE_RE.match(query_value) + + # The second match is the separator. Return a list without it if + # a type identifier was found. + return frags.group(1, 3) if frags else None + + opts = [] + queries = cli_query.split(';') + for q in queries: + query = split_by_op(q) + opt = {} + opt['field'] = query[0] + opt['op'] = OP_LOOKUP[query[1]] + + # Allow the data type of the value to be specified via ::, + # where type can be one of integer, string, float, datetime, boolean + value_frags = split_by_data_type(query[2]) + if not value_frags: + opt['value'] = query[2] + opt['type'] = '' + else: + opt['type'] = value_frags[0] + opt['value'] = value_frags[1] + opts.append(opt) + return opts diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/utils.py b/python-inventoryclient/inventoryclient/inventoryclient/common/utils.py new file mode 100644 index 00000000..c6e16d2e --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/utils.py @@ -0,0 +1,777 @@ +# Copyright 2012 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +# from __future__ import print_function + +import argparse +import copy +import dateutil +import hashlib +import os +import prettytable +import re +import six +import six.moves.urllib.parse as urlparse +import textwrap +import uuid + +from datetime import datetime +from dateutil import parser +from functools import wraps +from inventoryclient import exc + +from prettytable import ALL +from prettytable import FRAME +from prettytable import NONE + +import wrapping_formatters + + +SENSITIVE_HEADERS = ('X-Auth-Token', ) + + +class HelpFormatter(argparse.HelpFormatter): + def start_section(self, heading): + # Title-case the headings + heading = '%s%s' % (heading[0].upper(), heading[1:]) + super(HelpFormatter, self).start_section(heading) + + +def safe_header(name, value): + if value is not None and name in SENSITIVE_HEADERS: + h = hashlib.sha1(value) + d = h.hexdigest() + return name, "{SHA1}%s" % d + else: + return name, value + + +def strip_version(endpoint): + if not isinstance(endpoint, six.string_types): + raise ValueError("Expected endpoint") + version = None + # Get rid of trailing '/' if present + endpoint = endpoint.rstrip('/') + url_parts = urlparse.urlparse(endpoint) + (scheme, netloc, path, __, __, __) = url_parts + path = path.lstrip('/') + # regex to match 'v1' or 'v2.0' etc + if re.match('v\d+\.?\d*', path): + version = float(path.lstrip('v')) + endpoint = scheme + '://' + netloc + return endpoint, version + + +def endpoint_version_from_url(endpoint, default_version=None): + if endpoint: + endpoint, version = strip_version(endpoint) + return endpoint, version or default_version + else: + return None, default_version + + +def env(*vars, **kwargs): + """Search for the first defined of possibly many env vars + + Returns the first environment variable defined in vars, or + returns the default defined in kwargs. + """ + for v in vars: + value = os.environ.get(v, None) + if value: + return value + return kwargs.get('default', '') + + +# noinspection PyUnusedLocal +def _wrapping_formatter_callback_decorator(subparser, command, callback): + """ + - Adds the --nowrap option to a CLI command. + This option, when on, deactivates word wrapping. + - Decorates the command's callback function in order to process + the nowrap flag + + :param subparser: + :return: decorated callback + """ + + try: + subparser.add_argument('--nowrap', action='store_true', + help='No wordwrapping of output') + except Exception: + # exception happens when nowrap option already configured + # for command - so get out with callback undecorated + return callback + + def no_wrap_decorator_builder(callback): + + def process_callback_with_no_wrap(cc, args={}): + no_wrap = args.nowrap + # turn on/off wrapping formatters when outputting CLI results + wrapping_formatters.set_no_wrap(no_wrap) + return callback(cc, args=args) + + return process_callback_with_no_wrap + + decorated_callback = no_wrap_decorator_builder(callback) + return decorated_callback + + +def _does_command_need_no_wrap(callback): + if callback.__name__.startswith("do_") and \ + callback.__name__.endswith("_list"): + return True + + if callback.__name__ in \ + ['donot_config_ntp_list', + 'donot_config_ptp_list', + 'do_host_apply_memprofile', + 'do_host_apply_cpuprofile', + 'do_host_apply_ifprofile', + 'do_host_apply_profile', + 'do_host_apply_storprofile', + 'donot_config_oam_list', + 'donot_dns_list', + 'do_host_cpu_modify', + 'do_event_suppress', + 'do_event_unsuppress', + 'do_event_unsuppress_all']: + return True + return False + + +def get_terminal_size(): + """Returns a tuple (x, y) representing the width(x) and the height(x) + in characters of the terminal window. + """ + + def ioctl_GWINSZ(fd): + try: + import fcntl + import struct + import termios + cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, + '1234')) + except Exception: + return None + if cr == (0, 0): + return None + if cr == (0, 0): + return None + return cr + + cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + if not cr: + try: + fd = os.open(os.ctermid(), os.O_RDONLY) + cr = ioctl_GWINSZ(fd) + os.close(fd) + except Exception: + pass + if not cr: + cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) + return int(cr[1]), int(cr[0]) + + +def normalize_field_data(obj, fields): + for f in fields: + if hasattr(obj, f): + data = getattr(obj, f, '') + try: + data = str(data) + except UnicodeEncodeError: + setattr(obj, f, data.encode('utf-8')) + + +# Decorator for cli-args +def arg(*args, **kwargs): + def _decorator(func): + # Because of the sematics of decorator composition if we just append + # to the options list positional options will appear to be backwards. + func.__dict__.setdefault('arguments', []).insert(0, (args, kwargs)) + return func + + return _decorator + + +def define_command(subparsers, command, callback, cmd_mapper): + '''Define a command in the subparsers collection. + + :param subparsers: subparsers collection where the command will go + :param command: command name + :param callback: function that will be used to process the command + ''' + desc = callback.__doc__ or '' + help = desc.strip().split('\n')[0] + arguments = getattr(callback, 'arguments', []) + + subparser = subparsers.add_parser(command, help=help, + description=desc, + add_help=False, + formatter_class=HelpFormatter) + subparser.add_argument('-h', '--help', action='help', + help=argparse.SUPPRESS) + + # Are we a list command? + if _does_command_need_no_wrap(callback): + # then decorate it with wrapping data formatter functionality + func = _wrapping_formatter_callback_decorator( + subparser, command, callback) + else: + func = callback + + cmd_mapper[command] = subparser + for (args, kwargs) in arguments: + subparser.add_argument(*args, **kwargs) + subparser.set_defaults(func=func) + + +def define_commands_from_module(subparsers, command_module, cmd_mapper): + '''Find all methods beginning with 'do_' in a module, and add them + as commands into a subparsers collection. + ''' + for method_name in (a for a in dir(command_module) if a.startswith('do_')): + # Commands should be hypen-separated instead of underscores. + command = method_name[3:].replace('_', '-') + callback = getattr(command_module, method_name) + define_command(subparsers, command, callback, cmd_mapper) + + +def parse_date(string_data): + """Parses a date-like input string into a timezone aware Python + datetime. + """ + + if not isinstance(string_data, six.string_types): + return string_data + + pattern = r'(\d{4}-\d{2}-\d{2}[T ])?\d{2}:\d{2}:\d{2}(\.\d{6})?Z?' + + def convert_date(matchobj): + formats = ["%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%d %H:%M:%S.%f", + "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", + "%Y-%m-%dT%H:%M:%SZ"] + datestring = matchobj.group(0) + if datestring: + for format in formats: + try: + datetime.strptime(datestring, format) + datestring += "+0000" + parsed = parser.parse(datestring) + converted = parsed.astimezone(dateutil.tz.tzlocal()) + converted = datetime.strftime(converted, format) + return converted + except Exception: + pass + return datestring + + return re.sub(pattern, convert_date, string_data) + + +def _sort_for_list(objs, fields, formatters={}, sortby=0, reversesort=False): + + # Sort only if necessary + if sortby is None: + return objs + + sort_field = fields[sortby] + # figure out sort key function + if sort_field in formatters: + field_formatter = formatters[sort_field] + if wrapping_formatters.WrapperFormatter.is_wrapper_formatter( + field_formatter): + def sort_key(x): + return field_formatter.\ + wrapper_formatter.get_unwrapped_field_value(x) + else: + def sort_key(x): + return field_formatter(x) + else: + def sort_key(x): + return getattr(x, sort_field, '') + + objs.sort(reverse=reversesort, key=sort_key) + + return objs + + +def str_height(text): + if not text: + return 1 + lines = str(text).split("\n") + height = len(lines) + return height + + +def row_height(texts): + if not texts or len(texts) == 0: + return 1 + height = max(str_height(text) for text in texts) + return height + + +class WRPrettyTable(prettytable.PrettyTable): + """A PrettyTable that allows word wrapping of its headers.""" + + def __init__(self, field_names=None, **kwargs): + super(WRPrettyTable, self).__init__(field_names, **kwargs) + + def _stringify_header(self, options): + """ + This overridden version of _stringify_header can wrap its + header data. It leverages the functionality in _stringify_row + to perform this task. + :returns string of header, including border text + """ + bits = [] + if options["border"]: + if options["hrules"] in (ALL, FRAME): + bits.append(self._hrule) + bits.append("\n") + # For tables with no data or field names + if not self._field_names: + if options["vrules"] in (ALL, FRAME): + bits.append(options["vertical_char"]) + bits.append(options["vertical_char"]) + else: + bits.append(" ") + bits.append(" ") + + header_row_data = [] + for field in self._field_names: + if options["fields"] and field not in options["fields"]: + continue + if self._header_style == "cap": + fieldname = field.capitalize() + elif self._header_style == "title": + fieldname = field.title() + elif self._header_style == "upper": + fieldname = field.upper() + elif self._header_style == "lower": + fieldname = field.lower() + else: + fieldname = field + header_row_data.append(fieldname) + + # output actual header row data, word wrap when necessary + bits.append(self._stringify_row(header_row_data, options)) + + if options["border"] and options["hrules"] != NONE: + bits.append("\n") + bits.append(self._hrule) + + return "".join(bits) + + +def prettytable_builder(field_names=None, **kwargs): + return WRPrettyTable(field_names, **kwargs) + + +def wordwrap_header(field, field_label, formatter): + """ + Given a field label (the header text for one column) and the word + wrapping formatter for a column, + this function asks the formatter for the desired column width and then + performs a wordwrap of field_label + + :param field: the field name associated with the field_label + :param field_label: field_label to word wrap + :param formatter: the field formatter + :return: word wrapped field_label + """ + if wrapping_formatters.is_nowrap_set(): + return field_label + + if not wrapping_formatters.WrapperFormatter.is_wrapper_formatter( + formatter): + return field_label + # go to the column's formatter and ask it what the width should be + wrapper_formatter = formatter.wrapper_formatter + actual_width = wrapper_formatter.get_actual_column_char_len( + wrapper_formatter.get_calculated_desired_width()) + # now word wrap based on column width + wrapped_header = textwrap.fill(field_label, actual_width) + return wrapped_header + + +def default_printer(s): + print(s) + + +def pt_builder(field_labels, fields, formatters, paging, + printer=default_printer): + """ + returns an object that 'fronts' a prettyTable object + that can handle paging as well as automatically falling back + to not word wrapping when word wrapping does not cause the + output to fit the terminal width. + """ + + class PT_Builder(object): + + def __init__(self, field_labels, fields, formatters, no_paging): + self.objs_in_pt = [] + self.unwrapped_field_labels = field_labels + self.fields = fields + self.formatters = formatters + self.header_height = 0 + self.terminal_width, self.terminal_height = get_terminal_size() + self.terminal_lines_left = self.terminal_height + self.paging = not no_paging + self.paged_rows_added = 0 + self.pt = None + self.quit = False + + def add_row(self, obj): + if self.quit: + return False + if not self.pt: + self.build_pretty_table() + return self._row_add(obj) + + def __add_row_and_obj(self, row, obj): + self.pt.add_row(row) + self.objs_in_pt.append(obj) + + def _row_add(self, obj): + + row = _build_row_from_object(self.fields, self.formatters, obj) + + if not paging: + self.__add_row_and_obj(row, obj) + return True + + rheight = row_height(row) + if (self.terminal_lines_left - rheight) >= 0 or \ + self.paged_rows_added == 0: + self.__add_row_and_obj(row, obj) + self.terminal_lines_left -= rheight + else: + printer(self.get_string()) + if self.terminal_lines_left > 0: + printer("\n" * (self.terminal_lines_left - 1)) + + s = six.moves.input( + "Press Enter to continue or 'q' to exit...") + if s == 'q': + self.quit = True + return False + self.terminal_lines_left = \ + self.terminal_height - self.header_height + self.build_pretty_table() + self.__add_row_and_obj(row, obj) + self.terminal_lines_left -= rheight + self.paged_rows_added += 1 + + def get_string(self): + if not self.pt: + self.build_pretty_table() + objs = copy.copy(self.objs_in_pt) + self.objs_in_pt = [] + output = self.pt.get_string() + if wrapping_formatters.is_nowrap_set(): + return output + output_width = wrapping_formatters.get_width(output) + if output_width <= self.terminal_width: + return output + # At this point pretty Table (self.pt) does not fit the terminal + # width so let's temporarily turn wrapping off, + # rebuild the pretty Table with the data unwrapped. + orig_no_wrap_settings = \ + wrapping_formatters.set_no_wrap_on_formatters( + True, self.formatters) + self.build_pretty_table() + for o in objs: + self.add_row(o) + wrapping_formatters.unset_no_wrap_on_formatters( + orig_no_wrap_settings) + return self.pt.get_string() + + def build_pretty_table(self): + field_labels = [wordwrap_header(field, field_label, formatter) + for field, field_label, formatter in + zip(self.fields, self.unwrapped_field_labels, [ + formatters.get(f, None) for f in self.fields])] + self.pt = prettytable_builder( + field_labels, caching=False, print_empty=False) + self.pt.align = 'l' + # 2 header border lines + 1 bottom border + 1 prompt + # + header data height + self.header_height = 2 + 1 + 1 + row_height(field_labels) + self.terminal_lines_left = \ + self.terminal_height - self.header_height + return self.pt + + def done(self): + if self.quit: + return + + if not self.paging or ( + self.terminal_lines_left < + self.terminal_height - self.header_height): + printer(self.get_string()) + + return PT_Builder(field_labels, fields, formatters, not paging) + + +def print_long_list(objs, fields, field_labels, + formatters={}, sortby=0, reversesort=False, + no_wrap_fields=[], no_paging=False, + printer=default_printer): + + formatters = wrapping_formatters.as_wrapping_formatters( + objs, fields, field_labels, formatters, + no_wrap_fields=no_wrap_fields) + + objs = _sort_for_list(objs, fields, + formatters=formatters, + sortby=sortby, + reversesort=reversesort) + + pt = pt_builder(field_labels, fields, formatters, not no_paging, + printer=printer) + + for o in objs: + pt.add_row(o) + + pt.done() + + +def print_dict(d, dict_property="Property", wrap=0): + pt = prettytable.PrettyTable([dict_property, 'Value'], + caching=False, print_empty=False) + pt.align = 'l' + for k, v in sorted(d.iteritems()): + v = parse_date(v) + # convert dict to str to check length + if isinstance(v, dict): + v = str(v) + if wrap > 0: + v = textwrap.fill(six.text_type(v), wrap) + # if value has a newline, add in multiple rows + # e.g. fault with stacktrace + if v and isinstance(v, str) and r'\n' in v: + lines = v.strip().split(r'\n') + col1 = k + for line in lines: + pt.add_row([col1, line]) + col1 = '' + else: + pt.add_row([k, v]) + + print(pt.get_string()) + + +def _build_row_from_object(fields, formatters, o): + """ + takes an object o and converts to an array of values + compatible with the input for prettyTable.add_row(row) + """ + row = [] + for field in fields: + if field in formatters: + data = parse_date(getattr(o, field, '')) + setattr(o, field, data) + data = formatters[field](o) + row.append(data) + else: + data = parse_date(getattr(o, field, '')) + row.append(data) + return row + + +def print_list(objs, fields, field_labels, formatters={}, sortby=0, + reversesort=False, no_wrap_fields=[], printer=default_printer): + # print_list() is the same as print_long_list() with paging turned off + return print_long_list( + objs, fields, field_labels, formatters=formatters, sortby=sortby, + reversesort=reversesort, no_wrap_fields=no_wrap_fields, + no_paging=True, printer=printer) + + +def find_resource(manager, name_or_id): + """Helper for the _find_* methods.""" + # first try to get entity as integer id + try: + if isinstance(name_or_id, int) or name_or_id.isdigit(): + return manager.get(int(name_or_id)) + except exc.NotFound: + pass + + # now try to get entity as uuid + try: + uuid.UUID(str(name_or_id)) + return manager.get(name_or_id) + except (ValueError, exc.NotFound): + pass + + # finally try to find entity by name + try: + return manager.find(name=name_or_id) + except exc.NotFound: + msg = "No %s with a name or ID of '%s' exists." % \ + (manager.resource_class.__name__.lower(), name_or_id) + raise exc.CommandError(msg) + + +def string_to_bool(arg): + return arg.strip().lower() in ('t', 'true', 'yes', '1') + + +def objectify(func): + """Mimic an object given a dictionary. + + Given a dictionary, create an object and make sure that each of its + keys are accessible via attributes. + Ignore everything if the given value is not a dictionary. + :param func: A dictionary or another kind of object. + :returns: Either the created object or the given value. + + >>> obj = {'old_key': 'old_value'} + >>> oobj = objectify(obj) + >>> oobj['new_key'] = 'new_value' + >>> print oobj['old_key'], oobj['new_key'], oobj.old_key, oobj.new_key + + >>> @objectify + ... def func(): + ... return {'old_key': 'old_value'} + >>> obj = func() + >>> obj['new_key'] = 'new_value' + >>> print obj['old_key'], obj['new_key'], obj.old_key, obj.new_key + + + """ + + def create_object(value): + if isinstance(value, dict): + # Build a simple generic object. + class Object(dict): + def __setitem__(self, key, val): + setattr(self, key, val) + return super(Object, self).__setitem__(key, val) + + # Create that simple generic object. + ret_obj = Object() + # Assign the attributes given the dictionary keys. + for key, val in value.iteritems(): + ret_obj[key] = val + setattr(ret_obj, key, val) + return ret_obj + else: + return value + + # If func is a function, wrap around and act like a decorator. + if hasattr(func, '__call__'): + @wraps(func) + def wrapper(*args, **kwargs): + """Wrapper function for the decorator. + + :returns: The return value of the decorated function. + + """ + value = func(*args, **kwargs) + return create_object(value) + + return wrapper + + # Else just try to objectify the value given. + else: + return create_object(func) + + +def is_uuid_like(val): + """Returns validation of a value as a UUID. + + For our purposes, a UUID is canonical form string: + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaa + + """ + try: + return str(uuid.UUID(val)) == val + except (TypeError, ValueError, AttributeError): + return False + + +def args_array_to_dict(kwargs, key_to_convert): + values_to_convert = kwargs.get(key_to_convert) + if values_to_convert: + try: + kwargs[key_to_convert] = dict(v.split("=", 1) + for v in values_to_convert) + except ValueError: + raise exc.CommandError('%s must be a list of KEY=VALUE not "%s"' % + (key_to_convert, values_to_convert)) + return kwargs + + +def args_array_to_patch(op, attributes): + patch = [] + for attr in attributes: + # Sanitize + if not attr.startswith('/'): + attr = '/' + attr + + if op in ['add', 'replace']: + try: + path, value = attr.split("=", 1) + patch.append({'op': op, 'path': path, 'value': value}) + except ValueError: + raise exc.CommandError('Attributes must be a list of ' + 'PATH=VALUE not "%s"' % attr) + elif op == "remove": + # For remove only the key is needed + patch.append({'op': op, 'path': attr}) + else: + raise exc.CommandError('Unknown PATCH operation: %s' % op) + return patch + + +def dict_to_patch(values, op='replace'): + patch = [] + for key, value in values.iteritems(): + path = '/' + key + patch.append({'op': op, 'path': path, 'value': value}) + return patch + + +def print_tuple_list(tuples, tuple_labels=[], formatters={}): + pt = prettytable.PrettyTable(['Property', 'Value'], + caching=False, print_empty=False) + pt.align = 'l' + + if not tuple_labels: + for t in tuples: + if len(t) == 2: + f, v = t + v = parse_date(v) + if f in formatters: + v = formatters[f](v) + pt.add_row([f, v]) + else: + for t, l in zip(tuples, tuple_labels): + if len(t) == 2: + f, v = t + v = parse_date(v) + if f in formatters: + v = formatters[f](v) + pt.add_row([l, v]) + + print(pt.get_string()) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/common/wrapping_formatters.py b/python-inventoryclient/inventoryclient/inventoryclient/common/wrapping_formatters.py new file mode 100644 index 00000000..0bbb45a4 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/common/wrapping_formatters.py @@ -0,0 +1,871 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Manages WrapperFormatter objects. + +WrapperFormatter objects can be used for wrapping CLI column celldata in order +for the CLI table (using prettyTable) to fit the terminal screen + +The basic idea is: + + Once celldata is retrieved and ready to display, first iterate through + the celldata and word wrap it so that fits programmer desired column widths. + The WrapperFormatter objects fill this role. + + Once the celldata is formatted to their desired widths, + then it can be passed to the existing prettyTable code base for rendering. + +""" +import copy +import re +import six +import textwrap + +from cli_no_wrap import is_nowrap_set +from cli_no_wrap import set_no_wrap +from prettytable import _get_size + +UUID_MIN_LENGTH = 36 + +# monkey patch (customize) how the textwrap module breaks text into chunks +wordsep_re = re.compile( + r'(\s+|' # any whitespace + r',|' + r'=|' + r'\.|' + r':|' + r'[^\s\w]*\w+[^0-9\W]-(?=\w+[^0-9\W])|' # hyphenated word + r'(?<=[\w\!\"\'\&\.\,\?])-{2,}(?=\w))') # em-dash + +textwrap.TextWrapper.wordsep_re = wordsep_re + + +def get_width(value): + if value is None: + return 0 + + return _get_size(six.text_type(value))[0] # get width from [width,height] + + +def _get_terminal_width(): + from utils import get_terminal_size + result = get_terminal_size()[0] + return result + + +def is_uuid_field(field_name): + """ + :param field_name: + :return: True if field_name looks like a uuid name + """ + if field_name is not None and field_name in ["uuid", "UUID"] or \ + field_name.endswith("uuid"): + return True + return False + + +class WrapperContext(object): + """Context for the wrapper formatters + + Maintains a list of the current WrapperFormatters + being used to format the prettyTable celldata + + Allows wrappers access to its 'sibling' wrappers + contains convenience methods and attributes + for calculating current tableWidth. + """ + + def __init__(self): + self.wrappers = [] + self.wrappers_by_field = {} + self.non_data_chrs_used_by_table = 0 + self.num_columns = 0 + self.terminal_width = -1 + + def set_num_columns(self, num_columns): + self.num_columns = num_columns + self.non_data_chrs_used_by_table = (num_columns * 3) + 1 + + def add_column_formatter(self, field, wrapper): + self.wrappers.append(wrapper) + self.wrappers_by_field[field] = wrapper + + def get_terminal_width(self): + if self.terminal_width == -1: + self.terminal_width = _get_terminal_width() + return self.terminal_width + + def get_table_width(self): + """ + Calculates table width by looping through all + column formatters and summing up their widths + :return: total table width + """ + widths = [w.get_actual_column_char_len( + w.get_calculated_desired_width(), + check_remaining_row_chars=False) + for w in self.wrappers] + chars_used_by_data = sum(widths) + width = self.non_data_chrs_used_by_table + chars_used_by_data + return width + + def is_table_too_wide(self): + """ + :return: True if calculated table width is too wide + for the terminal width + """ + if self.get_terminal_width() < self.get_table_width(): + return True + return False + + +def field_value_function_factory(formatter, field): + """Builds function for getting a field value from table cell celldata + As a side-effect, attaches function as the 'get_field_value' attribute + of the formatter + :param formatter:the formatter to attach return function to + :param field: + :return: function that returns cell celldata + """ + + def field_value_function_builder(data): + if isinstance(data, dict): + formatter.get_field_value = \ + lambda celldata: celldata.get(field, None) + else: + formatter.get_field_value = \ + lambda celldata: getattr(celldata, field) + return formatter.get_field_value(data) + + return field_value_function_builder + + +class WrapperFormatter(object): + """Base (abstract) class definition of wrapping formatters""" + + def __init__(self, ctx, field): + self.ctx = ctx + self.add_blank_line = False + self.no_wrap = False + self.min_width = 0 + self.field = field + self.header_width = 0 + self.actual_column_char_len = -1 + self.textWrapper = None + + if self.field: + self.get_field_value = field_value_function_factory(self, field) + else: + self.get_field_value = lambda data: data + + def get_basic_desired_width(self): + return self.min_width + + def get_calculated_desired_width(self): + basic_desired_width = self.get_basic_desired_width() + if self.header_width > basic_desired_width: + return self.header_width + return basic_desired_width + + def get_sibling_wrappers(self): + """ + :return: a list of your sibling wrappers for the other fields + """ + others = [w for w in self.ctx.wrappers if w != self] + return others + + def get_remaining_row_chars(self): + used = [w.get_actual_column_char_len(w.get_calculated_desired_width(), + check_remaining_row_chars=False) + for w in self.get_sibling_wrappers()] + chrs_used_by_data = sum(used) + remaining_chrs_in_row = \ + (self.ctx.get_terminal_width() - + self.ctx.non_data_chrs_used_by_table) - chrs_used_by_data + return remaining_chrs_in_row + + def set_min_width(self, min_width): + self.min_width = min_width + + def set_actual_column_len(self, actual): + self.actual_column_char_len = actual + + def get_actual_column_char_len(self, desired_char_len, + check_remaining_row_chars=True): + """Utility method to adjust desired width to a width + that can actually be applied based on current table width + and current terminal width + + Will not allow actual width to be less than min_width + min_width is typically length of the column header text + or the longest 'word' in the celldata + + :param desired_char_len: + :param check_remaining_row_chars: + :return: + """ + if self.actual_column_char_len != -1: + return self.actual_column_char_len # already calculated + if desired_char_len < self.min_width: + actual = self.min_width + else: + actual = desired_char_len + if check_remaining_row_chars and actual > self.min_width: + remaining = self.get_remaining_row_chars() + if actual > remaining >= self.min_width: + actual = remaining + if check_remaining_row_chars: + self.set_actual_column_len(actual) + if self.ctx.is_table_too_wide(): + # Table too big can I shrink myself? + if actual > self.min_width: + # shrink column + while actual > self.min_width: + actual -= 1 # TODO(jkung): fix in next sprint + # each column needs to share in + # table shrinking - but this is good + # enough for now - also - why the loop? + self.set_actual_column_len(actual) + + return actual + + def _textwrap_fill(self, s, actual_width): + if not self.textWrapper: + self.textWrapper = textwrap.TextWrapper(actual_width) + else: + self.textWrapper.width = actual_width + return self.textWrapper.fill(s) + + def text_wrap(self, s, width): + """ + performs actual text wrap + :param s: + :param width: in characters + :return: formatted text + """ + if self.no_wrap: + return s + actual_width = self.get_actual_column_char_len(width) + new_s = self._textwrap_fill(s, actual_width) + wrapped = new_s != s + if self.add_blank_line and wrapped: + new_s += "\n".ljust(actual_width) + return new_s + + def format(self, data): + return str(self.get_field_value(data)) + + def get_unwrapped_field_value(self, data): + return self.get_field_value(data) + + def as_function(self): + def foo(data): + return self.format(data) + + foo.WrapperFormatterMarker = True + foo.wrapper_formatter = self + return foo + + @staticmethod + def is_wrapper_formatter(foo): + if not foo: + return False + return getattr(foo, "WrapperFormatterMarker", False) + + +class WrapperLambdaFormatter(WrapperFormatter): + """A wrapper formatter that adapts a function (callable) + to look like a WrapperFormatter + """ + + def __init__(self, ctx, field, format_function): + super(WrapperLambdaFormatter, self).__init__(ctx, field) + self.format_function = format_function + + def format(self, data): + return self.format_function(self.get_field_value(data)) + + +class WrapperFixedWidthFormatter(WrapperLambdaFormatter): + """A wrapper formatter that forces the text to wrap within + a specific width (in chars) + """ + + def __init__(self, ctx, field, width): + super(WrapperFixedWidthFormatter, self).__init__( + ctx, field, + lambda data: self.text_wrap( + str(data), self.get_calculated_desired_width())) + self.width = width + + def get_basic_desired_width(self): + return self.width + + +class WrapperPercentWidthFormatter(WrapperFormatter): + """A wrapper formatter that forces the text to wrap within + a specific percentage width of the current terminal width + """ + + def __init__(self, ctx, field, width_as_decimal): + super(WrapperPercentWidthFormatter, self).__init__(ctx, field) + self.width_as_decimal = width_as_decimal + + def get_basic_desired_width(self): + width = int((self.ctx.get_terminal_width() - + self.ctx.non_data_chrs_used_by_table) * + self.width_as_decimal) + return width + + def format(self, data): + width = self.get_calculated_desired_width() + field_value = self.get_field_value(data) + return self.text_wrap(str(field_value), width) + + +class WrapperWithCustomFormatter(WrapperLambdaFormatter): + """A wrapper formatter that allows the programmer to have a custom + formatter (in the form of a function) that is first applied + and then a wrapper function is applied to the result + + See wrapperFormatterFactory for a better explanation! :-) + """ + + # noinspection PyUnusedLocal + def __init__(self, ctx, field, custom_formatter, wrapper_formatter): + super(WrapperWithCustomFormatter, self).__init__( + ctx, None, + lambda data: wrapper_formatter.format(custom_formatter(data))) + self.wrapper_formatter = wrapper_formatter + self.custom_formatter = custom_formatter + + def get_unwrapped_field_value(self, data): + return self.custom_formatter(data) + + def __setattr__(self, name, value): + # + # Some attributes set onto this class need + # to be pushed down to the 'inner' wrapper_formatter + # + super(WrapperWithCustomFormatter, self).__setattr__(name, value) + if hasattr(self, "wrapper_formatter"): + if name == "no_wrap": + self.wrapper_formatter.no_wrap = value + if name == "add_blank_line": + self.wrapper_formatter.add_blank_line = value + if name == "header_width": + self.wrapper_formatter.header_width = value + + def set_min_width(self, min_width): + super(WrapperWithCustomFormatter, self).set_min_width(min_width) + self.wrapper_formatter.set_min_width(min_width) + + def set_actual_column_len(self, actual): + super(WrapperWithCustomFormatter, self).set_actual_column_len(actual) + self.wrapper_formatter.set_actual_column_len(actual) + + def get_basic_desired_width(self): + return self.wrapper_formatter.get_basic_desired_width() + + +def wrapper_formatter_factory(ctx, field, formatter): + """ + This function is a factory for building WrapperFormatter objects. + + The function needs to be called for each celldata column (field) + that will be displayed in the prettyTable. + + The function looks at the formatter parameter and based on its type, + determines what WrapperFormatter to construct per field (column). + + ex: + + formatter = 15 - type = int : Builds a WrapperFixedWidthFormatter that + will wrap at 15 chars + + formatter = .25 - type = int : Builds a WrapperPercentWidthFormatter that + will wrap at 25% terminal width + + formatter = type = callable : Builds a WrapperLambdaFormatter that + will call some arbitrary function + + formatter = type = dict : Builds a WrapperWithCustomFormatter that + will call some arbitrary function to format + and then apply a wrapping formatter + to the result + + ex: this dict + {"formatter" : captializeFunction,, + "wrapperFormatter": .12} + will apply the captializeFunction + to the column celldata and then + wordwrap at 12 % of terminal width + + :param ctx: the WrapperContext that the built WrapperFormatter will use + :param field: name of field (column_ that the WrapperFormatter + will execute on + :param formatter: specifies type and input for WrapperFormatter + that will be built + :return: WrapperFormatter + + """ + if isinstance(formatter, WrapperFormatter): + return formatter + if callable(formatter): + return WrapperLambdaFormatter(ctx, field, formatter) + if isinstance(formatter, int): + return WrapperFixedWidthFormatter(ctx, field, formatter) + if isinstance(formatter, float): + return WrapperPercentWidthFormatter(ctx, field, formatter) + if isinstance(formatter, dict): + if "wrapperFormatter" in formatter: + embedded_wrapper_formatter = wrapper_formatter_factory( + ctx, None, formatter["wrapperFormatter"]) + elif "hard_width" in formatter: + embedded_wrapper_formatter = WrapperFixedWidthFormatter( + ctx, field, formatter["hard_width"]) + embedded_wrapper_formatter.min_width = formatter["hard_width"] + else: + embedded_wrapper_formatter = WrapperFormatter( + ctx, None) # effectively a NOOP width formatter + if "formatter" not in formatter: + return embedded_wrapper_formatter + custom_formatter = formatter["formatter"] + wrapper = WrapperWithCustomFormatter( + ctx, field, custom_formatter, embedded_wrapper_formatter) + return wrapper + + raise Exception("Formatter Error! Unrecognized formatter {} " + "for field {}".format(formatter, field)) + + +def build_column_stats_for_best_guess_formatting(objs, fields, field_labels, + custom_formatters={}): + class ColumnStats(object): + def __init__(self, field, field_label, custom_formatter=None): + self.field = field + self.field_label = field_label + self.average_width = 0 + self.min_width = get_width(field_label) if field_label else 0 + self.max_width = get_width(field_label) if field_label else 0 + self.total_width = 0 + self.count = 0 + self.average_percent = 0 + self.max_percent = 0 + self.isUUID = is_uuid_field(field) + if custom_formatter: + self.get_field_value = custom_formatter + else: + self.get_field_value = \ + field_value_function_factory(self, field) + + def add_value(self, value): + if self.isUUID: + return + self.count += 1 + value_width = get_width(value) + self.total_width = self.total_width + value_width + if value_width < self.min_width: + self.min_width = value_width + if value_width > self.max_width: + self.max_width = value_width + if self.count > 0: + self.average_width = float( + self.total_width) / float(self.count) + + def set_max_percent(self, max_total_width): + if max_total_width > 0: + self.max_percent = float( + self.max_width) / float(max_total_width) + + def set_avg_percent(self, avg_total_width): + if avg_total_width > 0: + self.average_percent = float( + self.average_width) / float(avg_total_width) + + def __str__(self): + return str([self.field, + self.average_width, + self.min_width, + self.max_width, + self.total_width, + self.count, + self.average_percent, + self.max_percent, + self.isUUID]) + + def __repr__(self): + return str([self.field, + self.average_width, + self.min_width, + self.max_width, + self.total_width, + self.count, + self.average_percent, + self.max_percent, + self.isUUID]) + + if objs is None or len(objs) == 0: + return {"stats": {}, + "total_max_width": 0, + "total_avg_width": 0} + + stats = {} + for i in range(0, len(fields)): + stats[fields[i]] = ColumnStats( + fields[i], field_labels[i], custom_formatters.get(fields[i])) + + for obj in objs: + for field in fields: + column_stat = stats[field] + column_stat.add_value(column_stat.get_field_value(obj)) + + total_max_width = sum([s.max_width for s in stats.values()]) + total_avg_width = sum([s.average_width for s in stats.values()]) + return {"stats": stats, + "total_max_width": total_max_width, + "total_avg_width": total_avg_width} + + +def build_best_guess_formatters_using_average_widths( + objs, fields, field_labels, + custom_formatters={}, no_wrap_fields=[]): + + column_info = build_column_stats_for_best_guess_formatting( + objs, fields, field_labels, custom_formatters) + format_spec = {} + total_avg_width = float(column_info["total_avg_width"]) + if total_avg_width <= 0: + return format_spec + for f in [ff for ff in fields if ff not in no_wrap_fields]: + format_spec[f] = float( + column_info["stats"][f].average_width) / total_avg_width + custom_formatter = custom_formatters.get(f, None) + if custom_formatter: + format_spec[f] = {"formatter": custom_formatter, + "wrapperFormatter": format_spec[f]} + + # Handle no wrap fields by building formatters that will not wrap + for f in [ff for ff in fields if ff in no_wrap_fields]: + format_spec[f] = {"hard_width": column_info["stats"][f].max_width} + custom_formatter = custom_formatters.get(f, None) + if custom_formatter: + format_spec[f] = {"formatter": custom_formatter, + "wrapperFormatter": format_spec[f]} + return format_spec + + +def build_best_guess_formatters_using_max_widths(objs, fields, field_labels, + custom_formatters={}, + no_wrap_fields=[]): + column_info = build_column_stats_for_best_guess_formatting( + objs, fields, field_labels, custom_formatters) + format_spec = {} + for f in [ff for ff in fields if ff not in no_wrap_fields]: + format_spec[f] = \ + float(column_info["stats"][f].max_width) / float(column_info["total_max_width"]) # noqa + custom_formatter = custom_formatters.get(f, None) + if custom_formatter: + format_spec[f] = {"formatter": custom_formatter, + "wrapperFormatter": format_spec[f]} + + # Handle no wrap fields by building formatters that will not wrap + for f in [ff for ff in fields if ff in no_wrap_fields]: + format_spec[f] = {"hard_width": column_info["stats"][f].max_width} + custom_formatter = custom_formatters.get(f, None) + if custom_formatter: + format_spec[f] = {"formatter": custom_formatter, + "wrapperFormatter": format_spec[f]} + + return format_spec + + +def needs_wrapping_formatters(formatters, no_wrap=None): + no_wrap = is_nowrap_set(no_wrap) + if no_wrap: + return False + + # handle easy case: + if not formatters: + return True + + # If we have at least one wrapping formatter, + # then we assume we don't need to wrap + for f in formatters.values(): + if WrapperFormatter.is_wrapper_formatter(f): + return False + + # looks like we need wrapping + return True + + +def as_wrapping_formatters(objs, fields, field_labels, formatters, + no_wrap=None, no_wrap_fields=[]): + """This function is the entry point for building the "best guess" + word wrapping formatters. A best guess formatter guesses what the best + columns widths should be for the table celldata. It does this by + collecting various stats on the celldata (min, max average width of + column celldata) and from this celldata decides the desired widths + and the minimum widths. + + Given a list of formatters and the list of objects (objs), this + function first determines if we need to augment the passed formatters + with word wrapping formatters. + If the no_wrap parameter or global no_wrap flag is set, + then we do not build wrapping formatters. If any of the formatters + within formatters is a word wrapping formatter, + then it is assumed no more wrapping is required. + + :param objs: + :param fields: + :param field_labels: + :param formatters: + :param no_wrap: + :param no_wrap_fields: + :return: When no wrapping is required, the formatters parameter is returned + -- effectively a NOOP in this case + + When wrapping is required, best-guess word wrapping formatters + are returned with original parameter formatters embedded in the + word wrapping formatters + """ + no_wrap = is_nowrap_set(no_wrap) + + if not needs_wrapping_formatters(formatters, no_wrap): + return formatters + + format_spec = build_best_guess_formatters_using_average_widths( + objs, fields, field_labels, formatters, no_wrap_fields) + + formatters = build_wrapping_formatters( + objs, fields, field_labels, format_spec) + + return formatters + + +def build_wrapping_formatters(objs, fields, field_labels, format_spec, + add_blank_line=True, + no_wrap=None, + use_max=False): + """ + A convenience function for building all wrapper formatters that + will be used to format a CLI's output when its rendered + in a prettyTable object. + + It iterates through the keys of format_spec and + calls wrapperFormatterFactory to build + wrapperFormatter objects for each column. + + Its best to show by example parameters: + + field_labels = ['UUID', 'Time Stamp', 'State', 'Event Log ID', + 'Reason Text', 'Entity Instance ID', 'Severity'] + fields = ['uuid', 'timestamp', 'state', 'event_log_id', 'reason_text', + 'entity_instance_id', 'severity'] + format_spec = { + "uuid" : .10, + # float = so display as 10% of terminal width + "timestamp" : .08, + "state" : .08, + "event_log_id" : .07, + "reason_text" : .42, + "entity_instance_id" : .13, + "severity" : + {"formatter" : captializeFunction, + "wrapperFormatter": .12} + } + + :param objs: the actual celldata that will get word wrapped + :param fields: fields (attributes of the celldata) that will be + displayed in the table + :param field_labels: column (field headers) + :param format_spec: dict specify formatter for each column (field) + :param add_blank_line: default True, when tru adds blank line to column + if it wraps, aids readability + :param no_wrap: default False, when True turns wrapping off but does + not suppress other custom formatters + :param use_max + :return: wrapping formatters as functions + """ + + no_wrap = set_no_wrap(no_wrap) + + if objs is None or len(objs) == 0: + return {} + + biggest_word_pattern = re.compile("[\.:,;\!\?\\ =-\_]") + + def get_biggest_word(s): + return max(biggest_word_pattern.split(s), key=len) + + wrapping_formatters_as_functions = {} + + if len(fields) != len(field_labels): + raise Exception("Error in buildWrappingFormatters: " + "len(fields) = {}, len(field_labels) = {}," + " they must be the same length!".format( + len(fields), len(field_labels))) + field_to_label = {} + + for i in range(0, len(fields)): + field_to_label[fields[i]] = field_labels[i] + + ctx = WrapperContext() + ctx.set_num_columns(len(fields)) + + if not format_spec: + if use_max: + format_spec = build_best_guess_formatters_using_max_widths( + objs, fields, field_labels) + else: + format_spec = build_best_guess_formatters_using_average_widths( + objs, fields, field_labels) + + for k in format_spec.keys(): + if k not in fields: + raise Exception("Error in buildWrappingFormatters: format_spec " + "specifies a field {} that is not specified " + "in fields : {}".format(k, fields)) + + format_spec_for_k = copy.deepcopy(format_spec[k]) + if callable(format_spec_for_k): + format_spec_for_k = {"formatter": format_spec_for_k} + wrapper_formatter = wrapper_formatter_factory( + ctx, k, format_spec_for_k) + if wrapper_formatter.min_width <= 0: + # need to specify min-width so that + # column is not unnecessarily squashed + if is_uuid_field(k): # special case + wrapper_formatter.set_min_width(UUID_MIN_LENGTH) + else: + # column width cannot be smaller than the widest word + column_data = [ + str(wrapper_formatter.get_unwrapped_field_value(data)) + for data in objs] + widest_word_in_column = max( + [get_biggest_word(d) + " " + for d in column_data + [field_to_label[k]]], key=len) + wrapper_formatter.set_min_width(len(widest_word_in_column)) + wrapper_formatter.header_width = get_width(field_to_label[k]) + + wrapper_formatter.add_blank_line = add_blank_line + wrapper_formatter.no_wrap = no_wrap + wrapping_formatters_as_functions[k] = wrapper_formatter.as_function() + ctx.add_column_formatter(k, wrapper_formatter) + + return wrapping_formatters_as_functions + + +def set_no_wrap_on_formatters(no_wrap, formatters): + """ + Purpose of this function is to temporarily force + the no_wrap setting for the formatters parameter. + returns orig_no_wrap_settings defined for each formatter + Use unset_no_wrap_on_formatters(orig_no_wrap_settings) to undo what + this function does + """ + # handle easy case: + if not formatters: + return {} + + formatter_no_wrap_settings = {} + + global_orig_no_wrap = is_nowrap_set() + set_no_wrap(no_wrap) + + for k, f in formatters.iteritems(): + if WrapperFormatter.is_wrapper_formatter(f): + formatter_no_wrap_settings[k] = ( + f.wrapper_formatter.no_wrap, f.wrapper_formatter) + f.wrapper_formatter.no_wrap = no_wrap + + return {"global_orig_no_wrap": global_orig_no_wrap, + "formatter_no_wrap_settings": formatter_no_wrap_settings} + + +def unset_no_wrap_on_formatters(orig_no_wrap_settings): + """ + It only makes sense to call this function with the return value + from the last call to set_no_wrap_on_formatters(no_wrap, formatters). + It effectively undoes what set_no_wrap_on_formatters() does + """ + if not orig_no_wrap_settings: + return {} + + global_orig_no_wrap = orig_no_wrap_settings["global_orig_no_wrap"] + formatter_no_wrap_settings = \ + orig_no_wrap_settings["formatter_no_wrap_settings"] + + formatters = {} + + for k, v in formatter_no_wrap_settings.iteritems(): + formatters[k] = v[1] + formatters[k].no_wrap = v[0] + + set_no_wrap(global_orig_no_wrap) + + return formatters + + +def _simpleTestHarness(no_wrap): + + import utils + + def testFormatter(event): + return "*{}".format(event["state"]) + + def buildFormatter(field, width): + def f(dict): + if field == 'number': + return dict[field] + return "{}".format(dict[field]).replace("_", " ") + return {"formatter": f, "wrapperFormatter": width} + + set_no_wrap(no_wrap) + + field_labels = ['Time Stamp', 'State', 'Event Log ID', 'Reason Text', + 'Entity Instance ID', 'Severity', 'Number'] + fields = ['timestamp', 'state', 'event_log_id', 'reason_text', + 'entity_instance_id', 'severity', 'number'] + + formatterSpecX = {"timestamp": 10, + "state": 8, + "event_log_id": 70, + "reason_text": 30, + "entity_instance_id": 30, + "severity": 12, + "number": 4} + + formatterSpec = {} + for f in fields: + formatterSpec[f] = buildFormatter(f, formatterSpecX[f]) + + logs = [] + for i in range(0, 30): + log = {} + for f in fields: + if f == 'number': + log[f] = i + else: + log[f] = "{}{}".format(f, i) + logs.append(utils.objectify(log)) + + formatterSpec = formatterSpecX + + formatters = build_wrapping_formatters( + logs, fields, field_labels, formatterSpec) + + utils.print_list( + logs, fields, field_labels, formatters=formatters, sortby=6, + reversesort=True, no_wrap_fields=['entity_instance_id']) + + print("nowrap = {}".format(is_nowrap_set())) + + +if __name__ == "__main__": + _simpleTestHarness(True) + _simpleTestHarness(False) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/exc.py b/python-inventoryclient/inventoryclient/inventoryclient/exc.py new file mode 100644 index 00000000..5c550b8e --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/exc.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +class BaseException(Exception): + """An error occurred.""" + def __init__(self, message=None): + self.message = message + + def __str__(self): + return str(self.message) or self.__class__.__doc__ + + +class AuthSystem(BaseException): + """Could not obtain token and endpoint using provided credentials.""" + pass + + +class CommandError(BaseException): + """Invalid usage of CLI.""" + + +class InvalidEndpoint(BaseException): + """The provided endpoint is invalid.""" + + +class CommunicationError(BaseException): + """Unable to communicate with server.""" + + +class EndpointException(BaseException): + pass + + +class ClientException(Exception): + """DEPRECATED""" + + +class InvalidAttribute(ClientException): + pass + + +class InvalidAttributeValue(ClientException): + pass + + +class HTTPException(Exception): + """Base exception for all HTTP-derived exceptions.""" + code = 'N/A' + + def __init__(self, details=None): + self.details = details + + def __str__(self): + return str(self.details) or "%s (HTTP %s)" % (self.__class__.__name__, + self.code) + + +class HTTPMultipleChoices(HTTPException): + code = 300 + + def __str__(self): + self.details = "Requested version of INVENTORY API is not available." + return "%s (HTTP %s) %s" % (self.__class__.__name__, self.code, + self.details) + + +class Unauthorized(HTTPException): + code = 401 + + +class HTTPUnauthorized(Unauthorized): + pass + + +class NotFound(HTTPException): + """DEPRECATED.""" + code = 404 + + +class HTTPNotFound(NotFound): + pass + + +class HTTPMethodNotAllowed(HTTPException): + code = 405 + + +class HTTPInternalServerError(HTTPException): + code = 500 + + +class HTTPNotImplemented(HTTPException): + code = 501 + + +class HTTPBadGateway(HTTPException): + code = 502 diff --git a/python-inventoryclient/inventoryclient/inventoryclient/shell.py b/python-inventoryclient/inventoryclient/inventoryclient/shell.py new file mode 100644 index 00000000..f3e040a6 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/shell.py @@ -0,0 +1,326 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +Command-line interface for Inventory +""" + +import argparse +import httplib2 +import inventoryclient +from inventoryclient import client +from inventoryclient.common import utils +from inventoryclient import exc +import logging +from oslo_utils import importutils +import sys + + +class InventoryShell(object): + + def get_base_parser(self): + parser = argparse.ArgumentParser( + prog='inventory', + description=__doc__.strip(), + epilog='See "inventory help COMMAND" ' + 'for help on a specific command.', + add_help=False, + formatter_class=HelpFormatter, + ) + + # Global arguments + parser.add_argument('-h', '--help', + action='store_true', + help=argparse.SUPPRESS, + ) + + parser.add_argument('--version', + action='version', + version=inventoryclient.__version__) + + parser.add_argument('--debug', + default=bool(utils.env('INVENTORYCLIENT_DEBUG')), + action='store_true', + help='Defaults to env[INVENTORYCLIENT_DEBUG]') + + parser.add_argument('-v', '--verbose', + default=False, action="store_true", + help="Print more verbose output") + + parser.add_argument('--timeout', + default=600, + help='Number of seconds to wait for a response') + + parser.add_argument('--os-username', + default=utils.env('OS_USERNAME'), + help='Defaults to env[OS_USERNAME]') + + parser.add_argument('--os_username', + help=argparse.SUPPRESS) + + parser.add_argument('--os-password', + default=utils.env('OS_PASSWORD'), + help='Defaults to env[OS_PASSWORD]') + + parser.add_argument('--os_password', + help=argparse.SUPPRESS) + + parser.add_argument('--os-tenant-id', + default=utils.env('OS_TENANT_ID'), + help='Defaults to env[OS_TENANT_ID]') + + parser.add_argument('--os_tenant_id', + help=argparse.SUPPRESS) + + parser.add_argument('--os-tenant-name', + default=utils.env('OS_TENANT_NAME'), + help='Defaults to env[OS_TENANT_NAME]') + + parser.add_argument('--os_tenant_name', + help=argparse.SUPPRESS) + + parser.add_argument('--os-auth-url', + default=utils.env('OS_AUTH_URL'), + help='Defaults to env[OS_AUTH_URL]') + + parser.add_argument('--os_auth_url', + help=argparse.SUPPRESS) + + parser.add_argument('--os-region-name', + default=utils.env('OS_REGION_NAME'), + help='Defaults to env[OS_REGION_NAME]') + + parser.add_argument('--os_region_name', + help=argparse.SUPPRESS) + + parser.add_argument('--os-auth-token', + default=utils.env('OS_AUTH_TOKEN'), + help='Defaults to env[OS_AUTH_TOKEN]') + + parser.add_argument('--os_auth_token', + help=argparse.SUPPRESS) + + parser.add_argument('--inventory-url', + default=utils.env('INVENTORY_URL'), + help='Defaults to env[INVENTORY_URL]') + + parser.add_argument('--inventory_url', + help=argparse.SUPPRESS) + + parser.add_argument('--inventory-api-version', + default=utils.env( + 'INVENTORY_API_VERSION', default='1'), + help='Defaults to env[INVENTORY_API_VERSION] ' + 'or 1') + + parser.add_argument('--inventory_api_version', + help=argparse.SUPPRESS) + + parser.add_argument('--os-service-type', + default=utils.env('OS_SERVICE_TYPE', + default=client.SERVICE_TYPE), + help='Defaults to env[OS_SERVICE_TYPE]') + + parser.add_argument('--os_service_type', + help=argparse.SUPPRESS) + + parser.add_argument('--os-endpoint-type', + default=utils.env('OS_ENDPOINT_TYPE'), + help='Defaults to env[OS_ENDPOINT_TYPE]') + + parser.add_argument('--os_endpoint_type', + help=argparse.SUPPRESS) + + parser.add_argument('--os-user-domain-id', + default=utils.env('OS_USER_DOMAIN_ID'), + help='Defaults to env[OS_USER_DOMAIN_ID].') + + parser.add_argument('--os-user-domain-name', + default=utils.env('OS_USER_DOMAIN_NAME'), + help='Defaults to env[OS_USER_DOMAIN_NAME].') + + parser.add_argument('--os-project-id', + default=utils.env('OS_PROJECT_ID'), + help='Another way to specify tenant ID. ' + 'This option is mutually exclusive with ' + ' --os-tenant-id. ' + 'Defaults to env[OS_PROJECT_ID].') + + parser.add_argument('--os-project-name', + default=utils.env('OS_PROJECT_NAME'), + help='Another way to specify tenant name. ' + 'This option is mutually exclusive with ' + ' --os-tenant-name. ' + 'Defaults to env[OS_PROJECT_NAME].') + + parser.add_argument('--os-project-domain-id', + default=utils.env('OS_PROJECT_DOMAIN_ID'), + help='Defaults to env[OS_PROJECT_DOMAIN_ID].') + + parser.add_argument('--os-project-domain-name', + default=utils.env('OS_PROJECT_DOMAIN_NAME'), + help='Defaults to env[OS_PROJECT_DOMAIN_NAME].') + + return parser + + def get_subcommand_parser(self, version): + parser = self.get_base_parser() + + self.subcommands = {} + subparsers = parser.add_subparsers(metavar='') + submodule = importutils.import_versioned_module('inventoryclient', + version, 'shell') + submodule.enhance_parser(parser, subparsers, self.subcommands) + utils.define_commands_from_module(subparsers, self, self.subcommands) + self._add_bash_completion_subparser(subparsers) + return parser + + def _add_bash_completion_subparser(self, subparsers): + subparser = subparsers.add_parser( + 'bash_completion', + add_help=False, + formatter_class=HelpFormatter + ) + self.subcommands['bash_completion'] = subparser + subparser.set_defaults(func=self.do_bash_completion) + + def _setup_debugging(self, debug): + if debug: + logging.basicConfig( + format="%(levelname)s (%(module)s:%(lineno)d) %(message)s", + level=logging.DEBUG) + + httplib2.debuglevel = 1 + else: + logging.basicConfig(format="%(levelname)s %(message)s", + level=logging.CRITICAL) + + def main(self, argv): + # Parse args once to find version + parser = self.get_base_parser() + (options, args) = parser.parse_known_args(argv) + self._setup_debugging(options.debug) + + # build available subcommands based on version + api_version = options.inventory_api_version + subcommand_parser = self.get_subcommand_parser(api_version) + self.parser = subcommand_parser + + # Handle top-level --help/-h before attempting to parse + # a command off the command line + if options.help or not argv: + self.do_help(options) + return 0 + + # Parse args again and call whatever callback was selected + args = subcommand_parser.parse_args(argv) + + # Short-circuit and deal with help command right away. + if args.func == self.do_help: + self.do_help(args) + return 0 + elif args.func == self.do_bash_completion: + self.do_bash_completion(args) + return 0 + + if not (args.os_auth_token and args.inventory_url): + if not args.os_username: + raise exc.CommandError("You must provide a username via " + "either --os-username or via " + "env[OS_USERNAME]") + + if not args.os_password: + raise exc.CommandError("You must provide a password via " + "either --os-password or via " + "env[OS_PASSWORD]") + + if not (args.os_project_id or args.os_project_name): + raise exc.CommandError("You must provide a project name via " + "either --os-project-name or via " + "env[OS_PROJECT_NAME]") + + if not args.os_auth_url: + raise exc.CommandError("You must provide an auth url via " + "either --os-auth-url or via " + "env[OS_AUTH_URL]") + + if not args.os_region_name: + raise exc.CommandError("You must provide an region name via " + "either --os-region-name or via " + "env[OS_REGION_NAME]") + + client_args = ( + 'os_auth_token', 'inventory_url', 'os_username', 'os_password', + 'os_auth_url', 'os_project_id', 'os_project_name', 'os_tenant_id', + 'os_tenant_name', 'os_region_name', 'os_user_domain_id', + 'os_user_domain_name', 'os_project_domain_id', + 'os_project_domain_name', 'os_service_type', 'os_endpoint_type', + 'timeout' + ) + kwargs = {} + for key in client_args: + client_key = key.replace("os_", "", 1) + kwargs[client_key] = getattr(args, key) + + client = inventoryclient.client.get_client(api_version, **kwargs) + + try: + args.func(client, args) + except exc.Unauthorized: + raise exc.CommandError("Invalid Identity credentials.") + + def do_bash_completion(self, args): + """Prints all of the commands and options to stdout. + """ + commands = set() + options = set() + for sc_str, sc in self.subcommands.items(): + commands.add(sc_str) + for option in list(sc._optionals._option_string_actions): + options.add(option) + + commands.remove('bash_completion') + print(' '.join(commands | options)) + + @utils.arg('command', metavar='', nargs='?', + help='Display help for ') + def do_help(self, args): + """Display help about this program or one of its subcommands.""" + if getattr(args, 'command', None): + if args.command in self.subcommands: + self.subcommands[args.command].print_help() + else: + raise exc.CommandError("'%s' is not a valid subcommand" % + args.command) + else: + self.parser.print_help() + + +class HelpFormatter(argparse.HelpFormatter): + def start_section(self, heading): + # Title-case the headings + heading = '%s%s' % (heading[0].upper(), heading[1:]) + super(HelpFormatter, self).start_section(heading) + + +def main(): + try: + InventoryShell().main(sys.argv[1:]) + + except KeyboardInterrupt as e: + print >> sys.stderr, ('caught: %r, aborting' % (e)) + sys.exit(0) + + except IOError as e: + sys.exit(0) + + except Exception as e: + print >> sys.stderr, e + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/python-inventoryclient/inventoryclient/inventoryclient/tests/__init__.py b/python-inventoryclient/inventoryclient/inventoryclient/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-inventoryclient/inventoryclient/inventoryclient/tests/test_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/tests/test_shell.py new file mode 100644 index 00000000..c63073ac --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/tests/test_shell.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +import cStringIO +import httplib2 +import re +import sys + +import fixtures +from inventoryclient import exc +from inventoryclient import shell as inventoryclient_shell +from inventoryclient.tests import utils +from testtools import matchers + +FAKE_ENV = {'OS_USERNAME': 'username', + 'OS_PASSWORD': 'password', + 'OS_TENANT_NAME': 'tenant_name', + 'OS_AUTH_URL': 'http://no.where'} + + +class ShellTest(utils.BaseTestCase): + re_options = re.DOTALL | re.MULTILINE + + # Patch os.environ to avoid required auth info. + def make_env(self, exclude=None): + env = dict((k, v) for k, v in FAKE_ENV.items() if k != exclude) + self.useFixture(fixtures.MonkeyPatch('os.environ', env)) + + def setUp(self): + super(ShellTest, self).setUp() + + def shell(self, argstr): + orig = sys.stdout + try: + sys.stdout = cStringIO.StringIO() + _shell = inventoryclient_shell.InventoryShell() + _shell.main(argstr.split()) + except SystemExit: + exc_type, exc_value, exc_traceback = sys.exc_info() + self.assertEqual(exc_value.code, 0) + finally: + out = sys.stdout.getvalue() + sys.stdout.close() + sys.stdout = orig + + return out + + def test_help_unknown_command(self): + self.assertRaises(exc.CommandError, self.shell, 'help foofoo') + + def test_debug(self): + httplib2.debuglevel = 0 + self.shell('--debug help') + self.assertEqual(httplib2.debuglevel, 1) + + def test_help(self): + required = [ + '.*?^usage: inventory', + '.*?^See "inventory help COMMAND" ' + 'for help on a specific command', + ] + for argstr in ['--help', 'help']: + help_text = self.shell(argstr) + for r in required: + self.assertThat(help_text, + matchers.MatchesRegex(r, + self.re_options)) + + def test_help_on_subcommand(self): + required = [ + '.*?^usage: inventory host-show ' + '', + ".*?^Show host attributes.", + '', + ".*?^Positional arguments:", + ".*?^ Name or ID of host", + ] + argstrings = [ + 'help host-show', + ] + for argstr in argstrings: + help_text = self.shell(argstr) + for r in required: + self.assertThat(help_text, + matchers.MatchesRegex(r, self.re_options)) + + def test_auth_param(self): + self.make_env(exclude='OS_USERNAME') + self.test_help() diff --git a/python-inventoryclient/inventoryclient/inventoryclient/tests/test_utils.py b/python-inventoryclient/inventoryclient/inventoryclient/tests/test_utils.py new file mode 100644 index 00000000..8e17c117 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/tests/test_utils.py @@ -0,0 +1,92 @@ +# Copyright 2013 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import cStringIO +import sys + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.tests import utils as test_utils + + +class UtilsTest(test_utils.BaseTestCase): + + def test_prettytable(self): + class Struct(object): + def __init__(self, **entries): + self.__dict__.update(entries) + + # test that the prettytable output is wellformatted (left-aligned) + saved_stdout = sys.stdout + try: + sys.stdout = output_dict = cStringIO.StringIO() + utils.print_dict({'K': 'k', 'Key': 'Value'}) + + finally: + sys.stdout = saved_stdout + + self.assertEqual(output_dict.getvalue(), '''\ ++----------+-------+ +| Property | Value | ++----------+-------+ +| K | k | +| Key | Value | ++----------+-------+ +''') + + def test_args_array_to_dict(self): + my_args = { + 'matching_metadata': ['metadata.key=metadata_value'], + 'other': 'value' + } + cleaned_dict = utils.args_array_to_dict(my_args, + "matching_metadata") + self.assertEqual(cleaned_dict, { + 'matching_metadata': {'metadata.key': 'metadata_value'}, + 'other': 'value' + }) + + def test_args_array_to_patch(self): + my_args = { + 'attributes': ['foo=bar', '/extra/bar=baz'], + 'op': 'add', + } + patch = utils.args_array_to_patch(my_args['op'], + my_args['attributes']) + self.assertEqual(patch, [{'op': 'add', + 'value': 'bar', + 'path': '/foo'}, + {'op': 'add', + 'value': 'baz', + 'path': '/extra/bar'}]) + + def test_args_array_to_patch_format_error(self): + my_args = { + 'attributes': ['foobar'], + 'op': 'add', + } + self.assertRaises(exc.CommandError, utils.args_array_to_patch, + my_args['op'], my_args['attributes']) + + def test_args_array_to_patch_remove(self): + my_args = { + 'attributes': ['/foo', 'extra/bar'], + 'op': 'remove', + } + patch = utils.args_array_to_patch(my_args['op'], + my_args['attributes']) + self.assertEqual(patch, [{'op': 'remove', 'path': '/foo'}, + {'op': 'remove', 'path': '/extra/bar'}]) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/tests/utils.py b/python-inventoryclient/inventoryclient/inventoryclient/tests/utils.py new file mode 100644 index 00000000..cff9f7ba --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/tests/utils.py @@ -0,0 +1,69 @@ +# Copyright 2012 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import fixtures +import mox +import StringIO +import testtools + +from inventoryclient.common import http + + +class BaseTestCase(testtools.TestCase): + + def setUp(self): + super(BaseTestCase, self).setUp() + self.m = mox.Mox() + self.addCleanup(self.m.UnsetStubs) + self.useFixture(fixtures.FakeLogger()) + + +class FakeAPI(object): + def __init__(self, fixtures): + self.fixtures = fixtures + self.calls = [] + + def _request(self, method, url, headers=None, body=None): + call = (method, url, headers or {}, body) + self.calls.append(call) + return self.fixtures[url][method] + + def raw_request(self, *args, **kwargs): + fixture = self._request(*args, **kwargs) + body_iter = http.ResponseBodyIterator(StringIO.StringIO(fixture[1])) + return FakeResponse(fixture[0]), body_iter + + def json_request(self, *args, **kwargs): + fixture = self._request(*args, **kwargs) + return FakeResponse(fixture[0]), fixture[1] + + +class FakeResponse(object): + def __init__(self, headers, body=None, version=None): + """:param headers: dict representing HTTP response headers + :param body: file-like object + """ + self.headers = headers + self.body = body + + def getheaders(self): + return copy.deepcopy(self.headers).items() + + def getheader(self, key, default): + return self.headers.get(key, default) + + def read(self, amt): + return self.body.read(amt) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/__init__.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/client.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/client.py new file mode 100644 index 00000000..9cd7db20 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/client.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from inventoryclient.common import exceptions as exc +from inventoryclient.common import http +from inventoryclient.common.http import DEFAULT_VERSION +from inventoryclient.common.i18n import _ +from inventoryclient.v1 import cpu +from inventoryclient.v1 import ethernetport +from inventoryclient.v1 import host +from inventoryclient.v1 import lldp_agent +from inventoryclient.v1 import lldp_neighbour +from inventoryclient.v1 import memory +from inventoryclient.v1 import node +from inventoryclient.v1 import pci_device +from inventoryclient.v1 import port + + +class Client(object): + """Client for the INVENTORY v1 API. + + :param string endpoint: A user-supplied endpoint URL for the inventory + service. + :param function token: Provides token for authentication. + :param integer timeout: Allows customization of the timeout for client + http requests. (optional) + """ + + def __init__(self, endpoint=None, session=None, **kwargs): + """Initialize a new client for the INVENTORY v1 API.""" + if not session: + if kwargs.get('os_inventory_api_version'): + kwargs['api_version_select_state'] = "user" + else: + if not endpoint: + raise exc.EndpointException( + _("Must provide 'endpoint' " + "if os_inventory_api_version isn't specified")) + + # If the user didn't specify a version, use a default version + kwargs['api_version_select_state'] = "default" + kwargs['os_inventory_api_version'] = DEFAULT_VERSION + + self.http_client = http.get_http_client(endpoint, session, **kwargs) + self.host = host.HostManager(self.http_client) + self.cpu = cpu.CpuManager(self.http_client) + self.ethernetport = ethernetport.EthernetPortManager(self.http_client) + self.lldp_agent = lldp_agent.LldpAgentManager(self.http_client) + self.lldp_neighbour = lldp_neighbour.LldpNeighbourManager( + self.http_client) + self.memory = memory.MemoryManager(self.http_client) + self.node = node.NodeManager(self.http_client) + self.pci_device = pci_device.PciDeviceManager(self.http_client) + self.port = port.PortManager(self.http_client) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/cpu.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/cpu.py new file mode 100644 index 00000000..60eb47fb --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/cpu.py @@ -0,0 +1,206 @@ +# -*- encoding: utf-8 -*- +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + + +from inventoryclient.common import base +from inventoryclient.common.i18n import _ +from inventoryclient import exc +import json + + +CREATION_ATTRIBUTES = ['host_uuid', 'node_uuid', 'cpu', 'core', 'thread', + 'cpu_family', 'cpu_model', 'allocated_function', + 'numa_node', 'capabilities', 'function', + 'num_cores_on_processor0', 'num_cores_on_processor1', + 'num_cores_on_processor2', 'num_cores_on_processor3'] + +PLATFORM_CPU_TYPE = "Platform" +VSWITCH_CPU_TYPE = "Vswitch" +SHARED_CPU_TYPE = "Shared" +VMS_CPU_TYPE = "VMs" +NONE_CPU_TYPE = "None" + +CPU_TYPE_LIST = [PLATFORM_CPU_TYPE, VSWITCH_CPU_TYPE, + SHARED_CPU_TYPE, VMS_CPU_TYPE, + NONE_CPU_TYPE] + + +PLATFORM_CPU_TYPE_FORMAT = _("Platform") +VSWITCH_CPU_TYPE_FORMAT = _("vSwitch") +SHARED_CPU_TYPE_FORMAT = _("Shared") +VMS_CPU_TYPE_FORMAT = _("VMs") +NONE_CPU_TYPE_FORMAT = _("None") + +CPU_TYPE_FORMATS = {PLATFORM_CPU_TYPE: PLATFORM_CPU_TYPE_FORMAT, + VSWITCH_CPU_TYPE: VSWITCH_CPU_TYPE_FORMAT, + SHARED_CPU_TYPE: SHARED_CPU_TYPE_FORMAT, + VMS_CPU_TYPE: VMS_CPU_TYPE_FORMAT, + NONE_CPU_TYPE: NONE_CPU_TYPE_FORMAT} + + +def _cpu_function_formatter(allocated_function): + if allocated_function in CPU_TYPE_FORMATS: + return CPU_TYPE_FORMATS[allocated_function] + return "unknown({})".format(allocated_function) + + +def _cpu_function_tuple_formatter(data): + return _cpu_function_formatter(data.allocated_function) + + +class Cpu(base.Resource): + def __repr__(self): + return "" % self._info + + +class CpuManager(base.Manager): + resource_class = Cpu + + def list(self, host_id): + path = '/v1/hosts/%s/cpus' % host_id + return self._list(path, "cpus") + + def get(self, cpu_id): + path = '/v1/cpus/%s' % cpu_id + try: + return self._list(path)[0] + except IndexError: + return None + + def create(self, **kwargs): + path = '/v1/cpus/' + new = {} + for (key, value) in kwargs.items(): + if key in CREATION_ATTRIBUTES: + new[key] = value + else: + raise exc.InvalidAttribute(key) + return self._create(path, new) + + def delete(self, cpu_id): + path = '/v1/cpus/%s' % cpu_id + return self._delete(path) + + def update(self, cpu_id, patch): + path = '/v1/cpus/%s' % cpu_id + return self._update(path, + data=(json.dumps(patch))) + + +class CpuFunction(object): + def __init__(self, function): + self.allocated_function = function + self.socket_cores = {} + self.socket_cores_number = {} + + +def check_core_functions(personality, cpus): + platform_cores = 0 + vswitch_cores = 0 + vm_cores = 0 + for cpu in cpus: + allocated_function = cpu.allocated_function + if allocated_function == PLATFORM_CPU_TYPE: + platform_cores += 1 + elif allocated_function == VSWITCH_CPU_TYPE: + vswitch_cores += 1 + elif allocated_function == VMS_CPU_TYPE: + vm_cores += 1 + + error_string = "" + if platform_cores == 0: + error_string = ("There must be at least one core for %s." % + PLATFORM_CPU_TYPE_FORMAT) + elif personality == 'compute' and vswitch_cores == 0: + error_string = ("There must be at least one core for %s." % + VSWITCH_CPU_TYPE_FORMAT) + elif personality == 'compute' and vm_cores == 0: + error_string = ("There must be at least one core for %s." % + VMS_CPU_TYPE_FORMAT) + return error_string + + +def compress_range(c_list): + c_list.append(999) + c_list.sort() + c_sep = "" + c_item = "" + c_str = "" + pn = 0 # pn is not used until second loop anyways + for n in c_list: + if not c_item: + c_item = "%s" % n + else: + if n > (pn + 1): + if int(pn) == int(c_item): + c_str = "%s%s%s" % (c_str, c_sep, c_item) + else: + c_str = "%s%s%s-%s" % (c_str, c_sep, c_item, pn) + c_sep = "," + c_item = "%s" % n + pn = n + return c_str + + +def restructure_host_cpu_data(host): + host.core_assignment = [] + if host.cpus: + host.cpu_model = host.cpus[0].cpu_model + host.sockets = len(host.nodes) + host.hyperthreading = "No" + host.physical_cores = 0 + + core_assignment = {} + number_of_cores = {} + host.node_min_max_cores = {} + + for cpu in host.cpus: + if cpu.numa_node == 0 and cpu.thread == 0: + host.physical_cores += 1 + elif cpu.thread > 0: + host.hyperthreading = "Yes" + + if cpu.numa_node not in host.node_min_max_cores: + host.node_min_max_cores[cpu.numa_node] = \ + {'min': 99999, 'max': 0} + if cpu.cpu < host.node_min_max_cores[cpu.numa_node]['min']: + host.node_min_max_cores[cpu.numa_node]['min'] = cpu.cpu + if cpu.cpu > host.node_min_max_cores[cpu.numa_node]['max']: + host.node_min_max_cores[cpu.numa_node]['max'] = cpu.cpu + + if cpu.allocated_function is None: + cpu.allocated_function = NONE_CPU_TYPE + + if cpu.allocated_function not in core_assignment: + core_assignment[cpu.allocated_function] = {} + number_of_cores[cpu.allocated_function] = {} + if cpu.numa_node not in core_assignment[cpu.allocated_function]: + core_assignment[cpu.allocated_function][cpu.numa_node] = \ + [int(cpu.cpu)] + number_of_cores[cpu.allocated_function][cpu.numa_node] = 1 + else: + core_assignment[ + cpu.allocated_function][cpu.numa_node].append(int(cpu.cpu)) + number_of_cores[cpu.allocated_function][cpu.numa_node] = \ + number_of_cores[cpu.allocated_function][cpu.numa_node] + 1 + + for f in CPU_TYPE_LIST: + cpufunction = CpuFunction(f) + if f in core_assignment: + host.core_assignment.append(cpufunction) + for s, cores in core_assignment[f].items(): + cpufunction.socket_cores[s] = compress_range(cores) + cpufunction.socket_cores_number[s] = number_of_cores[f][s] + else: + if (f == PLATFORM_CPU_TYPE or + (hasattr(host, 'subfunctions') and + 'compute' in host.subfunctions)): + if f != NONE_CPU_TYPE: + host.core_assignment.append(cpufunction) + for s in range(0, len(host.nodes)): + cpufunction.socket_cores[s] = "" + cpufunction.socket_cores_number[s] = 0 diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/cpu_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/cpu_shell.py new file mode 100644 index 00000000..50af61cd --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/cpu_shell.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _print_cpu_show(cpu): + fields = ['cpu', 'numa_node', 'core', 'thread', + 'cpu_model', 'cpu_family', + 'capabilities', + 'uuid', 'host_uuid', 'node_uuid', + 'created_at', 'updated_at'] + labels = ['logical_core', 'processor (numa_node)', 'physical_core', + 'thread', 'processor_model', 'processor_family', + 'capabilities', + 'uuid', 'host_uuid', 'node_uuid', + 'created_at', 'updated_at'] + data = [(f, getattr(cpu, f, '')) for f in fields] + utils.print_tuple_list(data, labels) + + +def _find_cpu(cc, host, cpunameoruuid): + cpus = cc.cpu.list(host.uuid) + + if cpunameoruuid.isdigit(): + cpunameoruuid = int(cpunameoruuid) + + for c in cpus: + if c.uuid == cpunameoruuid or c.cpu == cpunameoruuid: + break + else: + raise exc.CommandError('CPU logical core not found: host %s cpu %s' % + (host.hostname, cpunameoruuid)) + return c + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('cpulcoreoruuid', + metavar='', + help="CPU logical core ID or UUID of cpu") +def do_host_cpu_show(cc, args): + """Show cpu core attributes.""" + host = host_utils._find_host(cc, args.hostnameorid) + cpu = _find_cpu(cc, host, args.cpulcoreoruuid) + _print_cpu_show(cpu) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_cpu_list(cc, args): + """List cpu cores.""" + + host = host_utils._find_host(cc, args.hostnameorid) + + cpus = cc.cpu.list(host.uuid) + + field_labels = ['uuid', 'log_core', 'processor', 'phy_core', 'thread', + 'processor_model'] + fields = ['uuid', 'cpu', 'numa_node', 'core', 'thread', + 'cpu_model'] + + utils.print_list(cpus, fields, field_labels, sortby=1) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport.py new file mode 100644 index 00000000..5817bdac --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base +from inventoryclient import exc +import json + + +CREATION_ATTRIBUTES = ['host_uuid', 'name', 'mtu', 'speed', 'bootp', + 'interface_uuid', 'pdevice', 'pclass', 'pciaddr', + 'psdevice', 'link_mode', 'psvendor', 'pvendor'] + + +class EthernetPort(base.Resource): + def __repr__(self): + return "" % self._info + + +class EthernetPortManager(base.Manager): + resource_class = EthernetPort + + def list(self, host_id): + path = '/v1/hosts/%s/ethernet_ports' % host_id + return self._list(path, "ethernet_ports") + + def get(self, port_id): + path = '/v1/ethernet_ports/%s' % port_id + try: + return self._list(path)[0] + except IndexError: + return None + + def create(self, **kwargs): + path = '/v1/ethernet_ports/' + new = {} + for (key, value) in kwargs.items(): + if key in CREATION_ATTRIBUTES: + new[key] = value + else: + raise exc.InvalidAttribute(key) + return self._create(path, new) + + def delete(self, port_id): + path = '/v1/ethernet_ports/%s' % port_id + return self._delete(path) + + def update(self, port_id, patch): + path = '/v1/ethernet_ports/%s' % port_id + return self._update(path, + data=(json.dumps(patch))) + + +def get_port_display_name(p): + if p.name: + return p.name + if p.namedisplay: + return p.namedisplay + else: + return '(' + str(p.uuid)[-8:] + ')' diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport_shell.py new file mode 100644 index 00000000..674aa3e1 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/ethernetport_shell.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _bootp_formatter(value): + return bool(value) + + +def _bootp_port_formatter(port): + return _bootp_formatter(port.bootp) + + +def _print_ethernet_port_show(port): + fields = ['name', 'namedisplay', + 'mac', 'pciaddr', + 'numa_node', + 'autoneg', 'bootp', + 'pclass', 'pvendor', 'pdevice', + 'link_mode', 'capabilities', + 'uuid', 'host_uuid', + 'created_at', 'updated_at'] + labels = ['name', 'namedisplay', + 'mac', 'pciaddr', + 'processor', + 'autoneg', 'bootp', + 'pclass', 'pvendor', 'pdevice', + 'link_mode', 'capabilities', + 'uuid', 'host_uuid', + 'created_at', 'updated_at'] + data = [(f, getattr(port, f, '')) for f in fields] + utils.print_tuple_list(data, labels, + formatters={'bootp': _bootp_formatter}) + + +def _find_port(cc, host, portnameoruuid): + ports = cc.ethernetport.list(host.uuid) + for p in ports: + if p.name == portnameoruuid or p.uuid == portnameoruuid: + break + else: + raise exc.CommandError('Ethernet port not found: host %s port %s' % + (host.id, portnameoruuid)) + p.autoneg = 'Yes' + return p + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('pnameoruuid', metavar='', + help="Name or UUID of port") +def do_host_ethernet_port_show(cc, args): + """Show host ethernet port attributes.""" + host = host_utils._find_host(cc, args.hostnameorid) + port = _find_port(cc, host, args.pnameoruuid) + _print_ethernet_port_show(port) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_ethernet_port_list(cc, args): + """List host ethernet ports.""" + host = host_utils._find_host(cc, args.hostnameorid) + + ports = cc.ethernetport.list(host.uuid) + for p in ports: + p.autoneg = 'Yes' # TODO(jkung) Remove when autoneg supported in DB + + field_labels = ['uuid', 'name', 'mac address', 'pci address', 'processor', + 'auto neg', 'device type', 'boot i/f'] + fields = ['uuid', 'name', 'mac', 'pciaddr', 'numa_node', + 'autoneg', 'pdevice', 'bootp'] + + utils.print_list(ports, fields, field_labels, sortby=1, + formatters={'bootp': _bootp_port_formatter}) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/host.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/host.py new file mode 100644 index 00000000..48adcec2 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/host.py @@ -0,0 +1,116 @@ +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base +from inventoryclient.common import utils +from inventoryclient import exc +import json + + +CREATION_ATTRIBUTES = ['hostname', 'personality', 'subfunctions', + 'mgmt_mac', 'mgmt_ip', + 'bm_ip', 'bm_type', 'bm_username', + 'bm_password', 'serialid', 'location', + 'boot_device', 'rootfs_device', 'install_output', + 'console', 'tboot', 'ttys_dcd', + 'administrative', 'operational', 'availability', + 'invprovision'] + + +class Host(base.Resource): + def __repr__(self): + return "" % self._info + + +class HostManager(base.Manager): + resource_class = Host + + @staticmethod + def _path(id=None): + return '/v1/hosts/%s' % id if id else '/v1/hosts' + + def list(self): + return self._list(self._path(), "hosts") + + def list_port(self, host_id): + path = "%s/ports" % host_id + return self._list(self._path(path), "ports") + + def list_ethernet_port(self, host_id): + path = "%s/ethernet_ports" % host_id + return self._list(self._path(path), "ethernet_ports") + + def list_personality(self, personality): + path = self._path() + "?personality=%s" % personality + return self._list(path, "hosts") + + def get(self, host_id): + try: + return self._list(self._path(host_id))[0] + except IndexError: + return None + + def create(self, **kwargs): + new = {} + for (key, value) in kwargs.items(): + if key in CREATION_ATTRIBUTES: + new[key] = value + else: + raise exc.InvalidAttribute() + return self._create(self._path(), new) + + def upgrade(self, hostid, force): + new = {} + new['force'] = force + resp, body = self.api.json_request( + 'POST', self._path(hostid) + "/upgrade", body=new) + return self.resource_class(self, body) + + def downgrade(self, hostid, force): + new = {} + new['force'] = force + resp, body = self.api.json_request( + 'POST', self._path(hostid) + "/downgrade", body=new) + return self.resource_class(self, body) + + def create_many(self, body): + return self._upload(self._path() + "/bulk_add", body) + + def update_install_uuid(self, hostid, install_uuid): + path = self._path(hostid) + "/state/update_install_uuid" + + self.api.json_request('PUT', path, body=install_uuid) + + def delete(self, host_id): + return self._delete(self._path(host_id)) + + def update(self, host_id, patch): + return self._update(self._path(host_id), + data=(json.dumps(patch))) + + def bulk_export(self): + result = self._json_get(self._path('bulk_export')) + return result + + +def _find_host(cc, host): + if host.isdigit() or utils.is_uuid_like(host): + try: + h = cc.host.get(host) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % host) + else: + return h + else: + hostlist = cc.host.list() + for h in hostlist: + if h.hostname == host: + return h + else: + raise exc.CommandError('host not found: %s' % host) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/host_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/host_shell.py new file mode 100644 index 00000000..c37624e4 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/host_shell.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013-2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from collections import OrderedDict +import datetime +import os + +from inventoryclient.common.i18n import _ +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _print_host_show(host): + fields = ['id', 'uuid', 'personality', 'hostname', 'invprovision', + 'administrative', 'operational', 'availability', 'task', + 'action', 'mgmt_mac', 'mgmt_ip', 'serialid', + 'capabilities', 'bm_type', 'bm_username', 'bm_ip', + 'location', 'uptime', 'reserved', 'created_at', 'updated_at', + 'boot_device', 'rootfs_device', 'install_output', 'console', + 'tboot', 'vim_progress_status', 'software_load', 'install_state', + 'install_state_info'] + optional_fields = ['ttys_dcd'] + if host.subfunctions != host.personality: + fields.append('subfunctions') + if 'controller' in host.subfunctions: + fields.append('subfunction_oper') + fields.append('subfunction_avail') + + # Do not display the trailing '+' which indicates the audit iterations + if host.install_state_info: + host.install_state_info = host.install_state_info.rstrip('+') + if host.install_state: + host.install_state = host.install_state.rstrip('+') + + data_list = [(f, getattr(host, f, '')) for f in fields] + data_list += [(f, getattr(host, f, '')) for f in optional_fields + if hasattr(host, f)] + data = dict(data_list) + ordereddata = OrderedDict(sorted(data.items(), key=lambda t: t[0])) + utils.print_dict(ordereddata, wrap=72) + + +@utils.arg('hostnameorid', metavar='', + help="Name or ID of host") +def do_host_show(cc, args): + """Show host attributes.""" + host = host_utils._find_host(cc, args.hostnameorid) + _print_host_show(host) + + +def do_host_list(cc, args): + """List hosts.""" + hosts = cc.host.list() + field_labels = ['id', 'hostname', 'personality', + 'administrative', 'operational', 'availability'] + fields = ['id', 'hostname', 'personality', + 'administrative', 'operational', 'availability'] + utils.print_list(hosts, fields, field_labels, sortby=0) + + +@utils.arg('-n', '--hostname', + metavar='', + help='Hostname of the host') +@utils.arg('-p', '--personality', + metavar='', + choices=['controller', 'compute', 'storage', 'network', 'profile'], + help='Personality or type of host [REQUIRED]') +@utils.arg('-s', '--subfunctions', + metavar='', + choices=['lowlatency'], + help='Performance profile or subfunctions of host.[Optional]') +@utils.arg('-m', '--mgmt_mac', + metavar='', + help='MAC Address of the host mgmt interface [REQUIRED]') +@utils.arg('-i', '--mgmt_ip', + metavar='', + help='IP Address of the host mgmt interface (when using static ' + 'address allocation)') +@utils.arg('-I', '--bm_ip', + metavar='', + help="IP Address of the host board management interface, " + "only necessary if this host's board management controller " + "is not in the primary region") +@utils.arg('-T', '--bm_type', + metavar='', + help='Type of the host board management interface') +@utils.arg('-U', '--bm_username', + metavar='', + help='Username for the host board management interface') +@utils.arg('-P', '--bm_password', + metavar='', + help='Password for the host board management interface') +@utils.arg('-b', '--boot_device', + metavar='', + help='Device for boot partition, relative to /dev. Default: sda') +@utils.arg('-r', '--rootfs_device', + metavar='', + help='Device for rootfs partition, relative to /dev. Default: sda') +@utils.arg('-o', '--install_output', + metavar='', + choices=['text', 'graphical'], + help='Installation output format, text or graphical. Default: text') +@utils.arg('-c', '--console', + metavar='', + help='Serial console. Default: ttyS0,115200') +@utils.arg('-l', '--location', + metavar='', + help='Physical location of the host') +@utils.arg('-D', '--ttys_dcd', + metavar='', + help='Enable/disable serial console data carrier detection') +def do_host_add(cc, args): + """Add a new host.""" + field_list = ['hostname', 'personality', 'subfunctions', + 'mgmt_mac', 'mgmt_ip', + 'bm_ip', 'bm_type', 'bm_username', 'bm_password', + 'boot_device', 'rootfs_device', 'install_output', 'console', + 'location', 'ttys_dcd'] + fields = dict((k, v) for (k, v) in vars(args).items() + if k in field_list and not (v is None)) + + # This is the expected format of the location field + if 'location' in fields: + fields['location'] = {"locn": fields['location']} + + host = cc.host.create(**fields) + suuid = getattr(host, 'uuid', '') + + try: + host = cc.host.get(suuid) + except exc.HTTPNotFound: + raise exc.CommandError('Host not found: %s' % suuid) + else: + _print_host_show(host) + + +@utils.arg('hostsfile', + metavar='', + help='File containing the XML descriptions of hosts to be ' + 'provisioned [REQUIRED]') +def do_host_bulk_add(cc, args): + """Add multiple new hosts.""" + field_list = ['hostsfile'] + fields = dict((k, v) for (k, v) in vars(args).items() + if k in field_list and not (v is None)) + + hostsfile = fields['hostsfile'] + if os.path.isdir(hostsfile): + raise exc.CommandError("Error: %s is a directory." % hostsfile) + try: + req = open(hostsfile, 'rb') + except Exception: + raise exc.CommandError("Error: Could not open file %s." % hostsfile) + + response = cc.host.create_many(req) + if not response: + raise exc.CommandError("The request timed out or there was an " + "unknown error") + success = response.get('success') + error = response.get('error') + if success: + print("Success: " + success + "\n") + if error: + print("Error:\n" + error) + + +@utils.arg('hostnameorid', + metavar='', + nargs='+', + help="Name or ID of host") +def do_host_delete(cc, args): + """Delete a host.""" + for n in args.hostnameorid: + try: + cc.host.delete(n) + print('Deleted host %s' % n) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % n) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('attributes', + metavar='', + nargs='+', + action='append', + default=[], + help="Attributes to update ") +def do_host_update(cc, args): + """Update host attributes.""" + patch = utils.args_array_to_patch("replace", args.attributes[0]) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('-f', '--force', + action='store_true', + default=False, + help="Force a lock operation ") +def do_host_lock(cc, args): + """Lock a host.""" + attributes = [] + + if args.force is True: + # Forced lock operation + attributes.append('action=force-lock') + else: + # Normal lock operation + attributes.append('action=lock') + + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('-f', '--force', + action='store_true', + default=False, + help="Force an unlock operation ") +def do_host_unlock(cc, args): + """Unlock a host.""" + attributes = [] + + if args.force is True: + # Forced unlock operation + attributes.append('action=force-unlock') + else: + # Normal unlock operation + attributes.append('action=unlock') + + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('-f', '--force', + action='store_true', + default=False, + help="Force a host swact operation ") +def do_host_swact(cc, args): + """Switch activity away from this active host.""" + attributes = [] + + if args.force is True: + # Forced swact operation + attributes.append('action=force-swact') + else: + # Normal swact operation + attributes.append('action=swact') + + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_reset(cc, args): + """Reset a host.""" + attributes = [] + attributes.append('action=reset') + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_reboot(cc, args): + """Reboot a host.""" + attributes = [] + attributes.append('action=reboot') + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_reinstall(cc, args): + """Reinstall a host.""" + attributes = [] + attributes.append('action=reinstall') + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_power_on(cc, args): + """Power on a host.""" + attributes = [] + attributes.append('action=power-on') + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_power_off(cc, args): + """Power off a host.""" + attributes = [] + attributes.append('action=power-off') + patch = utils.args_array_to_patch("replace", attributes) + host = host_utils._find_host(cc, args.hostnameorid) + try: + host = cc.host.update(host.id, patch) + except exc.HTTPNotFound: + raise exc.CommandError('host not found: %s' % args.hostnameorid) + _print_host_show(host) + + +def _timestamped(dname, fmt='%Y-%m-%d-%H-%M-%S_{dname}'): + return datetime.datetime.now().strftime(fmt).format(dname=dname) + + +@utils.arg('--filename', + help="The full file path to store the host file. Default './hosts.xml'") # noqa +def do_host_bulk_export(cc, args): + """Export host bulk configurations.""" + result = cc.host.bulk_export() + + xml_content = result['content'] + config_filename = './hosts.xml' + if hasattr(args, 'filename') and args.filename: + config_filename = args.filename + try: + with open(config_filename, 'wb') as fw: + fw.write(xml_content) + print(_('Export successfully to %s') % config_filename) + except IOError: + print(_('Cannot write to file: %s') % config_filename) + + return diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent.py new file mode 100644 index 00000000..c60c57f0 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base + + +class LldpAgent(base.Resource): + def __repr__(self): + return "" % self._info + + +class LldpAgentManager(base.Manager): + resource_class = LldpAgent + + def list(self, host_id): + path = '/v1/hosts/%s/lldp_agents' % host_id + agents = self._list(path, "lldp_agents") + return agents + + def get(self, uuid): + path = '/v1/lldp_agents/%s' % uuid + try: + return self._list(path)[0] + except IndexError: + return None + + def get_by_port(self, port_id): + path = '/v1/ports/%s/lldp_agents' % port_id + return self._list(path, "lldp_agents") diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent_shell.py new file mode 100644 index 00000000..cfcea11f --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_agent_shell.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient.v1 import host as host_utils + + +class LldpAgentObj(object): + def __init__(self, dictionary): + for k, v in dictionary.items(): + setattr(self, k, v) + + +def _print_lldp_agent_show(agent): + fields = ['uuid', 'host_uuid', + 'created_at', 'updated_at', + 'uuid', 'port_name', 'chassis_id', 'port_identifier', 'ttl', + 'system_description', 'system_name', 'system_capabilities', + 'management_address', 'port_description', 'dot1_lag', + 'dot1_vlan_names', + 'dot3_mac_status', 'dot3_max_frame' + ] + labels = ['uuid', 'host_uuid', + 'created_at', 'updated_at', + 'uuid', 'local_port', 'chassis_id', 'port_identifier', 'ttl', + 'system_description', 'system_name', 'system_capabilities', + 'management_address', 'port_description', 'dot1_lag', + 'dot1_vlan_names', + 'dot3_mac_status', 'dot3_max_frame' + ] + data = [(f, getattr(agent, f, '')) for f in fields] + utils.print_tuple_list(data, labels) + + +def _lldp_carriage_formatter(value): + chars = ['\n', '\\n', '\r', '\\r'] + for char in chars: + if char in value: + value = value.replace(char, '. ') + return value + + +def _lldp_system_name_formatter(lldp): + system_name = getattr(lldp, 'system_name') + if system_name: + return _lldp_carriage_formatter(system_name) + + +def _lldp_system_description_formatter(lldp): + system_description = getattr(lldp, 'system_description') + if system_description: + return _lldp_carriage_formatter(system_description) + + +def _lldp_port_description_formatter(lldp): + port_description = getattr(lldp, 'port_description') + if port_description: + return _lldp_carriage_formatter(port_description) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_lldp_agent_list(cc, args): + """List host lldp agents.""" + host = host_utils._find_host(cc, args.hostnameorid) + agents = cc.lldp_agent.list(host.uuid) + + field_labels = ['uuid', 'local_port', 'status', 'chassis_id', 'port_id', + 'system_name', 'system_description'] + fields = ['uuid', 'port_name', 'status', 'chassis_id', 'port_identifier', + 'system_name', 'system_description'] + formatters = {'system_name': _lldp_system_name_formatter, + 'system_description': _lldp_system_description_formatter, + 'port_description': _lldp_port_description_formatter} + + utils.print_list(agents, fields, field_labels, sortby=1, + formatters=formatters) + + +@utils.arg('uuid', + metavar='', + help="UUID of the LLDP agent") +def do_lldp_agent_show(cc, args): + """Show LLDP agent attributes.""" + agent = cc.lldp_agent.get(args.uuid) + _print_lldp_agent_show(agent) + return diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour.py new file mode 100644 index 00000000..93ab6283 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base + + +class LldpNeighbour(base.Resource): + def __repr__(self): + return "" % self._info + + +class LldpNeighbourManager(base.Manager): + resource_class = LldpNeighbour + + def list(self, host_id): + path = '/v1/hosts/%s/lldp_neighbours' % host_id + neighbours = self._list(path, "lldp_neighbours") + return neighbours + + def list_by_port(self, port_id): + path = '/v1/ports/%s/lldp_neighbours' % port_id + return self._list(path, "lldp_neighbours") + + def get(self, uuid): + path = '/v1/lldp_neighbours/%s' % uuid + try: + return self._list(path)[0] + except IndexError: + return None diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour_shell.py new file mode 100644 index 00000000..9dbe5d18 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/lldp_neighbour_shell.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +# +# Copyright (c) 2016 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient.v1 import host as host_utils + + +class LldpNeighbourObj(object): + def __init__(self, dictionary): + for k, v in dictionary.items(): + setattr(self, k, v) + + +def _lldp_carriage_formatter(value): + chars = ['\n', '\\n', '\r', '\\r'] + for char in chars: + if char in value: + value = value.replace(char, '. ') + return value + + +def _lldp_system_name_formatter(lldp): + system_name = getattr(lldp, 'system_name') + if system_name: + return _lldp_carriage_formatter(system_name) + + +def _lldp_system_description_formatter(lldp): + system_description = getattr(lldp, 'system_description') + if system_description: + return _lldp_carriage_formatter(system_description) + + +def _lldp_port_description_formatter(lldp): + port_description = getattr(lldp, 'port_description') + if port_description: + return _lldp_carriage_formatter(port_description) + + +def _print_lldp_neighbour_show(neighbour): + fields = ['uuid', 'host_uuid', + 'created_at', 'updated_at', + 'uuid', 'port_name', 'chassis_id', 'port_identifier', 'ttl', + 'msap', 'system_description', 'system_name', + 'system_capabilities', 'management_address', 'port_description', + 'dot1_lag', 'dot1_port_vid', 'dot1_vlan_names', + 'dot1_proto_vids', 'dot1_proto_ids', 'dot3_mac_status', + 'dot3_max_frame' + ] + + labels = ['uuid', 'host_uuid', + 'created_at', 'updated_at', + 'uuid', 'local_port', 'chassis_id', 'port_identifier', 'ttl', + 'msap', 'system_description', 'system_name', + 'system_capabilities', 'management_address', 'port_description', + 'dot1_lag', 'dot1_port_vid', 'dot1_vlan_names', + 'dot1_proto_vids', 'dot1_proto_ids', 'dot3_mac_status', + 'dot3_max_frame' + ] + data = [(f, getattr(neighbour, f, '')) for f in fields] + utils.print_tuple_list(data, labels) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_lldp_neighbor_list(cc, args): + """List host lldp neighbors.""" + host = host_utils._find_host(cc, args.hostnameorid) + neighbours = cc.lldp_neighbour.list(host.uuid) + + field_labels = ['uuid', 'local_port', 'remote_port', 'chassis_id', + 'system_name', 'system_description', + 'management_address'] + fields = ['uuid', 'port_name', 'port_identifier', 'chassis_id', + 'system_name', 'system_description', + 'management_address'] + formatters = {'system_name': _lldp_system_name_formatter, + 'system_description': _lldp_system_description_formatter, + 'port_description': _lldp_port_description_formatter} + + utils.print_list(neighbours, fields, field_labels, sortby=1, + formatters=formatters) + + +@utils.arg('uuid', + metavar='', + help="UUID of the LLDP neighbor") +def do_lldp_neighbor_show(cc, args): + """Show LLDP neighbor attributes.""" + neighbour = cc.lldp_neighbour.get(args.uuid) + _print_lldp_neighbour_show(neighbour) + return diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/memory.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/memory.py new file mode 100644 index 00000000..83f259cc --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/memory.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base +from inventoryclient import exc +import json + +CREATION_ATTRIBUTES = ['host_uuid', 'memtotal_mib', 'memavail_mib', + 'platform_reserved_mib', 'hugepages_configured', + 'vswitch_hugepages_size_mib', 'vswitch_hugepages_reqd', + 'vswitch_hugepages_nr', 'vswitch_hugepages_avail', + 'vm_hugepages_nr_2M_pending', + 'vm_hugepages_nr_1G_pending', + 'vm_hugepages_nr_2M', 'vm_hugepages_avail_2M', + 'vm_hugepages_nr_1G', 'vm_hugepages_avail_1G', + 'vm_hugepages_avail_1G', 'vm_hugepages_use_1G', + 'vm_hugepages_possible_2M', 'vm_hugepages_possible_1G', + 'capabilities', 'numa_node', + 'minimum_platform_reserved_mib'] + + +class Memory(base.Resource): + def __repr__(self): + return "" % self._info + + +class MemoryManager(base.Manager): + resource_class = Memory + + @staticmethod + def _path(id=None): + return '/v1/memorys/%s' % id if id else '/v1/memorys' + + def list(self, host_id): + path = '/v1/hosts/%s/memorys' % host_id + return self._list(path, "memorys") + + def get(self, memory_id): + path = '/v1/memorys/%s' % memory_id + try: + return self._list(path)[0] + except IndexError: + return None + + def update(self, memory_id, patch): + return self._update(self._path(memory_id), + data=(json.dumps(patch))) + + def create(self, **kwargs): + path = '/v1/memorys' + new = {} + for (key, value) in kwargs.items(): + if key in CREATION_ATTRIBUTES: + new[key] = value + else: + raise exc.InvalidAttribute('%s' % key) + return self._create(path, new) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/memory_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/memory_shell.py new file mode 100644 index 00000000..1d6c4812 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/memory_shell.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _print_memory_show(memory): + fields = ['memtotal_mib', + 'platform_reserved_mib', + 'memavail_mib', + 'hugepages_configured', + 'vswitch_hugepages_size_mib', + 'vswitch_hugepages_nr', + 'vswitch_hugepages_avail', + 'vm_hugepages_nr_4K', + 'vm_hugepages_nr_2M', + 'vm_hugepages_nr_2M_pending', + 'vm_hugepages_avail_2M', + 'vm_hugepages_nr_1G', + 'vm_hugepages_nr_1G_pending', + 'vm_hugepages_avail_1G', + 'uuid', 'host_uuid', 'node_uuid', + 'created_at', 'updated_at'] + labels = ['Memory: Usable Total (MiB)', + ' Platform (MiB)', + ' Available (MiB)', + 'Huge Pages Configured', + 'vSwitch Huge Pages: Size (MiB)', + ' Total', + ' Available', + 'VM Pages (4K): Total', + 'VM Huge Pages (2M): Total', + ' Total Pending', + ' Available', + 'VM Huge Pages (1G): Total', + ' Total Pending', + ' Available', + 'uuid', 'host_uuid', 'node_uuid', + 'created_at', 'updated_at'] + + data = [(f, getattr(memory, f, '')) for f in fields] + + for d in data: + if d[0] == 'vm_hugepages_nr_2M_pending': + if d[1] is None: + fields.remove(d[0]) + labels.pop(labels.index(' Total Pending')) + + if d[0] == 'vm_hugepages_nr_1G_pending': + if d[1] is None: + fields.remove(d[0]) + labels.pop(len(labels) - labels[::-1].index( + ' Total Pending') - 1) + + data = [(f, getattr(memory, f, '')) for f in fields] + utils.print_tuple_list(data, labels) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('numa_node', + metavar='', + help="processor") +def do_host_memory_show(cc, args): + """Show memory attributes.""" + host = host_utils._find_host(cc, args.hostnameorid) + nodes = cc.node.list(host.uuid) + memorys = cc.memory.list(host.uuid) + for m in memorys: + for n in nodes: + if m.node_uuid == n.uuid: + if int(n.numa_node) == int(args.numa_node): + _print_memory_show(m) + return + else: + raise exc.CommandError('Processor not found: host %s processor %s' % + (host.hostname, args.numa_node)) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_memory_list(cc, args): + """List memory nodes.""" + + host = host_utils._find_host(cc, args.hostnameorid) + + nodes = cc.node.list(host.uuid) + memorys = cc.memory.list(host.uuid) + for m in memorys: + for n in nodes: + if m.node_uuid == n.uuid: + m.numa_node = n.numa_node + break + + fields = ['numa_node', + 'memtotal_mib', + 'platform_reserved_mib', + 'memavail_mib', + 'hugepages_configured', + 'vswitch_hugepages_size_mib', + 'vswitch_hugepages_nr', + 'vswitch_hugepages_avail', + 'vm_hugepages_nr_4K', + 'vm_hugepages_nr_2M', + 'vm_hugepages_avail_2M', + 'vm_hugepages_nr_2M_pending', + 'vm_hugepages_nr_1G', + 'vm_hugepages_avail_1G', + 'vm_hugepages_nr_1G_pending', + 'vm_hugepages_use_1G'] + + field_labels = ['processor', + 'mem_total(MiB)', + 'mem_platform(MiB)', + 'mem_avail(MiB)', + 'hugepages(hp)_configured', + 'vs_hp_size(MiB)', + 'vs_hp_total', + 'vs_hp_avail', + 'vm_total_4K', + 'vm_hp_total_2M', + 'vm_hp_avail_2M', + 'vm_hp_pending_2M', + 'vm_hp_total_1G', + 'vm_hp_avail_1G', + 'vm_hp_pending_1G', + 'vm_hp_use_1G'] + + utils.print_list(memorys, fields, field_labels, sortby=1) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('numa_node', + metavar='', + help="processor") +@utils.arg('-m', '--platform_reserved_mib', + metavar='', + help='The amount of platform memory (MiB) for the numa node') +@utils.arg('-2M', '--vm_hugepages_nr_2M_pending', + metavar='<2M hugepages number>', + help='The number of 2M vm huge pages for the numa node') +@utils.arg('-1G', '--vm_hugepages_nr_1G_pending', + metavar='<1G hugepages number>', + help='The number of 1G vm huge pages for the numa node') +def do_host_memory_modify(cc, args): + """Modify platform reserved and/or libvirt vm huge page memory + attributes for compute nodes. + """ + + rwfields = ['platform_reserved_mib', + 'vm_hugepages_nr_2M_pending', + 'vm_hugepages_nr_1G_pending'] + + host = host_utils._find_host(cc, args.hostnameorid) + + user_specified_fields = dict((k, v) for (k, v) in vars(args).items() + if k in rwfields and not (v is None)) + + host = host_utils._find_host(cc, args.hostnameorid) + nodes = cc.node.list(host.uuid) + memorys = cc.memory.list(host.uuid) + mem = None + for m in memorys: + for n in nodes: + if m.node_uuid == n.uuid: + if int(n.numa_node) == int(args.numa_node): + mem = m + break + if mem: + break + + if mem is None: + raise exc.CommandError('Processor not found: host %s processor %s' % + (host.hostname, args.numa_node)) + + patch = [] + for (k, v) in user_specified_fields.items(): + patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) + + if patch: + memory = cc.memory.update(mem.uuid, patch) + _print_memory_show(memory) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/node.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/node.py new file mode 100644 index 00000000..8a4acd18 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/node.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base +from inventoryclient import exc +import json + + +CREATION_ATTRIBUTES = ['numa_node', 'capabilities', 'host_uuid'] + + +class Node(base.Resource): + def __repr__(self): + return "" % self._info + + +class NodeManager(base.Manager): + resource_class = Node + + def list(self, host_id): + path = '/v1/hosts/%s/nodes' % host_id + return self._list(path, "nodes") + + def get(self, node_id): + path = '/v1/nodes/%s' % node_id + try: + return self._list(path)[0] + except IndexError: + return None + + def create(self, **kwargs): + path = '/v1/nodes' + new = {} + for (key, value) in kwargs.items(): + if key in CREATION_ATTRIBUTES: + new[key] = value + else: + raise exc.InvalidAttribute('%s' % key) + return self._create(path, new) + + def delete(self, node_id): + path = '/v1/nodes/%s' % node_id + return self._delete(path) + + def update(self, node_id, patch): + path = '/v1/nodes/%s' % node_id + return self._update(path, + data=(json.dumps(patch))) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/node_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/node_shell.py new file mode 100644 index 00000000..d5ba51d3 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/node_shell.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _print_node_show(node): + fields = ['numa_node', + 'uuid', 'host_uuid', + 'created_at'] + data = [(f, getattr(node, f, '')) for f in fields] + utils.print_tuple_list(data) + + +def _find_node(cc, host, nodeuuid): + nodes = cc.node.list(host.uuid) + for i in nodes: + if i.uuid == nodeuuid: + break + else: + raise exc.CommandError('Inode not found: host %s if %s' % + (host.hostname, nodeuuid)) + return i + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('nodeuuid', + metavar='', + help="Name or UUID of node") +def donot_host_node_show(cc, args): + """Show a node. DEBUG only""" + host = host_utils._find_host(cc, args.hostnameorid) + # API actually doesnt need hostid once it has node uuid + + i = _find_node(cc, host, args.nodeuuid) + + _print_node_show(i) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def donot_host_node_list(cc, args): + """List nodes. DEBUG only""" + host = host_utils._find_host(cc, args.hostnameorid) + + nodes = cc.node.list(host.uuid) + + field_labels = ['uuid', 'numa_node', 'capabilities'] + fields = ['uuid', 'numa_node', 'capabilities'] + utils.print_list(nodes, fields, field_labels, sortby=0) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device.py new file mode 100755 index 00000000..edb6d715 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base +import json + + +class PciDevice(base.Resource): + def __repr__(self): + return "" % self._info + + +class PciDeviceManager(base.Manager): + resource_class = PciDevice + + def list(self, host_id): + path = '/v1/hosts/%s/pci_devices' % host_id + return self._list(path, "pci_devices") + + def list_all(self): + path = '/v1/pci_devices' + return self._list(path, "pci_devices") + + def get(self, pci_id): + path = '/v1/pci_devices/%s' % pci_id + try: + return self._list(path)[0] + except IndexError: + return None + + def update(self, pci_id, patch): + path = '/v1/pci_devices/%s' % pci_id + return self._update(path, + data=(json.dumps(patch))) + + +def get_pci_device_display_name(p): + if p.name: + return p.name + else: + return '(' + str(p.uuid)[-8:] + ')' diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device_shell.py new file mode 100644 index 00000000..f8afdb7b --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/pci_device_shell.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python +# +# Copyright (c) 2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _print_device_show(device): + fields = ['name', 'pciaddr', 'pclass_id', 'pvendor_id', 'pdevice_id', + 'pclass', 'pvendor', 'pdevice', 'numa_node', 'enabled', + 'sriov_totalvfs', 'sriov_numvfs', 'sriov_vfs_pci_address', + 'extra_info', 'created_at', 'updated_at'] + + labels = ['name', 'address', 'class id', 'vendor id', 'device id', + 'class name', 'vendor name', 'device name', 'numa_node', + 'enabled', 'sriov_totalvfs', 'sriov_numvfs', + 'sriov_vfs_pci_address', 'extra_info', 'created_at', + 'updated_at'] + + data = [(f, getattr(device, f, '')) for f in fields] + utils.print_tuple_list(data, labels) + + +def _find_device(cc, host, nameorpciaddr): + devices = cc.pci_device.list(host.uuid) + for d in devices: + if d.name == nameorpciaddr or d.pciaddr == nameorpciaddr: + break + else: + raise exc.CommandError('PCI devices not found: host %s device %s' % + (host.id, nameorpciaddr)) + return d + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('nameorpciaddr', + metavar='', + help="Name or PCI address of device") +def do_host_device_show(cc, args): + """Show device attributes.""" + host = host_utils._find_host(cc, args.hostnameorid) + device = _find_device(cc, host, args.nameorpciaddr) + _print_device_show(device) + return + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('-a', '--all', + action='store_true', + help='List all devices, including those that are not enabled') +def do_host_device_list(cc, args): + """List devices.""" + + host = host_utils._find_host(cc, args.hostnameorid) + devices = cc.pci_device.list(host.uuid) + for device in devices[:]: + if not args.all: + if not device.enabled: + devices.remove(device) + + fields = ['name', 'pciaddr', 'pclass_id', 'pvendor_id', 'pdevice_id', + 'pclass', 'pvendor', 'pdevice', 'numa_node', 'enabled'] + + labels = ['name', 'address', 'class id', 'vendor id', 'device id', + 'class name', 'vendor name', 'device name', 'numa_node', + 'enabled'] + + utils.print_list(devices, fields, labels, sortby=1) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('nameorpciaddr', + metavar='', + help="Name or PCI address of device") +@utils.arg('-n', '--name', + metavar='', + help='The new name of the device') +@utils.arg('-e', '--enabled', + metavar='', + help='The enabled status of the device') +def do_host_device_modify(cc, args): + """Modify device availability for compute nodes.""" + + rwfields = ['enabled', + 'name'] + + host = host_utils._find_host(cc, args.hostnameorid) + + user_specified_fields = dict((k, v) for (k, v) in vars(args).items() + if k in rwfields and not (v is None)) + + device = _find_device(cc, host, args.nameorpciaddr) + + fields = device.__dict__ + fields.update(user_specified_fields) + + patch = [] + for (k, v) in user_specified_fields.items(): + patch.append({'op': 'replace', 'path': '/' + k, 'value': v}) + + if patch: + try: + device = cc.pci_device.update(device.uuid, patch) + _print_device_show(device) + except exc.HTTPNotFound: + raise exc.CommandError('Device update failed: host %s ' + 'device %s : update %s' % + (args.hostnameorid, + args.nameorpciaddr, + patch)) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/port.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/port.py new file mode 100644 index 00000000..4ef27e27 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/port.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2013-2014 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# -*- encoding: utf-8 -*- +# + +from inventoryclient.common import base + + +class Port(base.Resource): + def __repr__(self): + return "" % self._info + + +class PortManager(base.Manager): + resource_class = Port + + def list(self, host_id): + path = '/v1/hosts/%s/ports' % host_id + return self._list(path, "ports") + + def get(self, port_id): + path = '/v1/ports/%s' % port_id + try: + return self._list(path)[0] + except IndexError: + return None + + +def get_port_display_name(p): + if p.name: + return p.name + if p.namedisplay: + return p.namedisplay + else: + return '(' + str(p.uuid)[-8:] + ')' diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/port_shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/port_shell.py new file mode 100644 index 00000000..d513245b --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/port_shell.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Copyright (c) 2013-2015 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# All Rights Reserved. +# + +from inventoryclient.common import utils +from inventoryclient import exc +from inventoryclient.v1 import host as host_utils + + +def _print_port_show(port): + fields = ['name', 'namedisplay', + 'type', 'pciaddr', 'dev_id', 'numa_node', + 'sriov_totalvfs', 'sriov_numvfs', + 'sriov_vfs_pci_address', 'driver', + 'pclass', 'pvendor', 'pdevice', + 'capabilities', + 'uuid', 'host_uuid', 'interface_uuid', + 'dpdksupport', + 'created_at', 'updated_at'] + labels = ['name', 'namedisplay', + 'type', 'pciaddr', 'dev_id', 'processor', + 'sriov_totalvfs', 'sriov_numvfs', + 'sriov_vfs_pci_address', 'driver', + 'pclass', 'pvendor', 'pdevice', + 'capabilities', + 'uuid', 'host_uuid', 'interface_uuid', + 'accelerated', + 'created_at', 'updated_at'] + data = [(f, getattr(port, f, '')) for f in fields] + utils.print_tuple_list(data, labels) + + +def _find_port(cc, host, portnameoruuid): + ports = cc.port.list(host.uuid) + for p in ports: + if p.name == portnameoruuid or p.uuid == portnameoruuid: + break + else: + raise exc.CommandError('Port not found: host %s port %s' % + (host.id, portnameoruuid)) + return p + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +@utils.arg('pnameoruuid', metavar='', + help="Name or UUID of port") +def do_host_port_show(cc, args): + """Show host port details.""" + host = host_utils._find_host(cc, args.hostnameorid) + port = _find_port(cc, host, args.pnameoruuid) + _print_port_show(port) + + +@utils.arg('hostnameorid', + metavar='', + help="Name or ID of host") +def do_host_port_list(cc, args): + """List host ports.""" + + from inventoryclient.common import wrapping_formatters + + terminal_width = utils.get_terminal_size()[0] + + host = host_utils._find_host(cc, args.hostnameorid) + + ports = cc.port.list(host.uuid) + + field_labels = ['uuid', 'name', 'type', 'pci address', 'device', + 'processor', 'accelerated', 'device type'] + fields = ['uuid', 'name', 'type', 'pciaddr', 'dev_id', 'numa_node', + 'dpdksupport', 'pdevice'] + + format_spec = \ + wrapping_formatters.build_best_guess_formatters_using_average_widths( + ports, fields, field_labels, no_wrap_fields=['pciaddr']) + # best-guess formatter does not make a good guess for + # proper width of pdevice until terminal is > 155 + # We override that width here. + pdevice_width = None + if terminal_width <= 130: + pdevice_width = .1 + elif 131 >= terminal_width <= 150: + pdevice_width = .13 + elif 151 >= terminal_width <= 155: + pdevice_width = .14 + + if pdevice_width and format_spec["pdevice"] > pdevice_width: + format_spec["pdevice"] = pdevice_width + + formatters = wrapping_formatters.build_wrapping_formatters( + ports, fields, field_labels, format_spec) + + utils.print_list( + ports, fields, field_labels, formatters=formatters, sortby=1) diff --git a/python-inventoryclient/inventoryclient/inventoryclient/v1/shell.py b/python-inventoryclient/inventoryclient/inventoryclient/v1/shell.py new file mode 100644 index 00000000..7f3a6dd0 --- /dev/null +++ b/python-inventoryclient/inventoryclient/inventoryclient/v1/shell.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from inventoryclient.common import utils +from inventoryclient.v1 import cpu_shell +from inventoryclient.v1 import ethernetport_shell +from inventoryclient.v1 import host_shell +from inventoryclient.v1 import lldp_agent_shell +from inventoryclient.v1 import lldp_neighbour_shell +from inventoryclient.v1 import memory_shell +from inventoryclient.v1 import node_shell +from inventoryclient.v1 import pci_device_shell +from inventoryclient.v1 import port_shell + + +COMMAND_MODULES = [ + cpu_shell, + ethernetport_shell, + host_shell, + lldp_agent_shell, + lldp_neighbour_shell, + memory_shell, + node_shell, + pci_device_shell, + port_shell, +] + + +def enhance_parser(parser, subparsers, cmd_mapper): + '''Take a basic (nonversioned) parser and enhance it with + commands and options specific for this version of API. + + :param parser: top level parser :param subparsers: top level + parser's subparsers collection where subcommands will go + ''' + for command_module in COMMAND_MODULES: + utils.define_commands_from_module(subparsers, command_module, + cmd_mapper) diff --git a/python-inventoryclient/inventoryclient/pylint.rc b/python-inventoryclient/inventoryclient/pylint.rc new file mode 100755 index 00000000..16a03be5 --- /dev/null +++ b/python-inventoryclient/inventoryclient/pylint.rc @@ -0,0 +1,218 @@ +[MASTER] +# Specify a configuration file. +rcfile=pylint.rc + +# Python code to execute, usually for sys.path manipulation such as pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not paths. +ignore=tests + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). +# https://pylint.readthedocs.io/en/latest/user_guide/output.html#source-code-analysis-section +# We are disabling (C)onvention +# We are disabling (R)efactor +# We are probably disabling (W)arning +# We are not disabling (F)atal, (E)rror +#disable=C,R,W +disable=C,R,W + + +[REPORTS] +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + + +[SIMILARITIES] +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[FORMAT] +# Maximum number of characters on a single line. +max-line-length=85 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 tab). +indent-string=' ' + + +[TYPECHECK] +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[BASIC] +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[MISCELLANEOUS] +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[IMPORTS] +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[CLASSES] +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + + +[EXCEPTIONS] +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/python-inventoryclient/inventoryclient/setup.cfg b/python-inventoryclient/inventoryclient/setup.cfg new file mode 100644 index 00000000..c522a032 --- /dev/null +++ b/python-inventoryclient/inventoryclient/setup.cfg @@ -0,0 +1,33 @@ +[metadata] +name = inventoryclient +summary = A python client library for Inventory +author = StarlingX +author-email = starlingx-discuss@lists.starlingx.io +home-page = http://www.starlingx.io/ +classifier = + Environment :: StarlingX + Intended Audience :: Information Technology + Intended Audience :: System Administrators + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[files] +packages = + inventoryclient + +[entry_points] +console_scripts = + inventory = inventoryclient.shell:main + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + diff --git a/python-inventoryclient/inventoryclient/setup.py b/python-inventoryclient/inventoryclient/setup.py new file mode 100644 index 00000000..ae3950a6 --- /dev/null +++ b/python-inventoryclient/inventoryclient/setup.py @@ -0,0 +1,30 @@ +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT + +import setuptools + +# In python < 2.7.4, a lazy loading of package `pbr` will break +# setuptools if some other modules registered functions in `atexit`. +# solution from: http://bugs.python.org/issue15881#msg170215 +try: + import multiprocessing # noqa +except ImportError: + pass + +setuptools.setup( + setup_requires=['pbr>=2.0.0'], + pbr=True) diff --git a/python-inventoryclient/inventoryclient/test-requirements.txt b/python-inventoryclient/inventoryclient/test-requirements.txt new file mode 100644 index 00000000..d58dff4b --- /dev/null +++ b/python-inventoryclient/inventoryclient/test-requirements.txt @@ -0,0 +1,23 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. + +# Hacking already pins down pep8, pyflakes and flake8 +hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 +bandit>=1.1.0 # Apache-2.0 +coverage!=4.4,>=4.0 # Apache-2.0 +fixtures>=3.0.0 # Apache-2.0/BSD +mock>=2.0 # BSD +mox +os-testr>=0.8.0 # Apache-2.0 +oslotest>=1.10.0 # Apache-2.0 +sphinx>=1.6.2 # BSD +testrepository>=0.0.18 # Apache-2.0/BSD +testscenarios>=0.4 # Apache-2.0/BSD +testtools>=1.4.0 # MIT +testresources>=0.2.4 # Apache-2.0/BSD +tempest>=16.1.0 # Apache-2.0 +httplib2 +python-keystoneclient +keyring +pyOpenSSL>=0.14 # Apache-2.0 diff --git a/python-inventoryclient/inventoryclient/tools/inventory.bash_completion b/python-inventoryclient/inventoryclient/tools/inventory.bash_completion new file mode 100644 index 00000000..3feda711 --- /dev/null +++ b/python-inventoryclient/inventoryclient/tools/inventory.bash_completion @@ -0,0 +1,33 @@ +# +# Copyright (c) 2018 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# bash completion for Titanium Cloud inventory commands + +_inventory_opts="" # lazy init +_inventory_flags="" # lazy init +_inventory_opts_exp="" # lazy init +_inventory() +{ + local cur prev kbc + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + + if [ "x$_inventory_opts" == "x" ] ; then + kbc="`inventory bash-completion | sed -e "s/ -h / /"`" + _inventory_opts="`echo "$kbc" | sed -e "s/--[a-z0-9_-]*//g" -e "s/[ ][ ]*/ /g"`" + _inventory_flags="`echo " $kbc" | sed -e "s/ [^-][^-][a-z0-9_-]*//g" -e "s/[ ][ ]*/ /g"`" + _inventory_opts_exp="`echo $_inventory_opts | sed -e "s/[ ]/|/g"`" + fi + + if [[ " ${COMP_WORDS[@]} " =~ " "($_inventory_opts_exp)" " && "$prev" != "help" ]] ; then + COMPREPLY=($(compgen -W "${_inventory_flags}" -- ${cur})) + else + COMPREPLY=($(compgen -W "${_inventory_opts}" -- ${cur})) + fi + return 0 +} +complete -F _inventory inventory diff --git a/python-inventoryclient/inventoryclient/tox.ini b/python-inventoryclient/inventoryclient/tox.ini new file mode 100644 index 00000000..3891da95 --- /dev/null +++ b/python-inventoryclient/inventoryclient/tox.ini @@ -0,0 +1,64 @@ +[tox] +envlist = py27,pep8,cover,pylint +minversion = 1.6 + +# tox does not work if the path to the workdir is too long, so move it to /tmp +toxworkdir = /tmp/{env:USER}_inventoryclienttox +cgcsdir = {toxinidir}/../../../.. +distshare={toxworkdir}/.tox/distshare + +[testenv] +setenv = VIRTUAL_ENV={envdir} + PYTHONWARNINGS=default::DeprecationWarning + OS_TEST_PATH=inventoryclient/tests + TESTR_START_DIR=inventoryclient/tests +basepython = python2.7 +usedevelop = True + +install_command = pip install -U -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/pike} {opts} {packages} +deps = -r{toxinidir}/test-requirements.txt + +commands = + find {toxinidir} -not -path '{toxinidir}/.tox/*' -name '*.py[c|o]' -delete + python setup.py testr --slowest --testr-args='{posargs}' + +whitelist_externals = + bash + find + rm +passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY + +[testenv:pep8] +commands = + flake8 inventoryclient + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +basepython = python2.7 +commands = + find . -type f -name ".coverage\.*" -delete + rm -f .coverage + rm -Rf cover + rm -f coverage.xml + python setup.py testr --coverage --testr-args='{posargs}' + coverage xml + +[flake8] +show-source = true +exclude=.*,dist,*lib/python*,*egg,build +max-complexity=25 +# H102 Apache 2.0 license header not found +# H233 Python 3.x incompatible use of print operator +# H404 multi line docstring should start without a leading new line +# H405 multi line docstring summary not separated with an empty line +ignore = H102,H233,H404,H405 + +[testenv:pylint] +basepython = python2.7 + +deps = {[testenv]deps} + pylint + +commands = pylint {posargs} inventoryclient --rcfile=./pylint.rc --extension-pkg-whitelist=lxml.etree,greenlet --ignored-classes=LookupDict diff --git a/tox.ini b/tox.ini index 5b11829f..3fcb63e6 100644 --- a/tox.ini +++ b/tox.ini @@ -25,7 +25,7 @@ commands = -not -name \*~ \ -not -name \*.md \ -name \*.sh \ - -print0 | xargs -0 bashate -v -i E010,E006" + -print0 | xargs -n 1 -0 bashate -v -i E010,E006" bash -c "find {toxinidir} \ \( -name middleware/io-monitor/recipes-common/io-monitor/io-monitor/io_monitor/test-tools/yaml/* -prune \) \ -o \( -name .tox -prune \) \ @@ -40,10 +40,17 @@ description = Run style checks commands = flake8 +# hacking can be added for additional pep8 once this passes for all stx-metal [flake8] show-source = True exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,release-tag-* +# H102 Apache License format +# H233 Python 3.x incompatible use of print operator +# H404 multi line docstring should start without a leading new line +# H405 multi line docstring summary not separated with an empty line +ignore = H102,H233,H404,H405 + [testenv:venv] commands = {posargs}