starlingx/test
Code Issues Proposed changes

Merge "Support for REST APIs going through a JumpHost"

Browse Source
This commit is contained in:
Zuul
2025-11-14 20:03:49 +00:00
committed by Gerrit Code Review
parent 494d988ad6 f811ace75a
commit f1a8161256
7 changed files with 987 additions and 587 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Pipfile
View File

@@ -12,26 +12,23 @@ pre-commit = "==3.7.0"
black = "==24.3.0"
isort = "==5.13.2"
flake8 = "==7.0.0"
# Docstring Compliance and Enforcement
pydocstyle = "==6.3.0" # Enforce PEP 257 and Google-style docstrings
pydoclint = "==0.4.1" # Validate function signatures match docstrings (Google-style)
interrogate = "==1.5.0" # Ensure all functions and classes have docstrings
# Tools Packages
pytest = "==8.1.1"
paramiko = "==3.4.0"
sshtunnel = "==0.4.0"
json5 = "==0.9.24"
selenium = "==4.20.0"
django = "==5.0.6"
psycopg2-binary = "==2.9.9"
jinja2 = "*"
requests = "*"
# Documentation Tools
sphinx = "==8.1.3"
sphinx-autobuild = "==2024.2.4"
openstackdocstheme = "==3.4.1"
reno = "==4.1.0"
redfish = "*"

1176
Pipfile.lock generated
View File

File diff suppressed because it is too large Load Diff

138
config/rest_api/objects/rest_api_config.py
View File

@@ -14,18 +14,43 @@ class RestAPIConfig:
raise
rest_dict = json5.load(json_data)
self.keystone_base = rest_dict['keystone_base']
self.bare_metal_base = rest_dict['bare_metal_base']
self.configuration_base = rest_dict['configuration_base']
self.fm_base = rest_dict['fm_base']
self.dc_base = rest_dict['dc_base']
self.node_interface_metrics_exporter_base = rest_dict['node_interface_metrics_exporter_base']
self.nfv_base = rest_dict['nfv_base']
self.barbican_base = rest_dict['barbican_base']
self.software_update_base = rest_dict['software_update_base']
self.usm_base = rest_dict['usm_base']
self.vim_base = rest_dict['vim_base']
self.high_availability_base = rest_dict['high_availability_base']
self.all_base_urls = []
self.keystone_base = rest_dict["keystone_base"]
self.all_base_urls.append(self.keystone_base)
self.bare_metal_base = rest_dict["bare_metal_base"]
self.all_base_urls.append(self.bare_metal_base)
self.configuration_base = rest_dict["configuration_base"]
self.all_base_urls.append(self.configuration_base)
self.fm_base = rest_dict["fm_base"]
self.all_base_urls.append(self.fm_base)
self.dc_base = rest_dict["dc_base"]
self.all_base_urls.append(self.dc_base)
self.node_interface_metrics_exporter_base = rest_dict["node_interface_metrics_exporter_base"]
self.all_base_urls.append(self.node_interface_metrics_exporter_base)
self.nfv_base = rest_dict["nfv_base"]
self.all_base_urls.append(self.nfv_base)
self.barbican_base = rest_dict["barbican_base"]
self.all_base_urls.append(self.barbican_base)
self.software_update_base = rest_dict["software_update_base"]
self.all_base_urls.append(self.software_update_base)
self.usm_base = rest_dict["usm_base"]
self.all_base_urls.append(self.usm_base)
self.vim_base = rest_dict["vim_base"]
self.all_base_urls.append(self.vim_base)
self.high_availability_base = rest_dict["high_availability_base"]
self.all_base_urls.append(self.high_availability_base)
def get_keystone_base(self) -> str:
"""
@@ -35,89 +60,120 @@ class RestAPIConfig:
def get_bare_metal_base(self) -> str:
"""
Getter for bare_metal_base
Returns: the bare_metal_base
Getter for bare_metal_base.
Returns:
str: the bare_metal_base
"""
return self.bare_metal_base
def get_configuration_base(self) -> str:
"""
Getter for configuration_base
Returns: the configuration_base
Getter for configuration_base.
Returns:
str: the configuration_base
"""
return self.configuration_base
def get_fm_base(self) -> str:
"""
Getter for fm_base
Returns:
Getter for fm_base.
Returns:
str: the fm_base
"""
return self.fm_base
def get_dc_base(self) -> str:
"""
Getter for dc_base
Returns:
Getter for dc_base.
Returns:
str: the dc_base
"""
return self.dc_base
def get_node_interface_metrics_exporter_base(self) -> str:
"""
Getter for node_interface_metrics_exporter_base
Returns:
Getter for node_interface_metrics_exporter_base.
Returns:
str: the node_interface_metrics_exporter_base
"""
return self.node_interface_metrics_exporter_base
def get_nfv_base(self) -> str:
"""
Getter for nfv_base
Returns:
Getter for nfv_base.
Returns:
str: the nfv_base
"""
return self.nfv_base
def get_barbican_base(self) -> str:
"""
Getter for barbican_base
Returns: the barbican_base
Getter for barbican_base.
Returns:
str: the barbican_base
"""
return self.barbican_base
def get_software_update_base(self) -> str:
"""
Getter for software_update_base
Returns:
Getter for software_update_base.
Returns:
str: the software_update_base
"""
return self.software_update_base
def get_usm_base(self) -> str:
"""
Getter for usm_base
Returns:
Getter for usm_base.
Returns:
str: the usm_base
"""
return self.usm_base
def get_vim_base(self) -> str:
"""
Getter for vim_base
Returns:
Getter for vim_base.
Returns:
str: the vim_base
"""
return self.vim_base
def get_high_availability_base(self) -> str:
"""
Getter for high_availability_base
Returns:
Getter for high_availability_base.
Returns:
str: the high_availability_base
"""
return self.high_availability_base
def get_all_ports(self) -> list[int]:
"""
Extract all port numbers from the REST API configuration
Returns:
list[int]: List of unique port numbers from all API endpoints
"""
ports = set()
# Extract port from each base URL
for base_url in self.all_base_urls:
# Extract port number (everything before the first '/')
port_part = base_url.split("/")[0]
try:
ports.add(int(port_part))
except ValueError:
# Skip if not a valid port number
continue
return sorted(list(ports))

149
framework/rest/ssh_tunnel_rest_client.py Normal file
View File

@@ -0,0 +1,149 @@
import socket
from typing import Any, Optional
from urllib.parse import urlparse
import requests
from sshtunnel import SSHTunnelForwarder
from urllib3.exceptions import InsecureRequestWarning
from config.configuration_manager import ConfigurationManager
from framework.rest.rest_response import RestResponse
class SSHTunnelRestClient:
"""
REST client that makes HTTP requests through SSH tunnel when jump host is configured
"""
def __init__(self):
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
self._tunnels = {}
self._port_mappings = {}
self._setup_tunnels()
def _setup_tunnels(self):
"""Setup SSH tunnels for common API ports if jump host is configured"""
config = ConfigurationManager.get_lab_config()
if not config.is_use_jump_server():
return
# Get jump host configurations
jump_host_config = ConfigurationManager.get_lab_config().get_jump_host_configuration()
lab_ip = config.get_floating_ip()
# Get API ports from configuration
rest_api_config = ConfigurationManager.get_rest_api_config()
api_ports = rest_api_config.get_all_ports()
for remote_port in api_ports:
local_port = self._find_free_port()
# Create SSH tunnel for this port
tunnel = SSHTunnelForwarder((jump_host_config.get_host(), jump_host_config.get_ssh_port()), ssh_username=jump_host_config.get_credentials().get_user_name(), ssh_password=jump_host_config.get_credentials().get_password(), remote_bind_address=(lab_ip, remote_port), local_bind_address=("127.0.0.1", local_port))
# Start the tunnel
tunnel.start()
# Store tunnel and port mapping
self._tunnels[remote_port] = tunnel
self._port_mappings[remote_port] = local_port
def _find_free_port(self):
"""Find an available local port"""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("", 0))
s.listen(1)
port = s.getsockname()[1]
return port
def _modify_url_for_tunnel(self, url):
"""Modify URL to use local tunnel if configured"""
if not self._tunnels:
return url
parsed = urlparse(url)
# Extract the port from the URL
if ":" in parsed.netloc:
host, port_str = parsed.netloc.rsplit(":", 1)
try:
remote_port = int(port_str)
except ValueError:
return url
else:
# Default HTTPS port
remote_port = 443
# Check if we have a tunnel for this port
if remote_port in self._port_mappings:
local_port = self._port_mappings[remote_port]
modified_url = url.replace(f"{parsed.netloc}", f"127.0.0.1:{local_port}")
return modified_url
return url
def _normalize_headers(self, headers):
"""Convert headers to dictionary format"""
if not headers:
return {}
if isinstance(headers, dict):
return headers
if isinstance(headers, list):
headers_dict = {}
for header in headers:
if isinstance(header, dict):
headers_dict.update(header)
return headers_dict
return {}
def get(self, url: str, headers: Optional[Any] = None) -> RestResponse:
"""
Runs a get request with the given url and headers, tunneling through SSH if configured
Args:
url (str): The URL for the request.
headers (Optional[Any]): Headers for the request (dict or list of dicts). Defaults to None.
Returns:
RestResponse: An object representing the response of the GET request.
"""
# Convert headers to dict format
headers_dict = self._normalize_headers(headers)
# Modify URL for tunnel if needed
modified_url = self._modify_url_for_tunnel(url)
response = requests.get(modified_url, headers=headers_dict, verify=False)
return RestResponse(response)
def post(self, url: str, data: Any, headers: Any) -> RestResponse:
"""
Runs a post request with the given url and headers, tunneling through SSH if configured
Args:
url (str): The URL for the request.
data (Any): The data to be sent in the body of the request.
headers (Any): Headers for the request (dict or list of dicts).
Returns:
RestResponse: An object containing the response from the request.
"""
# Convert headers to dict format
headers_dict = self._normalize_headers(headers)
# Modify URL for tunnel if needed
modified_url = self._modify_url_for_tunnel(url)
response = requests.post(modified_url, headers=headers_dict, data=data, verify=False)
return RestResponse(response)
def close(self):
"""Close all SSH tunnels if they exist"""
for tunnel in self._tunnels.values():
tunnel.stop()
self._tunnels.clear()
self._port_mappings.clear()

31
keywords/cloud_platform/rest/cloud_rest_client.py
View File

@@ -1,4 +1,7 @@
from config.configuration_manager import ConfigurationManager
from framework.rest.rest_client import RestClient
from framework.rest.rest_response import RestResponse
from framework.rest.ssh_tunnel_rest_client import SSHTunnelRestClient
from keywords.cloud_platform.rest.get_auth_token_keywords import GetAuthTokenKeywords
@@ -6,21 +9,27 @@ class CloudRestClient:
"""
Class for Cloud Rest Client.
"""
def __init__(self):
self.auth_token = GetAuthTokenKeywords().get_token()
# Use SSH tunnel client if jump host is configured
config = ConfigurationManager.get_lab_config()
if config.is_use_jump_server():
self.rest_client = SSHTunnelRestClient()
else:
self.rest_client = RestClient()
def get(self, url: str):
self.auth_token = GetAuthTokenKeywords(self.rest_client).get_token()
def get(self, url: str) -> RestResponse:
"""
Runs a get on the url
Runs a get on the url.
Args:
url: the url for the get request
url (str): the url for the get request
Returns: the response
Returns:
RestResponse: The response from the GET request
"""
headers = {'X-Auth-Token': self.auth_token}
rest_response = RestClient().get(url, headers)
headers = {"X-Auth-Token": self.auth_token}
rest_response = self.rest_client.get(url, headers)
return rest_response

71
keywords/cloud_platform/rest/get_auth_token_keywords.py
View File

@@ -1,48 +1,61 @@
from typing import Any, Dict, Optional
from config.configuration_manager import ConfigurationManager
from framework.logging.automation_logger import get_logger
from framework.rest.rest_client import RestClient
from keywords.base_keyword import BaseKeyword
from keywords.cloud_platform.rest.get_rest_url_keywords import GetRestUrlKeywords
from config.configuration_manager import ConfigurationManager
class GetAuthTokenKeywords(BaseKeyword):
"""
Class for Auth Token Keywords
Keywords for obtaining authentication tokens from Keystone API.
This class handles the authentication process with OpenStack Keystone
to retrieve X-Auth-Token for subsequent API calls.
"""
def __init__(self) -> None:
# set the headers
self.headers = {'Content-type': 'application/json'}
def __init__(self, rest_client: RestClient) -> None:
"""
Initialize the authentication token keywords.
Args:
rest_client (RestClient): REST client instance for making HTTP requests
"""
self.headers: Dict[str, str] = {"Content-type": "application/json"}
self.rest_client: RestClient = rest_client
# Get the rest credentials
rest_credentials = ConfigurationManager.get_lab_config().get_rest_credentials()
# reset body needed to get auth token
self.json_string = (
'{"auth":'
'{"identity":{"methods": ["password"],'
'"password": {"user": {"domain":'
'{"name": "Default"},"name":'
f'"{rest_credentials.get_user_name()}","password":"{rest_credentials.get_password()}"'
'}}},'
'"scope":{"project": {"name":'
'"admin","domain": {"name":"Default"}'
'}}}}'
)
# JSON body needed to get auth token
self.json_string: str = '{"auth":' '{"identity":{"methods": ["password"],' '"password": {"user": {"domain":' '{"name": "Default"},"name":' f'"{rest_credentials.get_user_name()}","password":"{rest_credentials.get_password()}"' "}}}," '"scope":{"project": {"name":' '"admin","domain": {"name":"Default"}' "}}}}"
def get_token(self):
def get_token(self) -> Optional[str]:
"""
Gets the token for a rest api call
Retrieve authentication token from Keystone API.
Makes a POST request to the Keystone /auth/tokens endpoint with
admin credentials to obtain an X-Subject-Token for API authentication.
Returns:
Optional[str]: Authentication token string if successful, None if failed
Raises:
None: May log errors but does not raise exceptions
"""
# Get the token from Keystone
response = self.rest_client.post(f"{GetRestUrlKeywords().get_keystone_url()}/auth/tokens", headers=self.headers, data=self.json_string)
# gets the token
response = RestClient().post(f"{GetRestUrlKeywords().get_keystone_url()}/auth/tokens", headers=self.headers, data=self.json_string)
# Check if authentication was successful
if response.get_status_code() != 201:
get_logger().log_error(f"Authentication failed with status {response.get_status_code()}: {response.response.text}")
return None
# token is in the header of the response
headers = response.get_headers()
token = headers['X-Subject-Token']
if token:
return token
get_logger().log_error("unable to find the token")
return None
# Token is in the response headers
headers: Dict[str, Any] = response.get_headers()
if "X-Subject-Token" in headers:
return headers["X-Subject-Token"]
get_logger().log_error("Unable to find X-Subject-Token in response headers")
return None

4
unit_tests/config/usm/usm_config_test.py
View File

@@ -13,7 +13,7 @@ def test_default_usm_config():
configuration_manager.load_configs(config_file_locations)
default_config = configuration_manager.get_usm_config()
assert default_config is not None, "Default usm config wasn't loaded successfully"
assert default_config.get_iso_path() == "/home/sysadmin/usm_test/starlingx-10.0.0.iso", "ISO path was incorrect"
assert default_config.get_iso_path() == "/opt/software/starlingx.iso", "ISO path was incorrect"
def test_custom_usm_config():
@@ -28,4 +28,4 @@ def test_custom_usm_config():
custom_config = configuration_manager.get_usm_config()
assert custom_config is not None, "Custom usm config wasn't loaded successfully"
assert custom_config.get_iso_path() == "/home/sysadmin/usm_test/starlingx-10.0.0.iso", "ISO path was incorrect"
assert custom_config.get_iso_path() == "/opt/software/starlingx.iso", "ISO path was incorrect"