From 88d721402785c2baccbea679295f2cba220b0303 Mon Sep 17 00:00:00 2001 From: Ron Stone Date: Tue, 10 Sep 2024 17:15:33 +0000 Subject: [PATCH] Generate port referece from source - Tidy tox.ini Story: 2011201 Change-Id: I62ed4bd045ce49d53b863d63b3d1301fe9ab6b5b Signed-off-by: Ron Stone --- .gitignore | 2 +- _p_columns.py | 5 + doc/requirements.txt | 2 + .../distributed-cloud-ports-reference.rst | 248 +----------------- fetch-ports-files.sh | 6 + py_2_xlsx.py | 124 +++++++++ tox.ini | 7 + xlst_2_csv.py | 64 +++++ 8 files changed, 212 insertions(+), 246 deletions(-) create mode 100644 _p_columns.py create mode 100755 fetch-ports-files.sh create mode 100755 py_2_xlsx.py create mode 100755 xlst_2_csv.py diff --git a/.gitignore b/.gitignore index 301054d7e..45fc360fa 100644 --- a/.gitignore +++ b/.gitignore @@ -70,7 +70,7 @@ tmp/ # templates/events.yaml *-series-log-messages.rst *-series-alarm-messages.rst - +doc/source/dist_cloud/kubernetes/FW_PORTS.csv # API Reference Guide api-ref/build/ diff --git a/_p_columns.py b/_p_columns.py new file mode 100644 index 000000000..ff4e71cd9 --- /dev/null +++ b/_p_columns.py @@ -0,0 +1,5 @@ +columns = ["Source", "Protocol", "Port", "Desc", "Context", "Network", "Endpoints","Hosts", "Note", "HTTPS", "_stx", "_pl", "_os", "_an"] +src_index = columns.index("Source") +port_index = columns.index("Port") +net_index = columns.index("Network") +COL_COUNT = len(columns) \ No newline at end of file diff --git a/doc/requirements.txt b/doc/requirements.txt index ee75d7535..d06348e4b 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -4,6 +4,8 @@ openstackdocstheme>=2.2.1,<=2.3.1 # Apache-2.0 docutils==0.18.1 PyYAML==6.0 sphinx-tabs<=3.4.1 +pandas +openpyxl # API Reference Guide os-api-ref>=1.5.0 # Apache-2.0 diff --git a/doc/source/dist_cloud/kubernetes/distributed-cloud-ports-reference.rst b/doc/source/dist_cloud/kubernetes/distributed-cloud-ports-reference.rst index 2a0314611..9e474e397 100644 --- a/doc/source/dist_cloud/kubernetes/distributed-cloud-ports-reference.rst +++ b/doc/source/dist_cloud/kubernetes/distributed-cloud-ports-reference.rst @@ -13,251 +13,9 @@ function correctly. .. begin-dc-ports-table -.. table:: Table 1. |prod-dc| port requirements - :widths: auto - - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | Protocol | Port | Network | Description | System Controller| Subcloud | Initiator | Destination | Notes | - +==========+=======+=========+==================+==================+==================+==================================================+=====================================+=========================================+ - | tcp | 22 | oam | ssh | allowed | allowed | System Controller | Subclouds | For admin login | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 22 | oam | ssh | allowed | allowed | Subclouds | System Controller | For admin login | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 22 | mgmt | ssh | allowed | allowed | System Controller | Subclouds | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 22 | mgmt | ssh | allowed | allowed | Subclouds | System Controller | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 123 | oam | ntp | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 123 | mgmt | ntp | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 161 | oam | snmp | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 161 | mgmt | snmp | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 162 | oam | snmp trap | allowed | allowed | System Controller | Subclouds | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 162 | oam | snmp trap | allowed | allowed | Subclouds | System Controller | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 162 | mgmt | snmp trap | allowed | allowed | System Controller | Subclouds | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 162 | mgmt | snmp trap | allowed | allowed | Subclouds | System Controller | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 162 | oam | snmp trap | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 162 | mgmt | snmp trap | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 389 | oam | openLDAP | blocked(by gnp) | NA | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 389 | mgmt | openLDAP | allowed | NA | Subclouds | System Controller | LDAP service | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 636 | oam | openLDAP | blocked(by gnp) | NA | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 636 | mgmt | openLDAP | allowed | NA | Subclouds | System Controller | LDAP service, https enable | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 873 | oam | rsyncd | blocked(by gnp) | blocked(by gnp) | Not used between System Controller and Subclouds | | Used for synchronizing patches among | - | | | | | | | | | nodes | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 873 | mgmt | rsyncd | allowed | allowed | Not used between System Controller and Subclouds | | Used for synchronizing patches among | - | | | | | | | | | nodes | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp/udp | 2049 | oam | nfs | blocked (by gnp) | blocked (by gnp) | Not used between System Controller and Subclouds | | Used for sharing data among nodes | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp/udp | 2049 | mgmt | nfs | allowed | allowed | Not used between System Controller and Subclouds | | Used for sharing data among nodes | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 2222 | oam | sm | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 2222 | mgmt | sm | allowed | NA | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | udp | 2223 | oam | sm | allowed | NA | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 3300 | mgmt | ceph-mon | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 4545 | oam | stx-nfv | allowed(service public endpoint) | Not used between System Controller and Subclouds | | vim-restapi public endpoint | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 4545 | mgmt | stx-nfv | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | vim-restapi public endpoint | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 4546 | mgmt | stx-nfv | allowed(service admin endpoint) | System Controller | Subclouds |vim-restapi admin endpoint, https enabled| - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 4546 | mgmt | stx-nfv | allowed(service admin endpoint) | Subclouds | System Controller |vim-restapi admin endpoint, https enabled| - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5000 | oam | keystone-api | allowed(service public endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5000 | mgmt | keystone-api | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5001 | mgmt | keystone-api | allowed(service admin endpoint) | System Controller | Subclouds | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5001 | mgmt | keystone-api | allowed(service admin endpoint) | Subclouds | System Controller | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5432 | oam | postgres | blocked (by gnp) | blocked (by gnp) | Not used between System Controller and Subclouds | | postgres db serving port | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5432 | mgmt | postgres | allowed(serving port) | Not used between System Controller and Subclouds | | postgres db serving port | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5491 | oam | patching-api | blocked (by gnp) | blocked (by gnp) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5491 | mgmt | patching-api | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | patching-api internal endpoint | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5492 | mgmt | patching-api | allowed(service admin endpoint) | System Controller | Subclouds |patching-api admin endpoint,https enabled| - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 5492 | mgmt | patching-api | allowed(service admin endpoint) | Subclouds | System Controller |patching-api admin endpoint,https enabled| - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 15491 | oam | patching-api | allowed(service public endpoint) | Not used between System Controller and Subclouds | | patching-api public endpoint | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6385 | oam | sysinv-api | allowed(service public endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6385 | mgmt | sysinv-api | allowed(service public endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6386 | mgmt | sysinv-api | allowed(service public endpoint) | System Controller | Subclouds | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6386 | mgmt | sysinv-api | allowed(service public endpoint) | Subclouds | System Controller | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6443 | oam | K8s API server | allowed | allowed | Not used between System Controller and Subclouds | | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6443 | mgmt | K8s API server | allowed | allowed | Not used between System Controller and Subclouds | | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 6789 | mgmt | ceph-mon | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 6800 | mgmt | ceph-mgr | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 6801 | mgmt | ceph-mgr | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 6802 | mgmt | ceph-mds | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 6803 | mgmt | ceph-mds | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6804 | mgmt | ceph-mds | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 6805 | mgmt | ceph-mds | allowed | allowed | Not used between SystemController and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 7777 | oam | stx-ha (sm) | allowed(service public endpoint) | Not used between System Controller and Subclouds | | sm-api public endpoint | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 7777 | mgmt | stx-ha (sm) | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | sm-api public endpoint | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 7778 | mgmt | stx-ha (sm) | allowed(service admin endpoint) | Not used between System Controller and Subclouds | | sm-api admin endpoint, https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp6 | 7999 | mgmt | ceph-mgr | allowed | allowed | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8080 | oam | horizon http | allowed | blocked(by gnp) | Not used between System Controller and Subclouds | | Not required if using https | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8080 | mgmt | horizon http | allowed | allowed | System Controller | Subclouds | Not required if using https | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8080 | mgmt | horizon http | allowed | allowed | Subclouds | System Controller | Not required if using https | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8119 | oam | stx-distcloud | allowed(service | NA | Not used between System Controller and Subclouds | | dcmanager-api | - | | | | | public endpoint) | | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8119 | mgmt | stx-distcloud | allowed(service | NA | Not used between System Controller and Subclouds | | dcmanager-api | - | | | | | public endpoint) | | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8120 | mgmt | stx-distcloud | allowed(service | NA | Not used between System Controller and Subclouds | | dcmanager-api, https enabled | - | | | | | public endpoint) | | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8219 | mgmt | dcdbsync-api | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8220 | mgmt | dcdbsync-api | allowed(service admin endpoint) | System Controller | Subclouds | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8220 | mgmt | dcdbsync-api | allowed(service admin endpoint) | Subclouds | System Controller | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8443 | oam | horizon https | allowed | blocked(by gnp) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8443 | mgmt | horizon https | allowed | allowed | System Controller | Subclouds | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 8443 | mgmt | horizon https | allowed | allowed | Subclouds | System Controller | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9001 | oam | Docker registry | allowed(serving port) | System Controller | Subclouds | https enabled | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9001 | oam | Docker registry | allowed(serving port) | Subclouds | System Controller | https enabled | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9001 | mgmt | Docker registry | allowed(serving port) | System Controller | Subclouds | https enabled | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9001 | mgmt | Docker registry | allowed(serving port) | Subclouds | System Controller | https enabled | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9002 | oam | Registry token | allowed(serving port) | System Controller | Subclouds | https enabled | - | | | | server | | | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9002 | oam | Registry token | allowed(serving port) | Subclouds | System Controller | https enabled | - | | | | server | | | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9002 | mgmt | Registry token | allowed(serving port) | System Controller | Subclouds | https enabled | - | | | | server | | | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9002 | mgmt | Registry token | allowed(serving port) | Subclouds | System Controller | https enabled | - | | | | server | | | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9311 | oam | barbican-api | allowed(service public endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9311 | mgmt | barbican-api | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9312 | mgmt | barbican-api | allowed(service admin endpoint) | System Controller |Subclouds | https enabled | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 9312 | mgmt | barbican-api | allowed(service admin endpoint) | Subclouds |System Controller | https enabled | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 11211 | mgmt | memcached | allowed(keystone cache backend) | Not used between System Controller and Subclouds | | keystone cache backend | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 18002 | oam | stx-fault | allowed(service public endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 18002 | mgmt | stx-fault | allowed(service internal endpoint) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 18003 | mgmt | stx-fault | allowed(service admin endpoint) | System Controller | Subclouds | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 18003 | mgmt | stx-fault | allowed(service admin endpoint) | Subclouds | System Controller | https enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | icmp | NA | oam | icmp | allowed | allowed | Not used between System Controller and Subclouds | | | - | | | | | | | | | | - | | | | | | | **The only exception is when using ICMP during | | | - | | | | | | | subcloud installs**. | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | icmp | NA | mgmt | icmp | allowed | allowed | Not used between System Controller and Subclouds | | | - | | | | | | | | | | - | | | | | | | **The only exception is when using ICMP during | | | - | | | | | | | subcloud installs**. | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 25491 | oam | dcorch-patch | allowed (service | NA | Not used between System Controller and Subclouds | | dcorch-patch-api-proxy public endpoint | - | | | | -api-proxy | public endpoint) | | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 25491 | mgmt | dcorch-patch |allowed(service | NA | Not used between System Controller and Subclouds | | dcorch-patch-api-proxy internal endpoint| - | | | | -api-proxy |internal endpoint)| | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 25492 | mgmt | dcorch-patch | allowed(service | NA | Not used between System Controller and Subclouds | | dcorch-patch-api-proxy admin endpoint | - | | | | -api-proxy | admin endpoint) | | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 30001-| mgmt | VIM | allowed | allowed | Not used between System Controller and Subclouds | | | - | | 30004 | | | | | | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 30555 | oam | OIDC Client | blocked(by gnp) | Not used between System Controller and Subclouds | | Only when OIDC app is applied | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 30555 | mgmt | OIDC Client | allowed(serving port) | Not used between System Controller and Subclouds | | Only when OIDC app is applied | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 30556 | oam | DEX OIDC Provider| blocked(by gnp) | Not used between System Controller and Subclouds | | Only when OIDC app is applied | - +----------+-------+---------+------------------+-------------------------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 30556 | mgmt | DEX OIDC Provider| allowed(serving port) | Not used between System Controller and Subclouds | | Only when OIDC app is applied | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 31001 | oam | Elastic Dashboard| allowed(NodePort)| NA | System Controller | Subclouds | Only when Analytics is applied, https | - | | | | and API | | | | | enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 31001 | oam | Elastic Dashboard| allowed(NodePort)| NA | Subclouds | System Controller | Only when Analytics is applied, https | - | | | | and API | | | | | enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 31001 | mgmt | Elastic Dashboard| allowed(NodePort)| NA | System Controller | Subclouds | Only when Analytics is applied, https | - | | | | and API | | | | | enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 31001 | mgmt | Elastic Dashboard| allowed(NodePort)| NA | Subclouds | System Controller | Only when Analytics is applied, https | - | | | | and API | | | | | enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 31090-| oam | Kafka Brokers | allowed(NodePort)| NA | Not used between System Controller and Subclouds | | Only when Analytics is applied, https | - | | 31099 | | (NodePort) | | | | | enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 31090-| mgmt | Kafka Brokers | allowed(NodePort)| NA | Subclouds | System Controller | Only when Analytics is applied, https | - | | 31099 | | (NodePort) | | | | | enabled | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 32000 | oam | Kubernetes | allowed(NodePort)| allowed | Not used between System Controller and Subclouds | | Only when Kubernetes Dashboard | - | | | | dashboard | | | | | is installed | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 32000 | mgmt | Kubernetes | allowed(NodePort)| allowed | Not used between System Controller and Subclouds | | Only when Kubernetes Dashboard | - | | | | dashboard | | | | | is installed | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ - | tcp | 32323 | oam | vim-webserver | blocked(by gnp) | blocked(by gnp) | Not used between System Controller and Subclouds | | | - +----------+-------+---------+------------------+------------------+------------------+--------------------------------------------------+-------------------------------------+-----------------------------------------+ +.. csv-table:: Table 1. |prod-dc| port requirements + :file: /dist_cloud/kubernetes/FW_PORTS.csv + :header-rows: 1 .. end-dc-ports-table diff --git a/fetch-ports-files.sh b/fetch-ports-files.sh new file mode 100755 index 000000000..09d3c3030 --- /dev/null +++ b/fetch-ports-files.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# HTML + +curl https://opendev.org/starlingx/config/raw/branch/master/sysinv/sysinv/sysinv/sysinv/common/platform_firewall.py --create-dirs -o tmp/platform_firewall.py +curl https://opendev.org/starlingx/config/raw/branch/master/sysinv/sysinv/sysinv/sysinv/common/constants.py --create-dirs -o tmp/constants.py diff --git a/py_2_xlsx.py b/py_2_xlsx.py new file mode 100755 index 000000000..4f2eb8d7f --- /dev/null +++ b/py_2_xlsx.py @@ -0,0 +1,124 @@ +import re +import os +import sys +import pandas as pd + +from _p_columns import columns, port_index, src_index, net_index + +df = pd.DataFrame(columns=columns) + +def convert_to_uppercase(input_string): + return input_string.upper() + +# Look up a port number assigned to a constant in another file +def find_port_number(filename, search_string): + found_port = None + with open(filename, 'r') as file: + for line in file: + match = re.search(rf'{search_string}\s*=\s*(\d+)', line) + if match: + found_port = int(match.group(1)) + break + return found_port + +def remove_prefix(input_string): + # Find the index of the first period + period_index = input_string.find('.') + + if period_index != -1: + return input_string[period_index + 1:] + else: + # Return the original string + return input_string + +def delete_file(file_path): + try: + # Check if the file exists + if os.path.exists(file_path): + # Delete the file + os.remove(file_path) + print(f"File '{file_path}' deleted successfully.") + else: + print(f"File '{file_path}' does not exist.") + except Exception as e: + print(f"An error occurred: {e}") + +def is_numeric(array, index): + array = [element.strip() for element in array] + # Check if the array has an integer at the element to be tested + if len(array) > index: + return array[index].isnumeric() + else: + return False + +def prepend_string(main_string, prepend_string): + return prepend_string + main_string + +def append_string(*args, **kwargs): + return prepend_string(*args, **kwargs) + +def extract_docu_comments(input_file, out_file): + sect = "N/A" + prot = "N/A" + with open(input_file, 'r') as file: + lines = file.readlines() + + for line in lines: + + match = re.search(r'^(\S+)\s*=\s*(\{|\\)', line) + if match: + sect = match.group(1).strip() + sect = append_string(',', sect) + prot = "N/A, " + match = re.search(r'("tcp":|"udp":)', line) + if match: + prot = match.group(1).strip() + prot = prot.replace(':', '').strip() + prot = convert_to_uppercase(prot) + prot = append_string(',', prot) + + # Check if the line contains a comment starting with 'docu' followed by + # a colon + if '#' in line and 'docu:' in line.lstrip(): + docu_line = re.sub(r',?\s*#\s*(noqa: E501)?\s+docu:\s*', ',', line).strip() + docu_line = docu_line.replace(':', ',').strip() + docu_line = prepend_string(docu_line, prot) + docu_line = prepend_string(docu_line, sect) + docu_line = docu_line.replace('"', '').strip() + column_values = docu_line.split(',') + + if not is_numeric(column_values, port_index): + const = column_values[port_index] + column_values[port_index] = find_port_number(const_file, remove_prefix(column_values[port_index])) + print("Replaced " + const.strip() + " with " + str(column_values[port_index])) + + if 'OAM' in column_values[src_index]: + column_values[net_index] = 'oam' + + # print("Processing: " + line) + df.loc[len(df)] = column_values + + ports_column_name = df.columns[port_index] + df[ports_column_name] = pd.to_numeric(df[ports_column_name], errors='coerce') + df.to_excel(excel_file, index=False) + + print(f"Ports list successfully extracted to '{excel_file}'.") + +if len(sys.argv) != 4: + print(f"""\ +This script reads a python file to create an Excel sheet of firewall +port definitions. + +Usage: {os.path.basename(__file__)} +Example: python ./py_2_xlsx.py platform_firewall.py constants.py FW_PORTS.xlsx +""") + sys.exit(1) + + +input_file = str(sys.argv[1]) +const_file = str(sys.argv[2]) +excel_file = str(sys.argv[3]) + +# Extract lines with docu comments +extract_docu_comments(input_file, excel_file) + diff --git a/tox.ini b/tox.ini index 2c6a9f825..4d1dc07c2 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,9 @@ commands = python parser.py -l templates/logs_template.rst -e tmp/events.yaml -s 100,200,300,400,500,700,800,900 -ts = -type Log -outputPath doc/source/fault-mgmt/openstack/ -sort Yes -product openstack -replace "|,OR" bash ./normalize-includes.sh bash ./dup-abbr-check.sh + bash ./fetch-ports-files.sh + python py_2_xlsx.py tmp/platform_firewall.py tmp/constants.py tmp/FW_PORTS.xlsx + python xlst_2_csv.py tmp/FW_PORTS.xlsx doc/source/dist_cloud/kubernetes/FW_PORTS.csv --columns Source Port Protocol Network Desc HTTPS Note _stx --sort_orders Port=asc --filters _stx=y [testenv:postbuild-docs] commands = @@ -32,6 +35,7 @@ commands = bash hide-empty-rows.sh doc/build/html bash htmlChecks.sh doc/build/html + [testenv:docs] deps = # -c{env:TOX_CONSTRAINTS_FILE:doc/upper-constraints.txt} @@ -45,6 +49,9 @@ allowlist_externals = bash ./hide-empty-rows.sh ./htmlChecks.sh ./get-remote-files.sh + ./fetch-ports-files.sh + ./py_2_xlsx.py + ./xlst_2_csv.py git # hw-updates.sh diff --git a/xlst_2_csv.py b/xlst_2_csv.py new file mode 100755 index 000000000..e1cfc9d1d --- /dev/null +++ b/xlst_2_csv.py @@ -0,0 +1,64 @@ +import pandas as pd +import argparse +import re + +from _p_columns import columns, port_index + +def export_to_csv(input_file, output_file, columns, filters, sort_orders): + # Load the Excel file + df = pd.read_excel(input_file) + + # Filter columns + df = df[columns] + + # Apply filters + for column, value in filters.items(): + if isinstance(value, list): + df = df[df[column].isin(value)] + else: + df = df[df[column] == value] + + # Apply sort orders + sort_columns = [col for col, order in sort_orders.items()] + sort_ascending = [order == 'asc' for order in sort_orders.values()] + df = df.sort_values(by=sort_columns, ascending=sort_ascending) + + # Drop filter-only columns that begin with an underscore + pattern = re.compile("^_[a-z]+$") + for c in columns: + if pattern.match(c): + df.pop(c) + + # Export to CSV + df.to_csv(output_file, index=False) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Export a CSV list of ports from Excel with specified columns, filters, and sort orders.") + parser.add_argument("input_file", help="Path to the input Excel file. Positioned BEFORE options.") + parser.add_argument("output_file", help="Path to the output CSV file. Positioned BEFORE options.") + parser.add_argument("--columns", nargs='+', required=True, help="Space separated list of columns to include in the CSV file") + parser.add_argument("--filters", nargs='*', required=True, action='append', help="Column filters in the format column=value or column=[value1,value2,...]") + parser.add_argument("--sort_orders", nargs='*', required=True, action='append', help="Sort orders in the format column=asc/desc") + + args = parser.parse_args() + + # Process filters argument + filters = {} + for filt in args.filters: + for f in filt: + column, value = f.split('=') + if value.startswith('[') and value.endswith(']'): + value = value.strip('[]').split(',') + filters[column] = value + + # Process sort orders argument + sort_orders = {} + for sort in args.sort_orders: + for s in sort: + column, order = s.split('=') + sort_orders[column] = order + + export_to_csv(args.input_file, args.output_file, args.columns, filters, sort_orders) + +# Note that positional args are first. Hidden filter columns must be listed in --columns +# e.g: python3.10 xlst_2_csv.py FW_PORTS.xlsx FW_PORTS.csv --columns Source Port Protocol Network Desc HTTPS Note _pl --sort_orders Port=asc --filters _pl=y