Merge "[k8s] Upgrade k8s dashboard version to v2.0.0"
This commit is contained in:
commit
5ada350502
@ -359,6 +359,10 @@ the table are linked to more details elsewhere in the user guide.
|
||||
| `kube_dashboard_enabled`_ | - true | true |
|
||||
| | - false | |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `kube_dashboard_version`_ | see below | see below |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `metrics_scraper_tag`_ | see below | see below |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
| `influx_grafana_dashboard_enabled`_ | - true | false |
|
||||
| | - false | |
|
||||
+---------------------------------------+--------------------+---------------+
|
||||
@ -1578,6 +1582,15 @@ _`containerd_tarball_sha256`
|
||||
sha256 of the tarball fetched with containerd_tarball_url or from
|
||||
https://storage.googleapis.com/cri-containerd-release/.
|
||||
|
||||
_`kube_dashboard_version`
|
||||
Default version of Kubernetes dashboard.
|
||||
Train default: v1.8.3
|
||||
Ussuri default: v2.0.0
|
||||
|
||||
_`metrics_scraper_tag`
|
||||
The version of metrics-scraper used by kubernetes dashboard.
|
||||
Ussuri default: v1.0.4
|
||||
|
||||
External load balancer for services
|
||||
-----------------------------------
|
||||
|
||||
|
@ -12,8 +12,9 @@ do
|
||||
done
|
||||
|
||||
if [ "$(echo $KUBE_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ]; then
|
||||
KUBE_DASH_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}kubernetes-dashboard-${ARCH}:${KUBE_DASHBOARD_VERSION}"
|
||||
KUBE_DASH_IMAGE="${CONTAINER_INFRA_PREFIX:-kubernetesui/}dashboard:${KUBE_DASHBOARD_VERSION}"
|
||||
HEAPSTER_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}heapster-${ARCH}:v1.4.2"
|
||||
METRICS_SCRAPER_IMAGE="${CONTAINER_INFRA_PREFIX:-kubernetesui/}metrics-scraper:${METRICS_SCRAPER_TAG}"
|
||||
|
||||
KUBE_DASH_DEPLOY=/srv/magnum/kubernetes/kubernetes-dashboard.yaml
|
||||
|
||||
@ -35,12 +36,33 @@ if [ "$(echo $KUBE_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "true" ];
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Configuration to deploy release version of the Dashboard UI compatible with
|
||||
# Kubernetes 1.8.
|
||||
#
|
||||
# Example usage: kubectl create -f <this_file>
|
||||
---
|
||||
|
||||
# ------------------- Dashboard Secret ------------------- #
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
|
||||
---
|
||||
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
spec:
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 8443
|
||||
selector:
|
||||
k8s-app: kubernetes-dashboard
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
@ -52,70 +74,117 @@ metadata:
|
||||
type: Opaque
|
||||
|
||||
---
|
||||
# ------------------- Dashboard Service Account ------------------- #
|
||||
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
kind: Secret
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard-csrf
|
||||
namespace: kube-system
|
||||
type: Opaque
|
||||
data:
|
||||
csrf: ""
|
||||
|
||||
---
|
||||
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard-key-holder
|
||||
namespace: kube-system
|
||||
type: Opaque
|
||||
|
||||
---
|
||||
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard-settings
|
||||
namespace: kube-system
|
||||
|
||||
---
|
||||
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
rules:
|
||||
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
|
||||
verbs: ["get", "update", "delete"]
|
||||
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
resourceNames: ["kubernetes-dashboard-settings"]
|
||||
verbs: ["get", "update"]
|
||||
# Allow Dashboard to get metrics.
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
resourceNames: ["heapster", "dashboard-metrics-scraper"]
|
||||
verbs: ["proxy"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services/proxy"]
|
||||
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
|
||||
verbs: ["get"]
|
||||
|
||||
---
|
||||
# ------------------- Dashboard Role & Role Binding ------------------- #
|
||||
|
||||
kind: Role
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: kubernetes-dashboard-minimal
|
||||
namespace: kube-system
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard
|
||||
rules:
|
||||
# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["create"]
|
||||
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs"]
|
||||
verbs: ["get", "update", "delete"]
|
||||
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
resourceNames: ["kubernetes-dashboard-settings"]
|
||||
verbs: ["get", "update"]
|
||||
# Allow Dashboard to get metrics from heapster.
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
resourceNames: ["heapster"]
|
||||
verbs: ["proxy"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services/proxy"]
|
||||
resourceNames: ["heapster", "http:heapster:", "https:heapster:"]
|
||||
verbs: ["get"]
|
||||
# Allow Metrics Scraper to get metrics from the Metrics server
|
||||
- apiGroups: ["metrics.k8s.io"]
|
||||
resources: ["pods", "nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
|
||||
---
|
||||
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: kubernetes-dashboard-minimal
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: Role
|
||||
name: kubernetes-dashboard-minimal
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
|
||||
---
|
||||
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: kubernetes-dashboard
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: kubernetes-dashboard
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: kubernetes-dashboard
|
||||
namespace: kube-system
|
||||
|
||||
---
|
||||
# ------------------- Dashboard Deployment ------------------- #
|
||||
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
@ -136,72 +205,120 @@ spec:
|
||||
k8s-app: kubernetes-dashboard
|
||||
spec:
|
||||
containers:
|
||||
- name: kubernetes-dashboard
|
||||
env:
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
image: ${KUBE_DASH_IMAGE}
|
||||
ports:
|
||||
- containerPort: 8443
|
||||
protocol: TCP
|
||||
args:
|
||||
- --auto-generate-certificates
|
||||
- --heapster-host=heapster:80
|
||||
# Uncomment the following line to manually specify Kubernetes API server Host
|
||||
# If not specified, Dashboard will attempt to auto discover the API server and connect
|
||||
# to it. Uncomment only if the default does not work.
|
||||
# - --apiserver-host=http://my-address:port
|
||||
volumeMounts:
|
||||
- name: kubernetes-dashboard-certs
|
||||
mountPath: /certs
|
||||
# Create on-disk volume to store exec logs
|
||||
- mountPath: /tmp
|
||||
name: tmp-volume
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
scheme: HTTPS
|
||||
path: /
|
||||
port: 8443
|
||||
initialDelaySeconds: 30
|
||||
timeoutSeconds: 30
|
||||
- name: kubernetes-dashboard
|
||||
image: ${KUBE_DASH_IMAGE}
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 8443
|
||||
protocol: TCP
|
||||
args:
|
||||
- --auto-generate-certificates
|
||||
- --namespace=kube-system
|
||||
# Uncomment the following line to manually specify Kubernetes API server Host
|
||||
# If not specified, Dashboard will attempt to auto discover the API server and connect
|
||||
# to it. Uncomment only if the default does not work.
|
||||
# - --apiserver-host=http://my-address:port
|
||||
volumeMounts:
|
||||
- name: kubernetes-dashboard-certs
|
||||
mountPath: /certs
|
||||
# Create on-disk volume to store exec logs
|
||||
- mountPath: /tmp
|
||||
name: tmp-volume
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
scheme: HTTPS
|
||||
path: /
|
||||
port: 8443
|
||||
initialDelaySeconds: 30
|
||||
timeoutSeconds: 30
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsUser: 1001
|
||||
runAsGroup: 2001
|
||||
volumes:
|
||||
- name: kubernetes-dashboard-certs
|
||||
secret:
|
||||
secretName: kubernetes-dashboard-certs
|
||||
- name: tmp-volume
|
||||
emptyDir: {}
|
||||
- name: kubernetes-dashboard-certs
|
||||
secret:
|
||||
secretName: kubernetes-dashboard-certs
|
||||
- name: tmp-volume
|
||||
emptyDir: {}
|
||||
serviceAccountName: kubernetes-dashboard
|
||||
nodeSelector:
|
||||
"kubernetes.io/os": linux
|
||||
# Comment the following tolerations if Dashboard must not be deployed on master
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/master
|
||||
effect: NoSchedule
|
||||
- key: node-role.kubernetes.io/master
|
||||
effect: NoSchedule
|
||||
|
||||
---
|
||||
# ------------------- Dashboard Service ------------------- #
|
||||
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: kubernetes-dashboard
|
||||
name: kubernetes-dashboard
|
||||
k8s-app: dashboard-metrics-scraper
|
||||
name: dashboard-metrics-scraper
|
||||
namespace: kube-system
|
||||
spec:
|
||||
ports:
|
||||
- port: 443
|
||||
targetPort: 8443
|
||||
- port: 8000
|
||||
targetPort: 8000
|
||||
selector:
|
||||
k8s-app: kubernetes-dashboard
|
||||
k8s-app: dashboard-metrics-scraper
|
||||
|
||||
---
|
||||
|
||||
kind: Deployment
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: dashboard-metrics-scraper
|
||||
name: dashboard-metrics-scraper
|
||||
namespace: kube-system
|
||||
spec:
|
||||
replicas: 1
|
||||
revisionHistoryLimit: 10
|
||||
selector:
|
||||
matchLabels:
|
||||
k8s-app: dashboard-metrics-scraper
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
k8s-app: dashboard-metrics-scraper
|
||||
annotations:
|
||||
seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
|
||||
spec:
|
||||
containers:
|
||||
- name: dashboard-metrics-scraper
|
||||
image: ${METRICS_SCRAPER_IMAGE}
|
||||
ports:
|
||||
- containerPort: 8000
|
||||
protocol: TCP
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
scheme: HTTP
|
||||
path: /
|
||||
port: 8000
|
||||
initialDelaySeconds: 30
|
||||
timeoutSeconds: 30
|
||||
volumeMounts:
|
||||
- mountPath: /tmp
|
||||
name: tmp-volume
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
runAsUser: 1001
|
||||
runAsGroup: 2001
|
||||
serviceAccountName: kubernetes-dashboard
|
||||
nodeSelector:
|
||||
"kubernetes.io/os": linux
|
||||
# Comment the following tolerations if Dashboard must not be deployed on master
|
||||
tolerations:
|
||||
- key: node-role.kubernetes.io/master
|
||||
effect: NoSchedule
|
||||
volumes:
|
||||
- name: tmp-volume
|
||||
emptyDir: {}
|
||||
EOF
|
||||
}
|
||||
|
||||
|
@ -146,6 +146,7 @@ CONTAINERD_VERSION="$CONTAINERD_VERSION"
|
||||
CONTAINERD_TARBALL_URL="$CONTAINERD_TARBALL_URL"
|
||||
CONTAINERD_TARBALL_SHA256="$CONTAINERD_TARBALL_SHA256"
|
||||
POST_INSTALL_MANIFEST_URL="$POST_INSTALL_MANIFEST_URL"
|
||||
METRICS_SCRAPER_TAG="$METRICS_SCRAPER_TAG"
|
||||
EOF
|
||||
}
|
||||
|
||||
|
@ -116,7 +116,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
|
||||
'draino_tag', 'autoscaler_tag',
|
||||
'min_node_count', 'max_node_count', 'npd_enabled',
|
||||
'ostree_remote', 'ostree_commit',
|
||||
'use_podman', 'kube_image_digest']
|
||||
'use_podman', 'kube_image_digest',
|
||||
'metrics_scraper_tag']
|
||||
|
||||
labels = self._get_relevant_labels(cluster, kwargs)
|
||||
|
||||
|
@ -500,7 +500,13 @@ parameters:
|
||||
kube_dashboard_version:
|
||||
type: string
|
||||
description: version of kubernetes dashboard used for kubernetes cluster
|
||||
default: v1.8.3
|
||||
default: v2.0.0
|
||||
|
||||
metrics_scraper_tag:
|
||||
type: string
|
||||
description: >
|
||||
Tag of metrics-scraper for kubernetes dashboard.
|
||||
default: v1.0.4
|
||||
|
||||
insecure_registry_url:
|
||||
type: string
|
||||
@ -1241,6 +1247,7 @@ resources:
|
||||
containerd_tarball_url: {get_param: containerd_tarball_url}
|
||||
containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
|
||||
post_install_manifest_url: {get_param: post_install_manifest_url}
|
||||
metrics_scraper_tag: {get_param: metrics_scraper_tag}
|
||||
|
||||
kube_cluster_config:
|
||||
condition: create_cluster_resources
|
||||
|
@ -635,6 +635,11 @@ parameters:
|
||||
Post install manifest url to setup some cloud provider/vendor
|
||||
specific configs
|
||||
|
||||
metrics_scraper_tag:
|
||||
type: string
|
||||
description: >
|
||||
Tag of metrics-scraper for kubernetes dashboard.
|
||||
|
||||
conditions:
|
||||
|
||||
image_based: {equals: [{get_param: boot_volume_size}, 0]}
|
||||
@ -824,6 +829,7 @@ resources:
|
||||
"$CONTAINERD_TARBALL_URL": {get_param: containerd_tarball_url}
|
||||
"$CONTAINERD_TARBALL_SHA256": {get_param: containerd_tarball_sha256}
|
||||
"$POST_INSTALL_MANIFEST_URL": {get_param: post_install_manifest_url}
|
||||
"$METRICS_SCRAPER_TAG": {get_param: metrics_scraper_tag}
|
||||
- get_file: ../../common/templates/kubernetes/fragments/install-cri.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/install-clients.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
|
||||
|
@ -504,7 +504,13 @@ parameters:
|
||||
kube_dashboard_version:
|
||||
type: string
|
||||
description: version of kubernetes dashboard used for kubernetes cluster
|
||||
default: v1.8.3
|
||||
default: v2.0.0
|
||||
|
||||
metrics_scraper_tag:
|
||||
type: string
|
||||
description: >
|
||||
Tag of metrics-scraper for kubernetes dashboard.
|
||||
default: v1.0.4
|
||||
|
||||
insecure_registry_url:
|
||||
type: string
|
||||
@ -1268,6 +1274,7 @@ resources:
|
||||
containerd_tarball_url: {get_param: containerd_tarball_url}
|
||||
containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
|
||||
post_install_manifest_url: {get_param: post_install_manifest_url}
|
||||
metrics_scraper_tag: {get_param: metrics_scraper_tag}
|
||||
|
||||
kube_cluster_config:
|
||||
condition: create_cluster_resources
|
||||
|
@ -645,6 +645,11 @@ parameters:
|
||||
Post install manifest url to setup some cloud provider/vendor
|
||||
specific configs
|
||||
|
||||
metrics_scraper_tag:
|
||||
type: string
|
||||
description: >
|
||||
Tag of metrics-scraper for kubernetes dashboard.
|
||||
|
||||
conditions:
|
||||
|
||||
image_based: {equals: [{get_param: boot_volume_size}, 0]}
|
||||
@ -836,6 +841,7 @@ resources:
|
||||
"$CONTAINERD_TARBALL_URL": {get_param: containerd_tarball_url}
|
||||
"$CONTAINERD_TARBALL_SHA256": {get_param: containerd_tarball_sha256}
|
||||
"$POST_INSTALL_MANIFEST_URL": {get_param: post_install_manifest_url}
|
||||
"$METRICS_SCRAPER_TAG": {get_param: metrics_scraper_tag}
|
||||
- get_file: ../../common/templates/kubernetes/fragments/install-cri.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/install-clients.sh
|
||||
- get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
|
||||
|
@ -636,6 +636,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||
containerd_tarball_sha256 = mock_cluster.labels.get(
|
||||
'containerd_tarball_sha256')
|
||||
kube_image_digest = mock_cluster.labels.get('kube_image_digest')
|
||||
metrics_scraper_tag = mock_cluster.labels.get('metrics_scraper_tag')
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
|
||||
@ -747,6 +748,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||
'containerd_tarball_url': containerd_tarball_url,
|
||||
'containerd_tarball_sha256': containerd_tarball_sha256,
|
||||
'post_install_manifest_url': '',
|
||||
'metrics_scraper_tag': metrics_scraper_tag,
|
||||
}}
|
||||
mock_get_params.assert_called_once_with(mock_context,
|
||||
mock_cluster_template,
|
||||
@ -1142,6 +1144,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||
containerd_tarball_sha256 = mock_cluster.labels.get(
|
||||
'containerd_tarball_sha256')
|
||||
kube_image_digest = mock_cluster.labels.get('kube_image_digest')
|
||||
metrics_scraper_tag = mock_cluster.labels.get('metrics_scraper_tag')
|
||||
|
||||
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
|
||||
|
||||
@ -1255,6 +1258,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
|
||||
'containerd_tarball_url': containerd_tarball_url,
|
||||
'containerd_tarball_sha256': containerd_tarball_sha256,
|
||||
'post_install_manifest_url': '',
|
||||
'metrics_scraper_tag': metrics_scraper_tag,
|
||||
}}
|
||||
mock_get_params.assert_called_once_with(mock_context,
|
||||
mock_cluster_template,
|
||||
|
@ -0,0 +1,5 @@
|
||||
---
|
||||
upgrade:
|
||||
- |
|
||||
The default version of Kubernetes dashboard has been upgraded to v2.0.0 and
|
||||
metrics-server is supported by k8s dashboard now.
|
Loading…
x
Reference in New Issue
Block a user