diff --git a/doc/source/test_plans/kubernetes_density/plan.rst b/doc/source/test_plans/kubernetes_density/plan.rst
index 779ae5e..a959e9a 100644
--- a/doc/source/test_plans/kubernetes_density/plan.rst
+++ b/doc/source/test_plans/kubernetes_density/plan.rst
@@ -65,6 +65,34 @@ List of performance metrics
+-------------------------+---------------------------------------------+
+Test Case #2: Measure Kubelet capacity
+--------------------------------------
+
+Description
+^^^^^^^^^^^
+The goal of this test is to investigate Kubelet capacity. For this, rerun
+test case #1 for different number of nodes.
+
+List of performance metrics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. table:: list of test metrics to be collected during this test
+
+ +-------------------------+---------------------------------------------+
+ | Parameter | Description |
+ +=========================+=============================================+
+ | POD_COUNT | Number of pods |
+ +-------------------------+---------------------------------------------+
+ | NODE_COUNT | Number of nodes |
+ +-------------------------+---------------------------------------------+
+ | POD_FIRST_REPORT | Time taken by pod to start and report |
+ +-------------------------+---------------------------------------------+
+ | KUBECTL_RUN | Time for all pods to be reported as running |
+ +-------------------------+---------------------------------------------+
+ | KUBECTL_TERMINATE | Time to terminate all pods |
+ +-------------------------+---------------------------------------------+
+
+
Reports
=======
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100.png
new file mode 100644
index 0000000..0d72150
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100_term.png
new file mode 100644
index 0000000..dfc1c63
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x100_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200.png
new file mode 100644
index 0000000..05e7404
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200_term.png
new file mode 100644
index 0000000..9c76bb3
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x200_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50.png
new file mode 100644
index 0000000..65a022a
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50_term.png
new file mode 100644
index 0000000..8550d50
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/100x50_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100.png
new file mode 100644
index 0000000..9cecae4
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100_term.png
new file mode 100644
index 0000000..c387dda
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x100_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200.png
new file mode 100644
index 0000000..1723a85
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200_term.png
new file mode 100644
index 0000000..9b608d5
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x200_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50.png
new file mode 100644
index 0000000..3ea24cf
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50_term.png
new file mode 100644
index 0000000..8d1911f
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/200x50_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100.png
new file mode 100644
index 0000000..feb527b
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100_term.png
new file mode 100644
index 0000000..1ebb04c
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x100_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200.png
new file mode 100644
index 0000000..5ef8e81
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200_term.png
new file mode 100644
index 0000000..da51b8c
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x200_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400.png
new file mode 100644
index 0000000..8aa83ea
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400_term.png
new file mode 100644
index 0000000..7f1d7ec
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x400_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50.png
new file mode 100644
index 0000000..340c39a
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50_term.png b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50_term.png
new file mode 100644
index 0000000..051e8a2
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/50x50_term.png differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/data/reports.tar.bz2 b/doc/source/test_results/container_cluster_systems/kubernetes/density/data/reports.tar.bz2
new file mode 100644
index 0000000..12899fc
Binary files /dev/null and b/doc/source/test_results/container_cluster_systems/kubernetes/density/data/reports.tar.bz2 differ
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst b/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst
index 5cad40d..8316b49 100644
--- a/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst
+++ b/doc/source/test_results/container_cluster_systems/kubernetes/density/index.rst
@@ -27,12 +27,18 @@ Node roles:
- node3: minion+etcd
- node4: minion
-Software versions:
+Software versions (test case #1):
- OS: Ubuntu 16.04.1 LTS (Xenial Xerus)
- Kernel: 4.4.0-47
- Docker: 1.12.1
- Kubernetes: 1.4.3
+Software versions (test case #2):
+ - OS: Ubuntu 16.04.1 LTS (Xenial Xerus)
+ - Kernel: 4.4.0-36
+ - Docker: 1.13.1
+ - Kubernetes: 1.5.3
+
Reports
=======
@@ -176,6 +182,252 @@ System metrics from API nodes and minion are below
Full `Kubernetes stats`_ are available online.
+Test Case #2: Measure Kubelet capacity
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Pod startup time is measured with help of
+`MMM(MySQL/Master/Minions) testing suite`_. Original code was updated. We added
+automatic creation of charts with pod's status, when pod startup (or down). To
+schedule all pods on a single node the original replication controller for
+minions is updated with scheduler hint. To do this add the following lines
+into template's spec section:
+
+.. code-block:: yaml
+
+ nodeSelector:
+ kubernetes.io/hostname: <node>
+
+Pod status from Kubernetes point of view is retrieved from kubectl tool.
+The process is automated with
+:download:`kubectl_mon_v2.py <kubectl-mon/kubectl_mon_v2.py>`, which collects
+information about pod's status and sends to database. Charts are created by
+updated `MMM(MySQL/Master/Minions) testing suite`_.
+
+Every measurement starts with empty namespace. Then Kubernetes replication
+controller is created with specified number of pods. We collect pod's report
+time and kubectl stats. The summary data is presented below.
+
+.. list-table:: POD stats
+ :header-rows: 1
+
+ *
+ - POD_COUNT
+ - NODE_COUNT
+ - POD_FIRST_REPORT, s
+ - KUBECTL_RUN, s
+ - KUBECTL_TERMINATE, s
+ *
+ - 50
+ - 50
+ - 197
+ - 290
+ - 289
+ *
+ - 100
+ - 50
+ - 415
+ - 597
+ - 577
+ *
+ - 200
+ - 50
+ - 952
+ - 1218
+ - 1154
+ *
+ - 50
+ - 100
+ - 381
+ - 425
+ - 536
+ *
+ - 100
+ - 100
+ - 788
+ - 2093 (with errors)
+ - 1076
+ *
+ - 200
+ - 100
+ - 2653
+ - 3838 (not finished)
+ - 3001 (not finished)
+ *
+ - 50
+ - 200
+ - 970
+ - 1256
+ - 1032
+ *
+ - 100
+ - 200
+ - 1632
+ - 3225
+ - 2248
+ *
+ - 200
+ - 200
+ - 7098 (6.5% lost)
+ - 8075 (not finished)
+ - ∞ (not finished)
+ *
+ - 50
+ - 400
+ - 1823
+ - 2667 (with errors)
+ - 2038
+ *
+ - 100
+ - 400
+ - 7582
+ - 8262 (with errors)
+ - 5200
+ *
+ - 400
+ - 50
+ - 9607
+ - ∞ (not finished)
+ - ∞ (not finished)
+
+
+Detailed Stats
+--------------
+
+Note: You can download these reports in HTML format
+:download:`here <data/reports.tar.bz2>`
+
+50 pods (~1 pod per core) on 50 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 50 pods on 50 nodes
+
+.. image:: 50x50.png
+ :width: 100%
+
+Terminate replication controller with 50 pods on 50 nodes
+
+.. image:: 50x50_term.png
+ :width: 100%
+
+100 pods (~2 pod per core) on 50 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 100 pods on 50 nodes
+
+.. image:: 50x100.png
+ :width: 100%
+
+Terminate replication controller with 100 pods on 50 nodes
+
+.. image:: 50x100_term.png
+ :width: 100%
+
+200 pods (~4 pod per core) on 50 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 200 pods on 50 nodes
+
+.. image:: 50x200.png
+ :width: 100%
+
+Terminate replication controller with 200 pods on 50 nodes
+
+.. image:: 50x200_term.png
+ :width: 100%
+
+50 pods (~1 pod per core) on 100 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 50 pods on 100 nodes
+
+.. image:: 100x50.png
+ :width: 100%
+
+Terminate replication controller with 50 pods on 100 nodes
+
+.. image:: 100x50_term.png
+ :width: 100%
+
+100 pods (~2 pod per core) on 100 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 100 pods on 100 nodes
+
+.. image:: 100x100.png
+ :width: 100%
+
+Terminate replication controller with 100 pods on 100 nodes
+
+.. image:: 100x100_term.png
+ :width: 100%
+
+200 pods (~4 pod per core) on 100 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 200 pods on 100 nodes
+
+.. image:: 100x200.png
+ :width: 100%
+
+Terminate replication controller with 200 pods on 100 nodes
+
+.. image:: 100x200_term.png
+ :width: 100%
+
+50 pods (~1 pod per core) on 200 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 50 pods on 100 nodes
+
+.. image:: 200x50.png
+ :width: 100%
+
+Terminate replication controller with 50 pods on 100 nodes
+
+.. image:: 200x50_term.png
+ :width: 100%
+
+100 pods (~2 pod per core) on 200 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 100 pods on 200 nodes
+
+.. image:: 200x100.png
+ :width: 100%
+
+Terminate replication controller with 100 pods on 200 nodes
+
+.. image:: 200x100_term.png
+ :width: 100%
+
+200 pods (~4 pod per core) on 200 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 200 pods on 200 nodes
+
+Note: Docker service is frozen on 27 nodes
+
+.. image:: 200x200.png
+ :width: 100%
+
+Terminate replication controller with 200 pods on 200 nodes
+
+.. image:: 200x200_term.png
+ :width: 100%
+
+400 pods (~8 pod per core) on 50 nodes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Start replication controller with 400 pods on 50 nodes
+
+.. image:: 50x400.png
+ :width: 100%
+
+Terminate replication controller with 400 pods on 50 nodes
+
+.. image:: 50x400_term.png
+ :width: 100%
+
.. references:
.. _Kargo: https://github.com/kubespray/kargo
diff --git a/doc/source/test_results/container_cluster_systems/kubernetes/density/kubectl-mon/kubectl_mon_v2.py b/doc/source/test_results/container_cluster_systems/kubernetes/density/kubectl-mon/kubectl_mon_v2.py
new file mode 100644
index 0000000..49f1efa
--- /dev/null
+++ b/doc/source/test_results/container_cluster_systems/kubernetes/density/kubectl-mon/kubectl_mon_v2.py
@@ -0,0 +1,75 @@
+#!/bin/python
+
+import optparse
+import re
+import requests
+from requests.adapters import HTTPAdapter
+import sys
+import subprocess
+import time
+
+KUBECTL_CMD = 'kubectl --namespace minions get pods -l k8s-app=minion'
+TIMEOUT = 1200 # in seconds
+
+
+class Monitor(object):
+ def __init__(self, **options):
+ self.options = options
+
+ def check_direction(self, n, l):
+ if self.direction == 'up':
+ if n > 0 and l == n + 1:
+ print 'Done.'
+ sys.exit(0)
+ if self.direction == 'down':
+ if l == 0:
+ print 'Done.'
+ sys.exit(0)
+
+ def run(self):
+ self.direction = self.options.get('direction', 'up')
+ base_time = int(time.time() * 1000000000)
+
+ s = requests.Session()
+ s.mount('http://master.minions:8888', HTTPAdapter(max_retries=10))
+
+ while True:
+ start = int(time.time() * 1000000000)
+ stdout = subprocess.Popen(KUBECTL_CMD, shell=True,
+ stdout=subprocess.PIPE).stdout.read()
+
+ n = 0
+ for line in stdout.split('\n')[1:]:
+ if line:
+ tokens = re.split('\s+', line)
+ name = tokens[0]
+ status = tokens[2]
+
+ if status == 'Running':
+ n += 1
+
+ url = "http://master.minions:8888/monitor?minion_time=%s&" \
+ "minion_name=%s&minion_status=%s&direction=%s" % (
+ start, name, status, self.direction)
+
+ s.get(url)
+
+ length_lines = len(stdout.split('\n')[1:])
+ self.check_direction(n, length_lines)
+
+ if (start - base_time) / 1000000000 > TIMEOUT:
+ print 'Timeout.(%d)' % TIMEOUT
+ sys.exit(0)
+
+ time.sleep(1)
+
+
+def main():
+ parser = optparse.OptionParser()
+ parser.add_option('-d', '--direction', dest='direction', default='up')
+ options, args = parser.parse_args()
+
+ Monitor(**vars(options)).run()
+
+if __name__ == '__main__':
+ main()