Import osa-gate-profile
A few cobbled together scripts to perform analytics on OSA gate check timings based on console logs retrieved from logstash.openstack.org. Change-Id: I5f7664482751386cca2697954573a55be06e6b9b
This commit is contained in:
Logan V
parent
03c48e2357
commit
afe1a99f74
|
|
@ -0,0 +1,103 @@
|
|||
#!/usr/bin/php
|
||||
<?php
|
||||
/*
|
||||
Copyright 2016, Logan Vig <logan2211@gmail.com>
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
$fetch_results = 10000;
|
||||
$search_days = 14;
|
||||
|
||||
$search = '{"query":{"filtered":{"filter":{"bool":{"must":[{"range":{"@timestamp":{"from":1463241312439,"to":1463846112439}}},{"fquery":{"query":{"query_string":{"query":"project:(\"openstack\/openstack-ansible\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"build_name:(\"gate-openstack-ansible-dsvm-commit\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"message:(\"- Operation: [\u00a0openstack-ansible --forks\")"}},"_cache":true}}]}}}},"size":10000,"sort":[{"@timestamp":{"order":"desc","ignore_unmapped":true}},{"@timestamp":{"order":"desc","ignore_unmapped":true}}]}';
|
||||
$search = json_decode($search);
|
||||
$ts =& $search->query->filtered->filter->bool->must[0]->range->{'@timestamp'};
|
||||
$ts->from = strtotime("-$search_days days")*1000;
|
||||
$ts->to = time()*1000;
|
||||
|
||||
//remove timestamp limit from search
|
||||
//$nothing = array_shift($search->query->filtered->filter->bool->must);
|
||||
|
||||
$search->size = $fetch_results;
|
||||
$search = json_encode($search);
|
||||
|
||||
$date = time();
|
||||
$stats = array();
|
||||
$total_samples = 0;
|
||||
while ($total_samples < $fetch_results && $date > strtotime("-$search_days days")) {
|
||||
$strdate = date('Y.m.d', $date);
|
||||
$date -= 86400;
|
||||
|
||||
echo "Fetching results for $strdate\n";
|
||||
$url = "http://logstash.openstack.org/elasticsearch/logstash-$strdate/_search";
|
||||
$result = json_decode(fetch_results($search, $url));
|
||||
if (empty($result) || isset($result->error) || $result->status == 404) {
|
||||
echo "Error fetching results for $strdate.. skipping\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
echo "Analyzing ".count($result->hits->hits)." samples\n";
|
||||
$total_samples += process_results($result, $stats);
|
||||
}
|
||||
|
||||
foreach ($stats as &$branch) {
|
||||
foreach ($branch as &$nodepool) {
|
||||
foreach ($nodepool as &$playbook) {
|
||||
$avg = array_sum($playbook) / count($playbook);
|
||||
$playbook['samples'] = count($playbook);
|
||||
$playbook['average'] = $avg;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($stats as $bname => $branch) {
|
||||
echo "Stats for openstack-ansible/$bname\n";
|
||||
foreach ($branch as $npname => $nodepool) {
|
||||
echo "Node pool $npname:\n";
|
||||
uasort($nodepool, 'avg_cmp');
|
||||
foreach ($nodepool as $pbname => $playbook) {
|
||||
echo "\t$pbname: ".ceil($playbook['average'])." avg, ({$playbook['samples']} samples)\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function fetch_results($search, $url) {
|
||||
$ch = curl_init($url);
|
||||
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, $search);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
|
||||
'Content-Type: application/json',
|
||||
'Content-Length: ' . strlen($search))
|
||||
);
|
||||
return curl_exec($ch);
|
||||
}
|
||||
|
||||
function process_results($result, &$stats) {
|
||||
$regex = '/Operation: \[ openstack-ansible --forks [0-9]+\s+(?<play_name>[^\.]+).yml \]\s+(?<play_seconds>[0-9]+)\s+seconds/';
|
||||
foreach ($result->hits->hits as $r) {
|
||||
$rs = $r->{'_source'};
|
||||
if (!preg_match($regex, $rs->message, $m)) continue;
|
||||
$stats[$rs->build_branch][$rs->node_provider][$m['play_name']][] = $m['play_seconds'];
|
||||
}
|
||||
return count($result->hits->hits);
|
||||
}
|
||||
|
||||
function avg_cmp($a, $b) {
|
||||
if ($a['average'] == $b['average']) {
|
||||
return 0;
|
||||
}
|
||||
return ($a['average'] > $b['average']) ? -1 : 1;
|
||||
}
|
||||
?>
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
#!/usr/bin/php
|
||||
<?php
|
||||
/*
|
||||
Copyright 2016, Logan Vig <logan2211@gmail.com>
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
* Argument required is json output from logstash.openstack.org's table of metrics. There is no an export option so hack this together by changing the paging size
|
||||
* and watch the network traffic in chrome, then save it to a file.
|
||||
* Once the file exists, feed it to this script as an argument and it will go through and download all of the log_url items to a directory called 'dump'
|
||||
* You should probably mkdir dump/ in the same directory as this script first.
|
||||
*/
|
||||
|
||||
date_default_timezone_set('US/Central');
|
||||
|
||||
$fetch_results = 500;
|
||||
$search_days = 30;
|
||||
|
||||
$search = '{"query":{"filtered":{"filter":{"bool":{"must":[{"range":{"@timestamp":{"from":1463241312439,"to":1463846112439}}},{"fquery":{"query":{"query_string":{"query":"project:(\"openstack\/openstack-ansible\")"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"build_name:(/gate-openstack-ansible-openstack-ansible-aio-ubuntu-(trusty|xenial-nv)/)"}},"_cache":true}},{"fquery":{"query":{"query_string":{"query":"message:(\"gate-check-commit.sh\")"}},"_cache":true}}]}}}},"size":'.$fetch_results.',"sort":[{"@timestamp":{"order":"desc","ignore_unmapped":true}},{"@timestamp":{"order":"desc","ignore_unmapped":true}}]}';
|
||||
$search = json_decode($search);
|
||||
$ts =& $search->query->filtered->filter->bool->must[0]->range->{'@timestamp'};
|
||||
$ts->from = strtotime("-$search_days days")*1000;
|
||||
$ts->to = time()*1000;
|
||||
|
||||
$search->size = $fetch_results;
|
||||
$search = json_encode($search);
|
||||
|
||||
$date = time();
|
||||
$stats = array();
|
||||
$total_samples = 0;
|
||||
while ($total_samples < $fetch_results && $date > strtotime("-$search_days days")) {
|
||||
$strdate = date('Y.m.d', $date);
|
||||
$date -= 86400;
|
||||
|
||||
echo "Fetching results for $strdate\n";
|
||||
$url = "http://logstash.openstack.org/elasticsearch/logstash-$strdate/_search";
|
||||
$result = json_decode(fetch_results($search, $url));
|
||||
if (empty($result) || isset($result->error) || $result->status == 404) {
|
||||
echo "Error fetching results for $strdate.. skipping\n";
|
||||
continue;
|
||||
}
|
||||
|
||||
echo "Fetching ".count($result->hits->hits)." samples\n";
|
||||
foreach($result->hits->hits as $h) {
|
||||
$outfile = $h->{'_source'}->{'build_change'}.'-'.$h->{'_source'}->{'build_patchset'}.'.html';
|
||||
if (file_exists('dump/'.$outfile)) {
|
||||
echo "Already downloaded $outfile..skipping\n";
|
||||
continue;
|
||||
}
|
||||
$url = $h->{'_source'}->{'log_url'};
|
||||
$wget = "wget -O dump/{$outfile} {$url}";
|
||||
echo "running $wget\n";
|
||||
echo "Fetching $url to $outfile\n";
|
||||
$exec = fetch_log($url, $outfile);
|
||||
if ($exec['exit_code'] != 0) {
|
||||
$url = $url.'.gz';
|
||||
echo "Fetch failed. Trying $url\n";
|
||||
$exec = fetch_log($url, $outfile);
|
||||
if ($exec['return_code'] != 0) echo "Fetch failed retry. Skipping file.\n";
|
||||
else echo "Fetch succeeded retry\n";
|
||||
}
|
||||
else echo "Fetch succeeded\n";
|
||||
}
|
||||
}
|
||||
|
||||
function fetch_log($url, $outfile) {
|
||||
$wget = "wget -O dump/{$outfile} {$url}";
|
||||
exec($wget, $shell_output, $code);
|
||||
return array('output' => $shell_output, 'exit_code' => $code);
|
||||
}
|
||||
|
||||
function fetch_results($search, $url) {
|
||||
$ch = curl_init($url);
|
||||
//curl_setopt($ch, CURLOPT_VERBOSE, true);
|
||||
//curl_setopt($ch, CURLOPT_HEADER, true);
|
||||
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
|
||||
curl_setopt($ch, CURLOPT_POST, true);
|
||||
curl_setopt($ch, CURLOPT_POSTFIELDS, $search);
|
||||
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($ch, CURLOPT_HTTPHEADER, array(
|
||||
'Content-Type: application/json',
|
||||
'Content-Length: ' . strlen($search))
|
||||
);
|
||||
$result = curl_exec($ch);
|
||||
return $result;
|
||||
}
|
||||
|
||||
function process_results($result, &$stats) {
|
||||
$regex = '/Operation: \[ openstack-ansible --forks [0-9]+\s+(?<play_name>[^\.]+).yml \]\s+(?<play_seconds>[0-9]+)\s+seconds/';
|
||||
foreach ($result->hits->hits as $r) {
|
||||
$rs = $r->{'_source'};
|
||||
if (!preg_match($regex, $rs->message, $m)) continue;
|
||||
$stats[$rs->build_branch][$rs->node_provider][$m['play_name']][] = $m['play_seconds'];
|
||||
}
|
||||
return count($result->hits->hits);
|
||||
}
|
||||
?>
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/php
|
||||
<?php
|
||||
/*
|
||||
Copyright 2016, Logan Vig <logan2211@gmail.com>
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
$f = json_decode(file_get_contents($argv[1]), true);
|
||||
$profiles = [];
|
||||
$sorted = array();
|
||||
foreach ($f as $taskname => $nodeproviders) {
|
||||
echo "$taskname:\n";
|
||||
foreach ($nodeproviders as $p => $n) {
|
||||
$avg = round(array_sum($n) / count($n), 2);
|
||||
$samples = count($n);
|
||||
echo "\t$p ($samples samples) - $avg\n";
|
||||
$sorted[$p][$taskname] = $avg;
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($sorted as $k => $v) {
|
||||
echo $k."\n";
|
||||
arsort($v);
|
||||
foreach ($v as $tasks => $avgs) {
|
||||
echo "\t$tasks = $avgs\n";
|
||||
}
|
||||
}
|
||||
?>
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/php
|
||||
<?php
|
||||
/*
|
||||
Copyright 2016, Logan Vig <logan2211@gmail.com>
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
* Parse all of the console logs saved to dump/*
|
||||
* This will try to regex match the first few lines which identify the nodepool
|
||||
* Then it will search through the console log for PLAY RECAP style output which contains execution times
|
||||
* for all of the ansible tasks run. Once it is done it will output a json object of metrics.
|
||||
* Example output at http://cdn.pasteraw.com/5efufwesy1nmxnl6qis9z1zr1hbce0
|
||||
*/
|
||||
$dir = new DirectoryIterator(dirname(__FILE__).'/dump/');
|
||||
$profile = array();
|
||||
$profiles=0;
|
||||
|
||||
pcntl_signal(SIGHUP, function($signo) {
|
||||
global $profile;
|
||||
echo json_encode($profile)."\n";
|
||||
});
|
||||
pcntl_signal(SIGTERM, function($signo) {
|
||||
print_exit();
|
||||
});
|
||||
pcntl_signal(SIGINT, function($signo) {
|
||||
print_exit();
|
||||
});
|
||||
|
||||
foreach ($dir as $f) {
|
||||
if (!$f->isFile()) continue;
|
||||
$c = file_get_contents('dump/'.$f);
|
||||
if (!preg_match('/Building remotely on (?:<a .*>)?([^<\s]+)-[0-9]+(?:<\/a>)?/', $c, $m)) {
|
||||
file_put_contents("php://stderr", "Failed to find node type of $f\n");
|
||||
continue;
|
||||
}
|
||||
$nodetype = $m[1];
|
||||
foreach (explode("\n", $c) as $l) {
|
||||
if (preg_match('/(?:[^|]+)\|\s+(.*)\s+[-]+\s+([0-9\.s]+)s\s*$/', $l, $m)) {
|
||||
$role_task = $m[1];
|
||||
$seconds = (float)$m[2];
|
||||
$profile[$role_task][$nodetype][] = $seconds;
|
||||
}
|
||||
}
|
||||
$profiles++;
|
||||
pcntl_signal_dispatch();
|
||||
}
|
||||
|
||||
print_exit();
|
||||
|
||||
function print_exit() {
|
||||
global $profile, $profiles;
|
||||
file_put_contents("php://stderr", "\nExiting with $profiles profiles completed.\n");
|
||||
exit(json_encode($profile));
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
Generate metrics from openstack-ansible gate check console logs
|
||||
###############################################################
|
||||
:date: 2016-10-10
|
||||
:tags: openstack, ansible
|
||||
:category: \*openstack, \*nix
|
||||
|
||||
|
||||
About this repository
|
||||
---------------------
|
||||
|
||||
These scripts will query logstash.openstack.org to find a set of OSA gate check
|
||||
console logs, download them, and perform task timing analytics.
|
||||
|
||||
- step1: fetchlogs.php
|
||||
- step2: parselogs.php
|
||||
- step3: generatereports.php
|
||||
|
||||
Example run
|
||||
-----------
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
mkdir dump
|
||||
php fetchlogs.php
|
||||
php parselogs.php > intermediary.json
|
||||
php generatereports.php intermediary.json
|
||||
rm -rf intermediary.json
|
||||
Loading…
Reference in New Issue