Remove storing xml files on each feed generation

This commit removes the file storing from the rss endpoint. The code
is buggy and also a potential sec issue. It also isn't actually needed
as most (if not all) rss reader will handle missing entries fine, as
long as the ids are unique it shouldn't matter. So even if we lose old
entries on a crash or restart it won't be an issue.

Change-Id: I595abc566880ae6c778b00affde12e2227c9ec35

openstack_health/api.py

@@ -17,14 +17,11 @@ import argparse
 from contextlib import contextmanager
 from dateutil import parser as date_parser
 import itertools
-import os
 import six
 from six.moves import configparser as ConfigParser
 from six.moves.urllib import parse
-import tempfile
 
 from feedgen import feed
-import feedparser
 import flask
 from flask import abort
 from flask.ext.jsonpify import jsonify
@@ -73,10 +70,6 @@ def setup():
                            pool_recycle=pool_recycle)
     global Session
     Session = sessionmaker(bind=engine)
-    try:
-        rss_opts['data_dir'] = config.get('default', 'data_dir')
-    except ConfigParser.Error:
-        rss_opts['data_dir'] = tempfile.gettempdir()
     try:
         rss_opts['frontend_url'] = config.get('default', 'frontend_url')
     except ConfigParser.Error:
@@ -356,29 +349,6 @@ def _gen_feed(url, key, value):
     return fg
 
 
-def _get_stored_feed(url, key, value):
-    filename = key + '_' + value + '.xml'
-    file_path = os.path.join(rss_opts['data_dir'], filename)
-    if not os.path.isfile(file_path):
-        return None
-    feed = feedparser.parse(file_path)
-    out_feed = _gen_feed(url, key, value)
-    if not feed.entries:
-        return None
-    last_run = sorted(
-        [date_parser.parse(x.published) for x in feed.entries])[-1]
-    last_run = last_run.replace(tzinfo=None)
-    for i in feed.entries:
-        entry = out_feed.add_entry()
-        entry.id(i.id)
-        entry.title(i.title)
-        time = date_parser.parse(i.published)
-        entry.published(time)
-        entry.link({'href': i.link, 'rel': 'alternate'})
-        entry.description(i.description)
-    return out_feed, last_run
-
-
 @app.route('/runs/key/<path:run_metadata_key>/<path:value>/recent/rss',
            methods=['GET'])
 def get_recent_failed_runs_rss(run_metadata_key, value):
@@ -386,25 +356,15 @@ def get_recent_failed_runs_rss(run_metadata_key, value):
     value = parse.unquote(value)
     url = request.url
     if run_metadata_key not in feeds:
-        stored_feed = _get_stored_feed(url, run_metadata_key, value)
+        feeds[run_metadata_key] = {value: _gen_feed(url,
-        if stored_feed:
+                                                    run_metadata_key,
-            feeds[run_metadata_key] = {value: stored_feed[0]}
+                                                    value)}
-            feeds["last runs"][run_metadata_key] = {value: stored_feed[1]}
+        feeds["last runs"][run_metadata_key] = {value: None}
-        else:
-            feeds[run_metadata_key] = {value: _gen_feed(url,
-                                                        run_metadata_key,
-                                                        value)}
-            feeds["last runs"][run_metadata_key] = {value: None}
     elif value not in feeds[run_metadata_key]:
-        stored_feed = _get_stored_feed(url, run_metadata_key, value)
+        feeds[run_metadata_key][value] = _gen_feed(url,
-        if stored_feed:
+                                                   run_metadata_key,
-            feeds[run_metadata_key][value] = stored_feed[0]
+                                                   value)
-            feeds["last runs"][run_metadata_key][value] = stored_feed[1]
+        feeds["last runs"][run_metadata_key][value] = None
-        else:
-            feeds[run_metadata_key][value] = _gen_feed(url,
-                                                       run_metadata_key,
-                                                       value)
-            feeds["last runs"][run_metadata_key][value] = None
     fg = feeds[run_metadata_key][value]
     with session_scope() as session:
         failed_runs = api.get_recent_failed_runs_by_run_metadata(
@@ -437,9 +397,6 @@ def get_recent_failed_runs_rss(run_metadata_key, value):
             content = 'Metadata page: %s\n' % metadata_url
             content += '\nJob Page %s' % job_url
             entry.description(content)
-    filename = run_metadata_key + '_' + value + '.xml'
-    out_path = os.path.join(rss_opts['data_dir'], filename)
-    feeds[run_metadata_key][value].rss_file(out_path)
     return feeds[run_metadata_key][value].rss_str()
 
 

openstack_health/tests/test_api.py

@@ -14,7 +14,6 @@
 
 import datetime
 import json
-import os
 import tempfile
 import uuid
 
@@ -22,7 +21,6 @@ from dateutil import parser as date_parser
 import feedparser
 import mock
 import numpy
-import pytz
 import six
 from subunit2sql.db import models
 
@@ -841,48 +839,6 @@ class TestRestAPI(base.TestCase):
         self.assertEqual(url, res['feed']['link'])
         self.assertEqual('en', res['feed']['language'])
 
-    def test__get_stored_feed_no_feed(self):
-        api.rss_opts['data_dir'] = tempfile.gettempdir()
-        self.assertIsNone(
-            api._get_stored_feed('fake_url', 'not_a_real_key', 'real_value'))
-
-    def test__get_stored_feed_with_file_no_entries(self):
-        api.rss_opts['data_dir'] = tempfile.gettempdir()
-        url = 'fake_url'
-        key = 'zeon'
-        value = 'zaku'
-        fg = api._gen_feed(url, key, value)
-        filename = key + '_' + value + '.xml'
-        path = os.path.join(api.rss_opts['data_dir'], filename)
-        fg.rss_file(path)
-        self.addCleanup(os.remove, path)
-        self.assertIsNone(api._get_stored_feed(url, key, value))
-
-    def test__get_stored_feed_with_file_with_entries(self):
-        api.rss_opts['data_dir'] = tempfile.gettempdir()
-        url = 'fake_url'
-        key = 'zeon'
-        value = 'gouf'
-        fg = api._gen_feed(url, key, value)
-        entry = fg.add_entry()
-        entry.id('not a zaku')
-        entry.title("This i snot the title you're looking for")
-        entry.published(pytz.utc.localize(timestamp_a))
-        entry.link({'href': 'a_link'})
-        entry.description('A description')
-        filename = key + '_' + value + '.xml'
-        path = os.path.join(api.rss_opts['data_dir'], filename)
-        fg.rss_file(path)
-        self.addCleanup(os.remove, path)
-        out = api._get_stored_feed(url, key, value)
-        res = feedparser.parse(out[0].rss_str())
-        title = 'Failures for %s: %s' % (key, value)
-        self.assertEqual(timestamp_a, out[1])
-        self.assertEqual(title, res['feed']['title'])
-        self.assertEqual(url, res['feed']['link'])
-        self.assertEqual('en', res['feed']['language'])
-        self.assertEqual(1, len(res.entries))
-
     @mock.patch('subunit2sql.db.api.get_recent_failed_runs_by_run_metadata',
                 return_value=[
                     models.Run(uuid='a_uuid', run_at=timestamp_b,
@@ -891,9 +847,6 @@ class TestRestAPI(base.TestCase):
                 ])
     def test_get_recent_failed_runs_rss_no_previous(self, db_mock):
         api.rss_opts['data_dir'] = tempfile.gettempdir()
-        filename = 'a_key_a_value.xml'
-        path = os.path.join(api.rss_opts['data_dir'], filename)
-        self.addCleanup(os.remove, path)
         api.rss_opts['frontend_url'] = 'http://status.openstack.org'
         build_uuid = str(uuid.uuid4())
         meta_mock = mock.patch(
@@ -925,9 +878,6 @@ class TestRestAPI(base.TestCase):
                 ])
     def test_get_recent_failed_runs_rss_with_previous(self, db_mock):
         api.rss_opts['data_dir'] = tempfile.gettempdir()
-        filename = 'b_key_b_value.xml'
-        path = os.path.join(api.rss_opts['data_dir'], filename)
-        self.addCleanup(os.remove, path)
         api.rss_opts['frontend_url'] = 'http://status.openstack.org'
         build_uuid = str(uuid.uuid4())
         meta_mock = mock.patch(

requirements.txt

@@ -13,4 +13,3 @@ numpy>=1.7.0 # BSD
 six>=1.9.0 # MIT
 pytz>=2013.6 # MIT
 feedgen>=0.3.2 # BSD
-feedparser>=5.2.1

test-requirements.txt

@@ -9,3 +9,4 @@ mock>=1.2 # BSD
 fixtures>=1.3.1 # Apache-2.0/BSD
 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
+feedparser>=5.2.1