Remove storing xml files on each feed generation
This commit removes the file storing from the rss endpoint. The code is buggy and also a potential sec issue. It also isn't actually needed as most (if not all) rss reader will handle missing entries fine, as long as the ids are unique it shouldn't matter. So even if we lose old entries on a crash or restart it won't be an issue. Change-Id: I595abc566880ae6c778b00affde12e2227c9ec35
This commit is contained in:
parent
6939cd42f6
commit
04610c054e
@ -17,14 +17,11 @@ import argparse
|
||||
from contextlib import contextmanager
|
||||
from dateutil import parser as date_parser
|
||||
import itertools
|
||||
import os
|
||||
import six
|
||||
from six.moves import configparser as ConfigParser
|
||||
from six.moves.urllib import parse
|
||||
import tempfile
|
||||
|
||||
from feedgen import feed
|
||||
import feedparser
|
||||
import flask
|
||||
from flask import abort
|
||||
from flask.ext.jsonpify import jsonify
|
||||
@ -73,10 +70,6 @@ def setup():
|
||||
pool_recycle=pool_recycle)
|
||||
global Session
|
||||
Session = sessionmaker(bind=engine)
|
||||
try:
|
||||
rss_opts['data_dir'] = config.get('default', 'data_dir')
|
||||
except ConfigParser.Error:
|
||||
rss_opts['data_dir'] = tempfile.gettempdir()
|
||||
try:
|
||||
rss_opts['frontend_url'] = config.get('default', 'frontend_url')
|
||||
except ConfigParser.Error:
|
||||
@ -356,29 +349,6 @@ def _gen_feed(url, key, value):
|
||||
return fg
|
||||
|
||||
|
||||
def _get_stored_feed(url, key, value):
|
||||
filename = key + '_' + value + '.xml'
|
||||
file_path = os.path.join(rss_opts['data_dir'], filename)
|
||||
if not os.path.isfile(file_path):
|
||||
return None
|
||||
feed = feedparser.parse(file_path)
|
||||
out_feed = _gen_feed(url, key, value)
|
||||
if not feed.entries:
|
||||
return None
|
||||
last_run = sorted(
|
||||
[date_parser.parse(x.published) for x in feed.entries])[-1]
|
||||
last_run = last_run.replace(tzinfo=None)
|
||||
for i in feed.entries:
|
||||
entry = out_feed.add_entry()
|
||||
entry.id(i.id)
|
||||
entry.title(i.title)
|
||||
time = date_parser.parse(i.published)
|
||||
entry.published(time)
|
||||
entry.link({'href': i.link, 'rel': 'alternate'})
|
||||
entry.description(i.description)
|
||||
return out_feed, last_run
|
||||
|
||||
|
||||
@app.route('/runs/key/<path:run_metadata_key>/<path:value>/recent/rss',
|
||||
methods=['GET'])
|
||||
def get_recent_failed_runs_rss(run_metadata_key, value):
|
||||
@ -386,21 +356,11 @@ def get_recent_failed_runs_rss(run_metadata_key, value):
|
||||
value = parse.unquote(value)
|
||||
url = request.url
|
||||
if run_metadata_key not in feeds:
|
||||
stored_feed = _get_stored_feed(url, run_metadata_key, value)
|
||||
if stored_feed:
|
||||
feeds[run_metadata_key] = {value: stored_feed[0]}
|
||||
feeds["last runs"][run_metadata_key] = {value: stored_feed[1]}
|
||||
else:
|
||||
feeds[run_metadata_key] = {value: _gen_feed(url,
|
||||
run_metadata_key,
|
||||
value)}
|
||||
feeds["last runs"][run_metadata_key] = {value: None}
|
||||
elif value not in feeds[run_metadata_key]:
|
||||
stored_feed = _get_stored_feed(url, run_metadata_key, value)
|
||||
if stored_feed:
|
||||
feeds[run_metadata_key][value] = stored_feed[0]
|
||||
feeds["last runs"][run_metadata_key][value] = stored_feed[1]
|
||||
else:
|
||||
feeds[run_metadata_key][value] = _gen_feed(url,
|
||||
run_metadata_key,
|
||||
value)
|
||||
@ -437,9 +397,6 @@ def get_recent_failed_runs_rss(run_metadata_key, value):
|
||||
content = 'Metadata page: %s\n' % metadata_url
|
||||
content += '\nJob Page %s' % job_url
|
||||
entry.description(content)
|
||||
filename = run_metadata_key + '_' + value + '.xml'
|
||||
out_path = os.path.join(rss_opts['data_dir'], filename)
|
||||
feeds[run_metadata_key][value].rss_file(out_path)
|
||||
return feeds[run_metadata_key][value].rss_str()
|
||||
|
||||
|
||||
|
@ -14,7 +14,6 @@
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
@ -22,7 +21,6 @@ from dateutil import parser as date_parser
|
||||
import feedparser
|
||||
import mock
|
||||
import numpy
|
||||
import pytz
|
||||
import six
|
||||
from subunit2sql.db import models
|
||||
|
||||
@ -841,48 +839,6 @@ class TestRestAPI(base.TestCase):
|
||||
self.assertEqual(url, res['feed']['link'])
|
||||
self.assertEqual('en', res['feed']['language'])
|
||||
|
||||
def test__get_stored_feed_no_feed(self):
|
||||
api.rss_opts['data_dir'] = tempfile.gettempdir()
|
||||
self.assertIsNone(
|
||||
api._get_stored_feed('fake_url', 'not_a_real_key', 'real_value'))
|
||||
|
||||
def test__get_stored_feed_with_file_no_entries(self):
|
||||
api.rss_opts['data_dir'] = tempfile.gettempdir()
|
||||
url = 'fake_url'
|
||||
key = 'zeon'
|
||||
value = 'zaku'
|
||||
fg = api._gen_feed(url, key, value)
|
||||
filename = key + '_' + value + '.xml'
|
||||
path = os.path.join(api.rss_opts['data_dir'], filename)
|
||||
fg.rss_file(path)
|
||||
self.addCleanup(os.remove, path)
|
||||
self.assertIsNone(api._get_stored_feed(url, key, value))
|
||||
|
||||
def test__get_stored_feed_with_file_with_entries(self):
|
||||
api.rss_opts['data_dir'] = tempfile.gettempdir()
|
||||
url = 'fake_url'
|
||||
key = 'zeon'
|
||||
value = 'gouf'
|
||||
fg = api._gen_feed(url, key, value)
|
||||
entry = fg.add_entry()
|
||||
entry.id('not a zaku')
|
||||
entry.title("This i snot the title you're looking for")
|
||||
entry.published(pytz.utc.localize(timestamp_a))
|
||||
entry.link({'href': 'a_link'})
|
||||
entry.description('A description')
|
||||
filename = key + '_' + value + '.xml'
|
||||
path = os.path.join(api.rss_opts['data_dir'], filename)
|
||||
fg.rss_file(path)
|
||||
self.addCleanup(os.remove, path)
|
||||
out = api._get_stored_feed(url, key, value)
|
||||
res = feedparser.parse(out[0].rss_str())
|
||||
title = 'Failures for %s: %s' % (key, value)
|
||||
self.assertEqual(timestamp_a, out[1])
|
||||
self.assertEqual(title, res['feed']['title'])
|
||||
self.assertEqual(url, res['feed']['link'])
|
||||
self.assertEqual('en', res['feed']['language'])
|
||||
self.assertEqual(1, len(res.entries))
|
||||
|
||||
@mock.patch('subunit2sql.db.api.get_recent_failed_runs_by_run_metadata',
|
||||
return_value=[
|
||||
models.Run(uuid='a_uuid', run_at=timestamp_b,
|
||||
@ -891,9 +847,6 @@ class TestRestAPI(base.TestCase):
|
||||
])
|
||||
def test_get_recent_failed_runs_rss_no_previous(self, db_mock):
|
||||
api.rss_opts['data_dir'] = tempfile.gettempdir()
|
||||
filename = 'a_key_a_value.xml'
|
||||
path = os.path.join(api.rss_opts['data_dir'], filename)
|
||||
self.addCleanup(os.remove, path)
|
||||
api.rss_opts['frontend_url'] = 'http://status.openstack.org'
|
||||
build_uuid = str(uuid.uuid4())
|
||||
meta_mock = mock.patch(
|
||||
@ -925,9 +878,6 @@ class TestRestAPI(base.TestCase):
|
||||
])
|
||||
def test_get_recent_failed_runs_rss_with_previous(self, db_mock):
|
||||
api.rss_opts['data_dir'] = tempfile.gettempdir()
|
||||
filename = 'b_key_b_value.xml'
|
||||
path = os.path.join(api.rss_opts['data_dir'], filename)
|
||||
self.addCleanup(os.remove, path)
|
||||
api.rss_opts['frontend_url'] = 'http://status.openstack.org'
|
||||
build_uuid = str(uuid.uuid4())
|
||||
meta_mock = mock.patch(
|
||||
|
@ -13,4 +13,3 @@ numpy>=1.7.0 # BSD
|
||||
six>=1.9.0 # MIT
|
||||
pytz>=2013.6 # MIT
|
||||
feedgen>=0.3.2 # BSD
|
||||
feedparser>=5.2.1
|
||||
|
@ -9,3 +9,4 @@ mock>=1.2 # BSD
|
||||
fixtures>=1.3.1 # Apache-2.0/BSD
|
||||
sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD
|
||||
oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0
|
||||
feedparser>=5.2.1
|
||||
|
Loading…
Reference in New Issue
Block a user