openinfra/openstackid-resources
Code Issues Proposed changes

added new public endpoint

Browse Source 
added new public endpoint api/public/v1/members
also added api rate limit middleware

Change-Id: I456e2a56f3afdc03b50cb771faf0cde7a373a18e
This commit is contained in:
Sebastian Marcet 2017-06-02 14:21:05 -03:00
parent 561627a437
commit 8d9968a187
17 changed files with 491 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/Http/Controllers/apis/protected/main/OAuth2MembersApiController.php
View File

@ -40,7 +40,8 @@ final class OAuth2MembersApiController extends OAuth2ProtectedController
(
IMemberRepository $member_repository,
IResourceServerContext $resource_server_context
) {
)
{
parent::__construct($resource_server_context);
$this->repository = $member_repository;
}

5
app/Http/Middleware/ETagsMiddleware.php
View File

@ -12,10 +12,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use Closure;
use Illuminate\Support\Facades\Log;
/**
* Class ETagsMiddleware
* @package App\Http\Middleware
@ -34,13 +32,12 @@ final class ETagsMiddleware
$response = $next($request);
if ($response->getStatusCode() === 200 && $request->getMethod() === 'GET')
{
$etag = md5($response->getContent());
$etag = md5($response->getContent());
$requestETag = str_replace('"', '', $request->getETags());
$requestETag = str_replace('-gzip', '', $requestETag);
if ($requestETag && $requestETag[0] == $etag)
{
Log::debug('ETAG 304');
$response->setNotModified();
}
$response->setEtag($etag);

125
app/Http/Middleware/RateLimitMiddleware.php
View File

@ -1,5 +1,4 @@
<?php namespace App\Http\Middleware;
/**
* Copyright 2015 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
@ -12,18 +11,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use Closure;
use Illuminate\Support\Facades\Response;
use libs\utils\ICacheService;
use libs\utils\RequestUtils;
use App\Models\ResourceServer\IApiEndpointRepository;
use App\Models\ResourceServer\IEndpointRateLimitByIPRepository;
use Closure;
use Illuminate\Cache\RateLimiter;
use Illuminate\Routing\Middleware\ThrottleRequests;
use libs\utils\RequestUtils;
/**
* Class RateLimitMiddleware
* @package App\Http\Middleware
*/
final class RateLimitMiddleware
final class RateLimitMiddleware extends ThrottleRequests
{
/**
@ -32,69 +31,79 @@ final class RateLimitMiddleware
private $endpoint_repository;
/**
* @var ICacheService
* @var IEndpointRateLimitByIPRepository
*/
private $cache_service;
private $endpoint_rate_limit_by_ip_repository;
/**
* RateLimitMiddleware constructor.
* @param IApiEndpointRepository $endpoint_repository
* @param ICacheService $cache_service
* @param IEndpointRateLimitByIPRepository $endpoint_rate_limit_by_ip_repository
* @param RateLimiter $limiter
*/
public function __construct(IApiEndpointRepository $endpoint_repository, ICacheService $cache_service)
public function __construct
(
IApiEndpointRepository $endpoint_repository,
IEndpointRateLimitByIPRepository $endpoint_rate_limit_by_ip_repository,
RateLimiter $limiter
)
{
$this->endpoint_repository = $endpoint_repository;
$this->cache_service = $cache_service;
parent::__construct($limiter);
$this->endpoint_repository = $endpoint_repository;
$this->endpoint_rate_limit_by_ip_repository = $endpoint_rate_limit_by_ip_repository;
}
/**
* Handle an incoming request.
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @return mixed
* @param \Illuminate\Http\Request $request
* @param Closure $next
* @param int $max_attempts
* @param int $decay_minutes
* @return \Illuminate\Http\Response|mixed
*/
public function handle($request, Closure $next)
public function handle($request, Closure $next, $max_attempts = 0, $decay_minutes = 0)
{
$route = RequestUtils::getCurrentRoutePath($request);
$method = $request->getMethod();
$endpoint = $this->endpoint_repository->getApiEndpointByUrlAndMethod($route, $method);
$key = $this->resolveRequestSignature($request);
$client_ip = $request->getClientIp();
if (!is_null($endpoint) && $endpoint->getRateLimit() > 0) {
$max_attempts = $endpoint->getRateLimit();
}
if (!is_null($endpoint) && $endpoint->getRateLimitDecay() > 0) {
$decay_minutes = $endpoint->getRateLimitDecay();
}
$endpoint_rate_limit_by_ip = $this->endpoint_rate_limit_by_ip_repository->getByIPRouteMethod
(
$client_ip,
$route,
$method
);
if(!is_null($endpoint_rate_limit_by_ip)){
$max_attempts = $endpoint_rate_limit_by_ip->getRateLimit();
$decay_minutes = $endpoint_rate_limit_by_ip->getRateLimitDecay();
}
if ($max_attempts == 0 || $decay_minutes == 0) {
// short circuit (infinite)
return $next($request);
}
if ($this->limiter->tooManyAttempts($key, $max_attempts, $decay_minutes)) {
return $this->buildResponse($key, $max_attempts);
}
$this->limiter->hit($key, $decay_minutes);
$response = $next($request);
// if response was not changed then short circuit ...
if ($response->getStatusCode() === 304) {
return $response;
}
$url = $request->getRequestUri();
try {
$route = RequestUtils::getCurrentRoutePath($request);
$method = $request->getMethod();
$endpoint = $this->endpoint_repository->getApiEndpointByUrlAndMethod($route, $method);
if (!is_null($endpoint->getRateLimit()) && ($requestsPerHour = (int)$endpoint->getRateLimit()) > 0) {
//do rate limit checking
$key = sprintf('rate.limit.%s_%s_%s', $url, $method, $request->getClientIp());
// Add if doesn't exist
// Remember for 1 hour
$this->cache_service->addSingleValue($key, 0, 3600);
// Add to count
$count = $this->cache_service->incCounter($key);
if ($count > $requestsPerHour) {
// Short-circuit response - we're ignoring
$response = Response::json(array(
'message' => "You have triggered an abuse detection mechanism and have been temporarily blocked.
Please retry your request again later."
), 403);
$ttl = (int)$this->cache_service->ttl($key);
$response->headers->set('X-RateLimit-Reset', $ttl, false);
}
$response->headers->set('X-Ratelimit-Limit', $requestsPerHour, false);
$remaining = $requestsPerHour - (int)$count;
if ($remaining < 0) {
$remaining = 0;
}
$response->headers->set('X-Ratelimit-Remaining', $remaining, false);
}
} catch (Exception $ex) {
Log::error($ex);
}
return $response;
return $this->addHeaders(
$response, $max_attempts,
$this->calculateRemainingAttempts($key, $max_attempts)
);
}
}

4
app/Http/Utils/Filters/DoctrineJoinFilterMapping.php
View File

@ -63,13 +63,13 @@ class DoctrineJoinFilterMapping extends FilterMapping
/**
* @param QueryBuilder $query
* @param FilterElement $filter
* @return QueryBuilder
* @return string
*/
public function applyOr(QueryBuilder $query, FilterElement $filter){
$where = str_replace(":value", $filter->getValue(), $this->where);
$where = str_replace(":operator", $filter->getOperator(), $where);
if(!in_array($this->alias, $query->getAllAliases()))
$query->innerJoin($this->table, $this->alias, Join::WITH);
return $query->orWhere($where);
return $where;
}
}

37
app/Http/Utils/Filters/Filter.php
View File

@ -190,25 +190,46 @@ final class Filter
}
else if (is_array($filter)) {
// OR
$sub_or_query = '';
foreach ($filter as $e) {
if ($e instanceof FilterElement && isset($mappings[$e->getField()])) {
$mapping = $mappings[$e->getField()];
$mapping = $mappings[$e->getField()];
if ($mapping instanceof DoctrineJoinFilterMapping) {
$query = $mapping->applyOr($query, $e);
$condition = $mapping->applyOr($query, $e);
if(!empty($condition)) $condition .= ' OR ';
$sub_or_query .= $condition;
continue;
}
else if(is_array($mapping)){
$condition = '';
foreach ($mapping as $mapping_or){
$mapping_or = explode(':', $mapping_or);
$value = $e->getValue();
if (count($mapping_or) > 1) {
$value = $this->convertValue($value, $mapping_or[1]);
}
$mapping = explode(':', $mapping);
$value = $e->getValue();
if (count($mapping) > 1) {
$value = $this->convertValue($value, $mapping[1]);
if(!empty($condition)) $condition .= ' OR ';
$condition .= sprintf(" %s %s %s ", $mapping_or[0], $e->getOperator(), $value);
}
if(!empty($sub_or_query)) $sub_or_query .= ' OR ';
$sub_or_query .= ' ( '.$condition.' ) ';
}
else {
$mapping = explode(':', $mapping);
$value = $e->getValue();
$query->orWhere(sprintf("%s %s %s",$mapping[0], $e->getOperator(), $value));
if (count($mapping) > 1) {
$value = $this->convertValue($value, $mapping[1]);
}
if(!empty($sub_or_query)) $sub_or_query .= ' OR ';
$sub_or_query .= sprintf(" %s %s %s ", $mapping[0], $e->getOperator(), $value);
}
}
}
$query->andWhere($sub_or_query);
}
}
return $this;

12
app/Http/Utils/Filters/FilterParser.php
View File

@ -1,5 +1,4 @@
<?php namespace utils;
/**
* Copyright 2015 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
@ -22,8 +21,8 @@ final class FilterParser
*/
public static function parse($filters, $allowed_fields = array())
{
$res = array();
$matches = array();
$res = [];
$matches = [];
if (!is_array($filters))
$filters = array($filters);
@ -36,7 +35,7 @@ final class FilterParser
$or_filters = explode(',', $filter);
if (count($or_filters) > 1) {
$f = array();
$f = [];
foreach ($or_filters as $of) {
//single filter
@ -54,7 +53,7 @@ final class FilterParser
if (!in_array($op, $allowed_fields[$field])) continue;
$f_or = self::buildFilter($field, $op, $value);
if (!is_null($f_or))
array_push($f, $f_or);
$f[] = $f_or;
}
} else {
//single filter
@ -67,13 +66,14 @@ final class FilterParser
$operands = explode($op, $filter);
$field = $operands[0];
$value = $operands[1];
if (!isset($allowed_fields[$field])) continue;
if (!in_array($op, $allowed_fields[$field])) continue;
$f = self::buildFilter($field, $op, $value);
}
if (!is_null($f))
array_push($res, $f);
$res[] = $f;
}
return new Filter($res);
}

18
app/Http/routes.php
View File

@ -13,6 +13,24 @@
use Illuminate\Support\Facades\Config;
// public api ( without AUTHZ [OAUTH2.0])
Route::group([
'namespace' => 'App\Http\Controllers',
'prefix' => 'api/public/v1',
'before' => [],
'after' => [],
'middleware' => [
'ssl',
'rate.limit:100,1', // 100 request per minute
'etags'
]
], function(){
// members
Route::group(['prefix'=>'members'], function() {
Route::get('', 'OAuth2MembersApiController@getMembers');
});
});
//OAuth2 Protected API
Route::group([
'namespace' => 'App\Http\Controllers',

22
app/Models/ResourceServer/ApiEndpoint.php
View File

@ -235,6 +235,12 @@ class ApiEndpoint extends ResourceServerEntity implements IApiEndpoint
*/
private $rate_limit;
/**
* @ORM\Column(name="rate_limit_decay", type="integer")
* @var int
*/
private $rate_limit_decay;
/**
* ApiEndpoint constructor.
*/
@ -291,4 +297,20 @@ class ApiEndpoint extends ResourceServerEntity implements IApiEndpoint
$this->rate_limit = $rate_limit;
}
/**
* @return int
*/
public function getRateLimitDecay()
{
return $this->rate_limit_decay;
}
/**
* @param int $rate_limit_decay
*/
public function setRateLimitDecay($rate_limit_decay)
{
$this->rate_limit_decay = $rate_limit_decay;
}
}

161
app/Models/ResourceServer/EndpointRateLimitByIP.php Normal file
View File

@ -0,0 +1,161 @@
<?php namespace App\Models\ResourceServer;
/**
* Copyright 2017 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use Doctrine\Common\Collections\ArrayCollection;
use Doctrine\ORM\Mapping AS ORM;
use Illuminate\Support\Facades\Cache as CacheFacade;
use Illuminate\Support\Facades\Config;
/**
* @ORM\Entity(repositoryClass="repositories\resource_server\DoctrineEndPointRateLimitByIPRepository")
* @ORM\Table(name="ip_rate_limit_routes")
* Class EndPointRateLimitByIP
* @package App\Models\ResourceServer
*/
final class EndPointRateLimitByIP extends ResourceServerEntity
{
/**
* @ORM\Column(name="ip", type="string")
* @var string
*/
private $ip;
/**
* @return string
*/
public function getIp()
{
return $this->ip;
}
/**
* @param string $ip
*/
public function setIp($ip)
{
$this->ip = $ip;
}
/**
* @return bool
*/
public function isActive()
{
return $this->active;
}
/**
* @param bool $active
*/
public function setActive($active)
{
$this->active = $active;
}
/**
* @return string
*/
public function getRoute()
{
return $this->route;
}
/**
* @param string $route
*/
public function setRoute($route)
{
$this->route = $route;
}
/**
* @return string
*/
public function getHttpMethod()
{
return $this->http_method;
}
/**
* @param string $http_method
*/
public function setHttpMethod($http_method)
{
$this->http_method = $http_method;
}
/**
* @return int
*/
public function getRateLimit()
{
return $this->rate_limit;
}
/**
* @param int $rate_limit
*/
public function setRateLimit($rate_limit)
{
$this->rate_limit = $rate_limit;
}
/**
* @return int
*/
public function getRateLimitDecay()
{
return $this->rate_limit_decay;
}
/**
* @param int $rate_limit_decay
*/
public function setRateLimitDecay($rate_limit_decay)
{
$this->rate_limit_decay = $rate_limit_decay;
}
/**
* @ORM\Column(name="active", type="boolean")
* @var bool
*/
private $active;
/**
* @ORM\Column(name="route", type="string")
* @var string
*/
private $route;
/**
* @ORM\Column(name="http_method", type="string")
* @var string
*/
private $http_method;
/**
* @ORM\Column(name="rate_limit", type="integer")
* @var int
*/
private $rate_limit;
/**
* @ORM\Column(name="rate_limit_decay", type="integer")
* @var int
*/
private $rate_limit_decay;
}

5
app/Models/ResourceServer/IApiEndpoint.php
View File

@ -100,4 +100,9 @@ interface IApiEndpoint extends IEntity
*/
public function getRateLimit();
/**
* @return int
*/
public function getRateLimitDecay();
}

31
app/Models/ResourceServer/IEndpointRateLimitByIPRepository.php Normal file
View File

@ -0,0 +1,31 @@
<?php namespace App\Models\ResourceServer;
/**
* Copyright 2017 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use models\utils\IBaseRepository;
/**
* Interface IEndpointRateLimitByIPRepository
* @package App\Models\ResourceServer
*/
interface IEndpointRateLimitByIPRepository extends IBaseRepository
{
/**
* @param string $ip
* @param string $route
* @param string $http_method
* @return EndPointRateLimitByIP
*/
public function getByIPRouteMethod($ip, $route, $http_method);
}

7
app/Repositories/RepositoriesProvider.php
View File

@ -48,6 +48,12 @@ final class RepositoriesProvider extends ServiceProvider
return EntityManager::getRepository(\App\Models\ResourceServer\ApiEndpoint::class);
});
App::singleton(
'App\Models\ResourceServer\IEndpointRateLimitByIPRepository',
function(){
return EntityManager::getRepository(\App\Models\ResourceServer\EndPointRateLimitByIP::class);
});
App::singleton(
'models\summit\ISummitRepository',
function(){
@ -132,5 +138,6 @@ final class RepositoriesProvider extends ServiceProvider
function(){
return EntityManager::getRepository(\models\summit\RSVP::class);
});
}
}

53
app/Repositories/ResourceServer/DoctrineEndPointRateLimitByIPRepository.php Normal file
View File

@ -0,0 +1,53 @@
<?php namespace repositories\resource_server;
/**
* Copyright 2017 OpenStack Foundation
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* http://www.apache.org/licenses/LICENSE-2.0
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
**/
use App\Models\ResourceServer\EndPointRateLimitByIP;
use App\Models\ResourceServer\IEndpointRateLimitByIPRepository;
use repositories\DoctrineRepository;
use Illuminate\Support\Facades\Log;
/**
* Class DoctrineEndPointRateLimitByIPRepository
* @package repositories\resource_server
*/
final class DoctrineEndPointRateLimitByIPRepository
extends DoctrineRepository
implements IEndpointRateLimitByIPRepository
{
/**
* @param string $ip
* @param string $route
* @param string $http_method
* @return EndPointRateLimitByIP
*/
public function getByIPRouteMethod($ip, $route, $http_method)
{
try {
return $this->getEntityManager()->createQueryBuilder()
->select("c")
->from(\App\Models\ResourceServer\EndPointRateLimitByIP::class, "c")
->where('c.route = :route')
->andWhere('c.http_method = :http_method')
->andWhere('c.ip = :ip')
->andWhere('c.active = 1')
->setParameter('ip', trim($ip))
->setParameter('route', trim($route))
->setParameter('http_method', trim($http_method))
->getQuery()
->getOneOrNullResult();
}
catch(\Exception $ex){
Log::error($ex);
return null;
}
}
}

37
database/migrations/2017_06_01_143808_create_table_ip_rate_limit_route.php Normal file
View File

@ -0,0 +1,37 @@
<?php
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class CreateTableIpRateLimitRoute extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('ip_rate_limit_routes', function(Blueprint $table)
{
$table->bigIncrements('id');
$table->string('ip',255);
$table->text("route");
$table->boolean('active')->default(true);
$table->enum('http_method', array('GET', 'HEAD','POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'OPTIONS', 'PATCH'));
$table->bigInteger("rate_limit")->unsigned()->default(0);
$table->bigInteger("rate_limit_decay")->unsigned()->default(0);
$table->timestamps();
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('ip_rate_limit_routes');
}
}

45
database/migrations/2017_06_01_144624_update_table_api_endpoint.php Normal file
View File

@ -0,0 +1,45 @@
<?php
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Database\Migrations\Migration;
class UpdateTableApiEndpoint extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::table('api_endpoints', function(Blueprint $table)
{
$table->dropColumn("rate_limit");
});
Schema::table('api_endpoints', function(Blueprint $table)
{
$table->bigInteger("rate_limit")->unsigned()->default(0);
$table->bigInteger("rate_limit_decay")->unsigned()->default(0);
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::table('api_endpoints', function(Blueprint $table)
{
$table->dropColumn('rate_limit_decay');
$table->dropColumn("rate_limit");
});
Schema::table('api_endpoints', function(Blueprint $table)
{
$table->bigInteger("rate_limit")->unsigned()->nullable();
});
}
}

2
readme.md
View File

@ -21,6 +21,8 @@ run following commands on root folder
* php artisan doctrine:clear:query:cache
* php artisan doctrine:clear:result:cache
* php artisan doctrine:ensure:production
* php artisan route:clear
* php artisan route:cache
* give proper rights to storage folder (775 and proper users)
* chmod 777 vendor/ezyang/htmlpurifier/library/HTMLPurifier/DefinitionCache/Serializer

4
update_doctrine.sh
View File

@ -2,4 +2,6 @@
php artisan doctrine:generate:proxies
php artisan doctrine:clear:metadata:cache
php artisan doctrine:clear:query:cache
php artisan doctrine:clear:result:cache
php artisan doctrine:clear:result:cache
php artisan route:clear
php artisan route:cache