From 8d9968a1877baefabf19872dac5e16fb40938e50 Mon Sep 17 00:00:00 2001 From: Sebastian Marcet Date: Fri, 2 Jun 2017 14:21:05 -0300 Subject: [PATCH] added new public endpoint added new public endpoint api/public/v1/members also added api rate limit middleware Change-Id: I456e2a56f3afdc03b50cb771faf0cde7a373a18e --- .../main/OAuth2MembersApiController.php | 3 +- app/Http/Middleware/ETagsMiddleware.php | 5 +- app/Http/Middleware/RateLimitMiddleware.php | 125 +++++++------- .../Filters/DoctrineJoinFilterMapping.php | 4 +- app/Http/Utils/Filters/Filter.php | 37 +++- app/Http/Utils/Filters/FilterParser.php | 12 +- app/Http/routes.php | 18 ++ app/Models/ResourceServer/ApiEndpoint.php | 22 +++ .../ResourceServer/EndpointRateLimitByIP.php | 161 ++++++++++++++++++ app/Models/ResourceServer/IApiEndpoint.php | 5 + .../IEndpointRateLimitByIPRepository.php | 31 ++++ app/Repositories/RepositoriesProvider.php | 7 + ...octrineEndPointRateLimitByIPRepository.php | 53 ++++++ ...43808_create_table_ip_rate_limit_route.php | 37 ++++ ...06_01_144624_update_table_api_endpoint.php | 45 +++++ readme.md | 2 + update_doctrine.sh | 4 +- 17 files changed, 491 insertions(+), 80 deletions(-) create mode 100644 app/Models/ResourceServer/EndpointRateLimitByIP.php create mode 100644 app/Models/ResourceServer/IEndpointRateLimitByIPRepository.php create mode 100644 app/Repositories/ResourceServer/DoctrineEndPointRateLimitByIPRepository.php create mode 100644 database/migrations/2017_06_01_143808_create_table_ip_rate_limit_route.php create mode 100644 database/migrations/2017_06_01_144624_update_table_api_endpoint.php diff --git a/app/Http/Controllers/apis/protected/main/OAuth2MembersApiController.php b/app/Http/Controllers/apis/protected/main/OAuth2MembersApiController.php index 6e29cdbb..cc746037 100644 --- a/app/Http/Controllers/apis/protected/main/OAuth2MembersApiController.php +++ b/app/Http/Controllers/apis/protected/main/OAuth2MembersApiController.php @@ -40,7 +40,8 @@ final class OAuth2MembersApiController extends OAuth2ProtectedController ( IMemberRepository $member_repository, IResourceServerContext $resource_server_context - ) { + ) + { parent::__construct($resource_server_context); $this->repository = $member_repository; } diff --git a/app/Http/Middleware/ETagsMiddleware.php b/app/Http/Middleware/ETagsMiddleware.php index 35382f34..6355bb38 100644 --- a/app/Http/Middleware/ETagsMiddleware.php +++ b/app/Http/Middleware/ETagsMiddleware.php @@ -12,10 +12,8 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ - use Closure; use Illuminate\Support\Facades\Log; - /** * Class ETagsMiddleware * @package App\Http\Middleware @@ -34,13 +32,12 @@ final class ETagsMiddleware $response = $next($request); if ($response->getStatusCode() === 200 && $request->getMethod() === 'GET') { - $etag = md5($response->getContent()); + $etag = md5($response->getContent()); $requestETag = str_replace('"', '', $request->getETags()); $requestETag = str_replace('-gzip', '', $requestETag); if ($requestETag && $requestETag[0] == $etag) { - Log::debug('ETAG 304'); $response->setNotModified(); } $response->setEtag($etag); diff --git a/app/Http/Middleware/RateLimitMiddleware.php b/app/Http/Middleware/RateLimitMiddleware.php index 4d323abc..92cebcab 100644 --- a/app/Http/Middleware/RateLimitMiddleware.php +++ b/app/Http/Middleware/RateLimitMiddleware.php @@ -1,5 +1,4 @@ endpoint_repository = $endpoint_repository; - $this->cache_service = $cache_service; + parent::__construct($limiter); + $this->endpoint_repository = $endpoint_repository; + $this->endpoint_rate_limit_by_ip_repository = $endpoint_rate_limit_by_ip_repository; } /** - * Handle an incoming request. - * @param \Illuminate\Http\Request $request - * @param \Closure $next - * @return mixed + * @param \Illuminate\Http\Request $request + * @param Closure $next + * @param int $max_attempts + * @param int $decay_minutes + * @return \Illuminate\Http\Response|mixed */ - public function handle($request, Closure $next) + public function handle($request, Closure $next, $max_attempts = 0, $decay_minutes = 0) { + $route = RequestUtils::getCurrentRoutePath($request); + $method = $request->getMethod(); + $endpoint = $this->endpoint_repository->getApiEndpointByUrlAndMethod($route, $method); + $key = $this->resolveRequestSignature($request); + $client_ip = $request->getClientIp(); + + if (!is_null($endpoint) && $endpoint->getRateLimit() > 0) { + $max_attempts = $endpoint->getRateLimit(); + } + + if (!is_null($endpoint) && $endpoint->getRateLimitDecay() > 0) { + $decay_minutes = $endpoint->getRateLimitDecay(); + } + + $endpoint_rate_limit_by_ip = $this->endpoint_rate_limit_by_ip_repository->getByIPRouteMethod + ( + $client_ip, + $route, + $method + ); + + if(!is_null($endpoint_rate_limit_by_ip)){ + $max_attempts = $endpoint_rate_limit_by_ip->getRateLimit(); + $decay_minutes = $endpoint_rate_limit_by_ip->getRateLimitDecay(); + } + + if ($max_attempts == 0 || $decay_minutes == 0) { + // short circuit (infinite) + return $next($request); + } + + if ($this->limiter->tooManyAttempts($key, $max_attempts, $decay_minutes)) { + return $this->buildResponse($key, $max_attempts); + } + + $this->limiter->hit($key, $decay_minutes); + $response = $next($request); - // if response was not changed then short circuit ... - if ($response->getStatusCode() === 304) { - return $response; - } - $url = $request->getRequestUri(); - - try { - $route = RequestUtils::getCurrentRoutePath($request); - $method = $request->getMethod(); - $endpoint = $this->endpoint_repository->getApiEndpointByUrlAndMethod($route, $method); - - if (!is_null($endpoint->getRateLimit()) && ($requestsPerHour = (int)$endpoint->getRateLimit()) > 0) { - //do rate limit checking - $key = sprintf('rate.limit.%s_%s_%s', $url, $method, $request->getClientIp()); - // Add if doesn't exist - // Remember for 1 hour - $this->cache_service->addSingleValue($key, 0, 3600); - // Add to count - $count = $this->cache_service->incCounter($key); - if ($count > $requestsPerHour) { - // Short-circuit response - we're ignoring - $response = Response::json(array( - 'message' => "You have triggered an abuse detection mechanism and have been temporarily blocked. - Please retry your request again later." - ), 403); - $ttl = (int)$this->cache_service->ttl($key); - $response->headers->set('X-RateLimit-Reset', $ttl, false); - } - $response->headers->set('X-Ratelimit-Limit', $requestsPerHour, false); - $remaining = $requestsPerHour - (int)$count; - if ($remaining < 0) { - $remaining = 0; - } - $response->headers->set('X-Ratelimit-Remaining', $remaining, false); - } - } catch (Exception $ex) { - Log::error($ex); - } - - return $response; + return $this->addHeaders( + $response, $max_attempts, + $this->calculateRemainingAttempts($key, $max_attempts) + ); } } \ No newline at end of file diff --git a/app/Http/Utils/Filters/DoctrineJoinFilterMapping.php b/app/Http/Utils/Filters/DoctrineJoinFilterMapping.php index 0f45a31c..bc4e5b19 100644 --- a/app/Http/Utils/Filters/DoctrineJoinFilterMapping.php +++ b/app/Http/Utils/Filters/DoctrineJoinFilterMapping.php @@ -63,13 +63,13 @@ class DoctrineJoinFilterMapping extends FilterMapping /** * @param QueryBuilder $query * @param FilterElement $filter - * @return QueryBuilder + * @return string */ public function applyOr(QueryBuilder $query, FilterElement $filter){ $where = str_replace(":value", $filter->getValue(), $this->where); $where = str_replace(":operator", $filter->getOperator(), $where); if(!in_array($this->alias, $query->getAllAliases())) $query->innerJoin($this->table, $this->alias, Join::WITH); - return $query->orWhere($where); + return $where; } } \ No newline at end of file diff --git a/app/Http/Utils/Filters/Filter.php b/app/Http/Utils/Filters/Filter.php index 50cb4d89..c9ae2802 100644 --- a/app/Http/Utils/Filters/Filter.php +++ b/app/Http/Utils/Filters/Filter.php @@ -190,25 +190,46 @@ final class Filter } else if (is_array($filter)) { // OR + $sub_or_query = ''; foreach ($filter as $e) { if ($e instanceof FilterElement && isset($mappings[$e->getField()])) { - $mapping = $mappings[$e->getField()]; + $mapping = $mappings[$e->getField()]; if ($mapping instanceof DoctrineJoinFilterMapping) { - $query = $mapping->applyOr($query, $e); + $condition = $mapping->applyOr($query, $e); + if(!empty($condition)) $condition .= ' OR '; + $sub_or_query .= $condition; continue; } + else if(is_array($mapping)){ + $condition = ''; + foreach ($mapping as $mapping_or){ + $mapping_or = explode(':', $mapping_or); + $value = $e->getValue(); + if (count($mapping_or) > 1) { + $value = $this->convertValue($value, $mapping_or[1]); + } - $mapping = explode(':', $mapping); - $value = $e->getValue(); - - if (count($mapping) > 1) { - $value = $this->convertValue($value, $mapping[1]); + if(!empty($condition)) $condition .= ' OR '; + $condition .= sprintf(" %s %s %s ", $mapping_or[0], $e->getOperator(), $value); + } + if(!empty($sub_or_query)) $sub_or_query .= ' OR '; + $sub_or_query .= ' ( '.$condition.' ) '; } + else { + $mapping = explode(':', $mapping); + $value = $e->getValue(); - $query->orWhere(sprintf("%s %s %s",$mapping[0], $e->getOperator(), $value)); + if (count($mapping) > 1) { + $value = $this->convertValue($value, $mapping[1]); + } + + if(!empty($sub_or_query)) $sub_or_query .= ' OR '; + $sub_or_query .= sprintf(" %s %s %s ", $mapping[0], $e->getOperator(), $value); + } } } + $query->andWhere($sub_or_query); } } return $this; diff --git a/app/Http/Utils/Filters/FilterParser.php b/app/Http/Utils/Filters/FilterParser.php index 3b5b8104..1a2f5a32 100644 --- a/app/Http/Utils/Filters/FilterParser.php +++ b/app/Http/Utils/Filters/FilterParser.php @@ -1,5 +1,4 @@ 1) { - $f = array(); + $f = []; foreach ($or_filters as $of) { //single filter @@ -54,7 +53,7 @@ final class FilterParser if (!in_array($op, $allowed_fields[$field])) continue; $f_or = self::buildFilter($field, $op, $value); if (!is_null($f_or)) - array_push($f, $f_or); + $f[] = $f_or; } } else { //single filter @@ -67,13 +66,14 @@ final class FilterParser $operands = explode($op, $filter); $field = $operands[0]; $value = $operands[1]; + if (!isset($allowed_fields[$field])) continue; if (!in_array($op, $allowed_fields[$field])) continue; $f = self::buildFilter($field, $op, $value); } if (!is_null($f)) - array_push($res, $f); + $res[] = $f; } return new Filter($res); } diff --git a/app/Http/routes.php b/app/Http/routes.php index 5ecc4333..74b0bb88 100644 --- a/app/Http/routes.php +++ b/app/Http/routes.php @@ -13,6 +13,24 @@ use Illuminate\Support\Facades\Config; +// public api ( without AUTHZ [OAUTH2.0]) +Route::group([ + 'namespace' => 'App\Http\Controllers', + 'prefix' => 'api/public/v1', + 'before' => [], + 'after' => [], + 'middleware' => [ + 'ssl', + 'rate.limit:100,1', // 100 request per minute + 'etags' + ] +], function(){ + // members + Route::group(['prefix'=>'members'], function() { + Route::get('', 'OAuth2MembersApiController@getMembers'); + }); +}); + //OAuth2 Protected API Route::group([ 'namespace' => 'App\Http\Controllers', diff --git a/app/Models/ResourceServer/ApiEndpoint.php b/app/Models/ResourceServer/ApiEndpoint.php index 8b16b2f7..5a5602cc 100644 --- a/app/Models/ResourceServer/ApiEndpoint.php +++ b/app/Models/ResourceServer/ApiEndpoint.php @@ -235,6 +235,12 @@ class ApiEndpoint extends ResourceServerEntity implements IApiEndpoint */ private $rate_limit; + /** + * @ORM\Column(name="rate_limit_decay", type="integer") + * @var int + */ + private $rate_limit_decay; + /** * ApiEndpoint constructor. */ @@ -291,4 +297,20 @@ class ApiEndpoint extends ResourceServerEntity implements IApiEndpoint $this->rate_limit = $rate_limit; } + /** + * @return int + */ + public function getRateLimitDecay() + { + return $this->rate_limit_decay; + } + + /** + * @param int $rate_limit_decay + */ + public function setRateLimitDecay($rate_limit_decay) + { + $this->rate_limit_decay = $rate_limit_decay; + } + } \ No newline at end of file diff --git a/app/Models/ResourceServer/EndpointRateLimitByIP.php b/app/Models/ResourceServer/EndpointRateLimitByIP.php new file mode 100644 index 00000000..7b89a247 --- /dev/null +++ b/app/Models/ResourceServer/EndpointRateLimitByIP.php @@ -0,0 +1,161 @@ +ip; + } + + /** + * @param string $ip + */ + public function setIp($ip) + { + $this->ip = $ip; + } + + /** + * @return bool + */ + public function isActive() + { + return $this->active; + } + + /** + * @param bool $active + */ + public function setActive($active) + { + $this->active = $active; + } + + /** + * @return string + */ + public function getRoute() + { + return $this->route; + } + + /** + * @param string $route + */ + public function setRoute($route) + { + $this->route = $route; + } + + /** + * @return string + */ + public function getHttpMethod() + { + return $this->http_method; + } + + /** + * @param string $http_method + */ + public function setHttpMethod($http_method) + { + $this->http_method = $http_method; + } + + /** + * @return int + */ + public function getRateLimit() + { + return $this->rate_limit; + } + + /** + * @param int $rate_limit + */ + public function setRateLimit($rate_limit) + { + $this->rate_limit = $rate_limit; + } + + /** + * @return int + */ + public function getRateLimitDecay() + { + return $this->rate_limit_decay; + } + + /** + * @param int $rate_limit_decay + */ + public function setRateLimitDecay($rate_limit_decay) + { + $this->rate_limit_decay = $rate_limit_decay; + } + + /** + * @ORM\Column(name="active", type="boolean") + * @var bool + */ + private $active; + + /** + * @ORM\Column(name="route", type="string") + * @var string + */ + private $route; + + /** + * @ORM\Column(name="http_method", type="string") + * @var string + */ + private $http_method; + + /** + * @ORM\Column(name="rate_limit", type="integer") + * @var int + */ + private $rate_limit; + + /** + * @ORM\Column(name="rate_limit_decay", type="integer") + * @var int + */ + private $rate_limit_decay; + +} \ No newline at end of file diff --git a/app/Models/ResourceServer/IApiEndpoint.php b/app/Models/ResourceServer/IApiEndpoint.php index 8ccced33..502360f2 100644 --- a/app/Models/ResourceServer/IApiEndpoint.php +++ b/app/Models/ResourceServer/IApiEndpoint.php @@ -100,4 +100,9 @@ interface IApiEndpoint extends IEntity */ public function getRateLimit(); + /** + * @return int + */ + public function getRateLimitDecay(); + } \ No newline at end of file diff --git a/app/Models/ResourceServer/IEndpointRateLimitByIPRepository.php b/app/Models/ResourceServer/IEndpointRateLimitByIPRepository.php new file mode 100644 index 00000000..d346b312 --- /dev/null +++ b/app/Models/ResourceServer/IEndpointRateLimitByIPRepository.php @@ -0,0 +1,31 @@ +getEntityManager()->createQueryBuilder() + ->select("c") + ->from(\App\Models\ResourceServer\EndPointRateLimitByIP::class, "c") + ->where('c.route = :route') + ->andWhere('c.http_method = :http_method') + ->andWhere('c.ip = :ip') + ->andWhere('c.active = 1') + ->setParameter('ip', trim($ip)) + ->setParameter('route', trim($route)) + ->setParameter('http_method', trim($http_method)) + ->getQuery() + ->getOneOrNullResult(); + } + catch(\Exception $ex){ + Log::error($ex); + return null; + } + } +} \ No newline at end of file diff --git a/database/migrations/2017_06_01_143808_create_table_ip_rate_limit_route.php b/database/migrations/2017_06_01_143808_create_table_ip_rate_limit_route.php new file mode 100644 index 00000000..474ca5cf --- /dev/null +++ b/database/migrations/2017_06_01_143808_create_table_ip_rate_limit_route.php @@ -0,0 +1,37 @@ +bigIncrements('id'); + $table->string('ip',255); + $table->text("route"); + $table->boolean('active')->default(true); + $table->enum('http_method', array('GET', 'HEAD','POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'OPTIONS', 'PATCH')); + $table->bigInteger("rate_limit")->unsigned()->default(0); + $table->bigInteger("rate_limit_decay")->unsigned()->default(0); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::drop('ip_rate_limit_routes'); + } +} diff --git a/database/migrations/2017_06_01_144624_update_table_api_endpoint.php b/database/migrations/2017_06_01_144624_update_table_api_endpoint.php new file mode 100644 index 00000000..5deb1209 --- /dev/null +++ b/database/migrations/2017_06_01_144624_update_table_api_endpoint.php @@ -0,0 +1,45 @@ +dropColumn("rate_limit"); + }); + + Schema::table('api_endpoints', function(Blueprint $table) + { + $table->bigInteger("rate_limit")->unsigned()->default(0); + $table->bigInteger("rate_limit_decay")->unsigned()->default(0); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('api_endpoints', function(Blueprint $table) + { + $table->dropColumn('rate_limit_decay'); + $table->dropColumn("rate_limit"); + }); + + Schema::table('api_endpoints', function(Blueprint $table) + { + $table->bigInteger("rate_limit")->unsigned()->nullable(); + }); + } +} diff --git a/readme.md b/readme.md index e336bf94..2e6a29db 100644 --- a/readme.md +++ b/readme.md @@ -21,6 +21,8 @@ run following commands on root folder * php artisan doctrine:clear:query:cache * php artisan doctrine:clear:result:cache * php artisan doctrine:ensure:production + * php artisan route:clear + * php artisan route:cache * give proper rights to storage folder (775 and proper users) * chmod 777 vendor/ezyang/htmlpurifier/library/HTMLPurifier/DefinitionCache/Serializer diff --git a/update_doctrine.sh b/update_doctrine.sh index 09c6d263..5019b3f8 100755 --- a/update_doctrine.sh +++ b/update_doctrine.sh @@ -2,4 +2,6 @@ php artisan doctrine:generate:proxies php artisan doctrine:clear:metadata:cache php artisan doctrine:clear:query:cache -php artisan doctrine:clear:result:cache \ No newline at end of file +php artisan doctrine:clear:result:cache +php artisan route:clear +php artisan route:cache \ No newline at end of file