added new public endpoint

added new public endpoint api/public/v1/members
also added api rate limit middleware

Change-Id: I456e2a56f3afdc03b50cb771faf0cde7a373a18e

app/Http/Controllers/apis/protected/main/OAuth2MembersApiController.php

@@ -40,7 +40,8 @@ final class OAuth2MembersApiController extends OAuth2ProtectedController
     (
         IMemberRepository $member_repository,
         IResourceServerContext $resource_server_context
-    ) {
+    )
+    {
         parent::__construct($resource_server_context);
         $this->repository = $member_repository;
     }

app/Http/Middleware/ETagsMiddleware.php

@@ -12,10 +12,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  **/
-
 use Closure;
 use Illuminate\Support\Facades\Log;
-
 /**
  * Class ETagsMiddleware
  * @package App\Http\Middleware
@@ -34,13 +32,12 @@ final class ETagsMiddleware
         $response = $next($request);
         if ($response->getStatusCode() === 200 && $request->getMethod() === 'GET')
         {
-            $etag = md5($response->getContent());
+            $etag        = md5($response->getContent());
             $requestETag = str_replace('"', '', $request->getETags());
             $requestETag = str_replace('-gzip', '', $requestETag);
 
             if ($requestETag && $requestETag[0] == $etag)
             {
-                Log::debug('ETAG 304');
                 $response->setNotModified();
             }
             $response->setEtag($etag);

app/Http/Middleware/RateLimitMiddleware.php

@@ -1,5 +1,4 @@
 <?php namespace App\Http\Middleware;
-
 /**
  * Copyright 2015 OpenStack Foundation
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,18 +11,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  **/
-
-use Closure;
-use Illuminate\Support\Facades\Response;
-use libs\utils\ICacheService;
-use libs\utils\RequestUtils;
 use App\Models\ResourceServer\IApiEndpointRepository;
+use App\Models\ResourceServer\IEndpointRateLimitByIPRepository;
+use Closure;
+use Illuminate\Cache\RateLimiter;
+use Illuminate\Routing\Middleware\ThrottleRequests;
+use libs\utils\RequestUtils;
 
 /**
  * Class RateLimitMiddleware
  * @package App\Http\Middleware
  */
-final class RateLimitMiddleware
+final class RateLimitMiddleware extends ThrottleRequests
 {
 
     /**
@@ -32,69 +31,79 @@ final class RateLimitMiddleware
     private $endpoint_repository;
 
     /**
-     * @var ICacheService
+     * @var IEndpointRateLimitByIPRepository
      */
-    private $cache_service;
+    private $endpoint_rate_limit_by_ip_repository;
 
     /**
+     * RateLimitMiddleware constructor.
      * @param IApiEndpointRepository $endpoint_repository
-     * @param ICacheService $cache_service
+     * @param IEndpointRateLimitByIPRepository $endpoint_rate_limit_by_ip_repository
+     * @param RateLimiter $limiter
      */
-    public function __construct(IApiEndpointRepository $endpoint_repository, ICacheService $cache_service)
+    public function __construct
+    (
+        IApiEndpointRepository $endpoint_repository,
+        IEndpointRateLimitByIPRepository $endpoint_rate_limit_by_ip_repository,
+        RateLimiter $limiter
+    )
     {
-        $this->endpoint_repository = $endpoint_repository;
+        parent::__construct($limiter);
-        $this->cache_service       = $cache_service;
+        $this->endpoint_repository                  = $endpoint_repository;
+        $this->endpoint_rate_limit_by_ip_repository = $endpoint_rate_limit_by_ip_repository;
     }
 
     /**
-     * Handle an incoming request.
+     * @param \Illuminate\Http\Request $request
-     * @param  \Illuminate\Http\Request $request
+     * @param Closure $next
-     * @param  \Closure $next
+     * @param int $max_attempts
-     * @return mixed
+     * @param int $decay_minutes
+     * @return \Illuminate\Http\Response|mixed
      */
-    public function handle($request, Closure $next)
+    public function handle($request, Closure $next, $max_attempts = 0, $decay_minutes = 0)
     {
+        $route     = RequestUtils::getCurrentRoutePath($request);
+        $method    = $request->getMethod();
+        $endpoint  = $this->endpoint_repository->getApiEndpointByUrlAndMethod($route, $method);
+        $key       = $this->resolveRequestSignature($request);
+        $client_ip = $request->getClientIp();
+
+        if (!is_null($endpoint) && $endpoint->getRateLimit() > 0) {
+            $max_attempts = $endpoint->getRateLimit();
+        }
+
+        if (!is_null($endpoint) && $endpoint->getRateLimitDecay() > 0) {
+            $decay_minutes = $endpoint->getRateLimitDecay();
+        }
+
+        $endpoint_rate_limit_by_ip = $this->endpoint_rate_limit_by_ip_repository->getByIPRouteMethod
+        (
+            $client_ip,
+            $route,
+            $method
+        );
+
+        if(!is_null($endpoint_rate_limit_by_ip)){
+            $max_attempts  = $endpoint_rate_limit_by_ip->getRateLimit();
+            $decay_minutes = $endpoint_rate_limit_by_ip->getRateLimitDecay();
+        }
+
+        if ($max_attempts == 0 || $decay_minutes == 0) {
+            // short circuit (infinite)
+            return $next($request);
+        }
+
+        if ($this->limiter->tooManyAttempts($key, $max_attempts, $decay_minutes)) {
+            return $this->buildResponse($key, $max_attempts);
+        }
+
+        $this->limiter->hit($key, $decay_minutes);
+
         $response = $next($request);
-        // if response was not changed then short circuit ...
-        if ($response->getStatusCode() === 304) {
-            return $response;
-        }
 
-        $url = $request->getRequestUri();
+        return $this->addHeaders(
-
+            $response, $max_attempts,
-        try {
+            $this->calculateRemainingAttempts($key, $max_attempts)
-            $route    = RequestUtils::getCurrentRoutePath($request);
+        );
-            $method   = $request->getMethod();
-            $endpoint = $this->endpoint_repository->getApiEndpointByUrlAndMethod($route, $method);
-
-            if (!is_null($endpoint->getRateLimit()) && ($requestsPerHour = (int)$endpoint->getRateLimit()) > 0) {
-                //do rate limit checking
-                $key = sprintf('rate.limit.%s_%s_%s', $url, $method, $request->getClientIp());
-                // Add if doesn't exist
-                // Remember for 1 hour
-                $this->cache_service->addSingleValue($key, 0, 3600);
-                // Add to count
-                $count = $this->cache_service->incCounter($key);
-                if ($count > $requestsPerHour) {
-                    // Short-circuit response - we're ignoring
-                    $response = Response::json(array(
-                        'message' => "You have triggered an abuse detection mechanism and have been temporarily blocked.
-                         Please retry your request again later."
-                    ), 403);
-                    $ttl = (int)$this->cache_service->ttl($key);
-                    $response->headers->set('X-RateLimit-Reset', $ttl, false);
-                }
-                $response->headers->set('X-Ratelimit-Limit', $requestsPerHour, false);
-                $remaining = $requestsPerHour - (int)$count;
-                if ($remaining < 0) {
-                    $remaining = 0;
-                }
-                $response->headers->set('X-Ratelimit-Remaining', $remaining, false);
-            }
-        } catch (Exception $ex) {
-            Log::error($ex);
-        }
-
-        return $response;
     }
 }

app/Http/Utils/Filters/DoctrineJoinFilterMapping.php

@@ -63,13 +63,13 @@ class DoctrineJoinFilterMapping extends FilterMapping
     /**
      * @param QueryBuilder $query
      * @param FilterElement $filter
-     * @return QueryBuilder
+     * @return string
      */
     public function applyOr(QueryBuilder $query, FilterElement $filter){
         $where = str_replace(":value", $filter->getValue(), $this->where);
         $where = str_replace(":operator", $filter->getOperator(), $where);
         if(!in_array($this->alias, $query->getAllAliases()))
             $query->innerJoin($this->table, $this->alias, Join::WITH);
-        return $query->orWhere($where);
+        return $where;
     }
 }

app/Http/Utils/Filters/Filter.php

@@ -190,25 +190,46 @@ final class Filter
             }
             else if (is_array($filter)) {
                 // OR
+                $sub_or_query = '';
                 foreach ($filter as $e) {
                     if ($e instanceof FilterElement && isset($mappings[$e->getField()])) {
-                        $mapping = $mappings[$e->getField()];
 
+                        $mapping = $mappings[$e->getField()];
                         if ($mapping instanceof DoctrineJoinFilterMapping) {
-                            $query = $mapping->applyOr($query, $e);
+                            $condition = $mapping->applyOr($query, $e);
+                            if(!empty($condition)) $condition .= ' OR ';
+                            $sub_or_query .= $condition;
                             continue;
                         }
+                        else if(is_array($mapping)){
+                            $condition = '';
+                            foreach ($mapping as $mapping_or){
+                                $mapping_or = explode(':', $mapping_or);
+                                $value      = $e->getValue();
+                                if (count($mapping_or) > 1) {
+                                    $value = $this->convertValue($value, $mapping_or[1]);
+                                }
 
-                        $mapping  = explode(':', $mapping);
+                                if(!empty($condition)) $condition .= ' OR ';
-                        $value    = $e->getValue();
+                                $condition .= sprintf(" %s %s %s ", $mapping_or[0], $e->getOperator(), $value);
-
+                            }
-                        if (count($mapping) > 1) {
+                            if(!empty($sub_or_query)) $sub_or_query .= ' OR ';
-                            $value = $this->convertValue($value, $mapping[1]);
+                            $sub_or_query .= ' ( '.$condition.' ) ';
                         }
+                        else {
+                            $mapping = explode(':', $mapping);
+                            $value = $e->getValue();
 
-                        $query->orWhere(sprintf("%s %s %s",$mapping[0], $e->getOperator(), $value));
+                            if (count($mapping) > 1) {
+                                $value = $this->convertValue($value, $mapping[1]);
+                            }
+
+                            if(!empty($sub_or_query)) $sub_or_query .= ' OR ';
+                            $sub_or_query .= sprintf(" %s %s %s ", $mapping[0], $e->getOperator(), $value);
+                        }
                     }
                 }
+                $query->andWhere($sub_or_query);
             }
         }
         return $this;

app/Http/Utils/Filters/FilterParser.php

@@ -1,5 +1,4 @@
 <?php namespace utils;
-
 /**
  * Copyright 2015 OpenStack Foundation
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,8 +21,8 @@ final class FilterParser
      */
     public static function parse($filters, $allowed_fields = array())
     {
-        $res = array();
+        $res     = [];
-        $matches = array();
+        $matches = [];
 
         if (!is_array($filters))
             $filters = array($filters);
@@ -36,7 +35,7 @@ final class FilterParser
             $or_filters = explode(',', $filter);
 
             if (count($or_filters) > 1) {
-                $f = array();
+                $f = [];
                 foreach ($or_filters as $of) {
 
                     //single filter
@@ -54,7 +53,7 @@ final class FilterParser
                     if (!in_array($op, $allowed_fields[$field])) continue;
                     $f_or = self::buildFilter($field, $op, $value);
                     if (!is_null($f_or))
-                        array_push($f, $f_or);
+                        $f[] = $f_or;
                 }
             } else {
                 //single filter
@@ -67,13 +66,14 @@ final class FilterParser
                 $operands = explode($op, $filter);
                 $field    = $operands[0];
                 $value    = $operands[1];
+
                 if (!isset($allowed_fields[$field])) continue;
                 if (!in_array($op, $allowed_fields[$field])) continue;
                 $f = self::buildFilter($field, $op, $value);
             }
 
             if (!is_null($f))
-                array_push($res, $f);
+                $res[] = $f;
         }
         return new Filter($res);
     }

app/Http/routes.php

@@ -13,6 +13,24 @@
 
 use Illuminate\Support\Facades\Config;
 
+// public api ( without AUTHZ [OAUTH2.0])
+Route::group([
+    'namespace' => 'App\Http\Controllers',
+    'prefix'     => 'api/public/v1',
+    'before'     => [],
+    'after'      => [],
+    'middleware' => [
+        'ssl',
+        'rate.limit:100,1', // 100 request per minute
+        'etags'
+    ]
+], function(){
+    // members
+    Route::group(['prefix'=>'members'], function() {
+        Route::get('', 'OAuth2MembersApiController@getMembers');
+    });
+});
+
 //OAuth2 Protected API
 Route::group([
     'namespace' => 'App\Http\Controllers',

app/Models/ResourceServer/ApiEndpoint.php

@@ -235,6 +235,12 @@ class ApiEndpoint extends ResourceServerEntity implements IApiEndpoint
      */
     private $rate_limit;
 
+    /**
+     * @ORM\Column(name="rate_limit_decay", type="integer")
+     * @var int
+     */
+    private $rate_limit_decay;
+
     /**
      * ApiEndpoint constructor.
      */
@@ -291,4 +297,20 @@ class ApiEndpoint extends ResourceServerEntity implements IApiEndpoint
         $this->rate_limit = $rate_limit;
     }
 
+    /**
+     * @return int
+     */
+    public function getRateLimitDecay()
+    {
+        return $this->rate_limit_decay;
+    }
+
+    /**
+     * @param int $rate_limit_decay
+     */
+    public function setRateLimitDecay($rate_limit_decay)
+    {
+        $this->rate_limit_decay = $rate_limit_decay;
+    }
+
 }

app/Models/ResourceServer/EndpointRateLimitByIP.php

@@ -0,0 +1,161 @@
+<?php namespace App\Models\ResourceServer;
+/**
+ * Copyright 2017 OpenStack Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+use Doctrine\Common\Collections\ArrayCollection;
+use Doctrine\ORM\Mapping AS ORM;
+use Illuminate\Support\Facades\Cache as CacheFacade;
+use Illuminate\Support\Facades\Config;
+
+/**
+ * @ORM\Entity(repositoryClass="repositories\resource_server\DoctrineEndPointRateLimitByIPRepository")
+ * @ORM\Table(name="ip_rate_limit_routes")
+ * Class EndPointRateLimitByIP
+ * @package App\Models\ResourceServer
+ */
+final class EndPointRateLimitByIP extends ResourceServerEntity
+{
+
+    /**
+     * @ORM\Column(name="ip", type="string")
+     * @var string
+     */
+    private $ip;
+
+    /**
+     * @return string
+     */
+    public function getIp()
+    {
+        return $this->ip;
+    }
+
+    /**
+     * @param string $ip
+     */
+    public function setIp($ip)
+    {
+        $this->ip = $ip;
+    }
+
+    /**
+     * @return bool
+     */
+    public function isActive()
+    {
+        return $this->active;
+    }
+
+    /**
+     * @param bool $active
+     */
+    public function setActive($active)
+    {
+        $this->active = $active;
+    }
+
+    /**
+     * @return string
+     */
+    public function getRoute()
+    {
+        return $this->route;
+    }
+
+    /**
+     * @param string $route
+     */
+    public function setRoute($route)
+    {
+        $this->route = $route;
+    }
+
+    /**
+     * @return string
+     */
+    public function getHttpMethod()
+    {
+        return $this->http_method;
+    }
+
+    /**
+     * @param string $http_method
+     */
+    public function setHttpMethod($http_method)
+    {
+        $this->http_method = $http_method;
+    }
+
+    /**
+     * @return int
+     */
+    public function getRateLimit()
+    {
+        return $this->rate_limit;
+    }
+
+    /**
+     * @param int $rate_limit
+     */
+    public function setRateLimit($rate_limit)
+    {
+        $this->rate_limit = $rate_limit;
+    }
+
+    /**
+     * @return int
+     */
+    public function getRateLimitDecay()
+    {
+        return $this->rate_limit_decay;
+    }
+
+    /**
+     * @param int $rate_limit_decay
+     */
+    public function setRateLimitDecay($rate_limit_decay)
+    {
+        $this->rate_limit_decay = $rate_limit_decay;
+    }
+
+    /**
+     * @ORM\Column(name="active", type="boolean")
+     * @var bool
+     */
+    private $active;
+
+    /**
+     * @ORM\Column(name="route", type="string")
+     * @var string
+     */
+    private $route;
+
+    /**
+     * @ORM\Column(name="http_method", type="string")
+     * @var string
+     */
+    private $http_method;
+
+    /**
+     * @ORM\Column(name="rate_limit", type="integer")
+     * @var int
+     */
+    private $rate_limit;
+
+    /**
+     * @ORM\Column(name="rate_limit_decay", type="integer")
+     * @var int
+     */
+    private $rate_limit_decay;
+
+}

app/Models/ResourceServer/IApiEndpoint.php

@@ -100,4 +100,9 @@ interface IApiEndpoint extends IEntity
      */
     public function getRateLimit();
 
+    /**
+     * @return int
+     */
+    public function getRateLimitDecay();
+
 }

app/Models/ResourceServer/IEndpointRateLimitByIPRepository.php

@@ -0,0 +1,31 @@
+<?php namespace App\Models\ResourceServer;
+
+/**
+ * Copyright 2017 OpenStack Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+
+use models\utils\IBaseRepository;
+
+/**
+ * Interface IEndpointRateLimitByIPRepository
+ * @package App\Models\ResourceServer
+ */
+interface IEndpointRateLimitByIPRepository extends IBaseRepository
+{
+    /**
+     * @param string $ip
+     * @param string $route
+     * @param string $http_method
+     * @return EndPointRateLimitByIP
+     */
+    public function getByIPRouteMethod($ip, $route, $http_method);
+}

app/Repositories/RepositoriesProvider.php

@@ -48,6 +48,12 @@ final class RepositoriesProvider extends ServiceProvider
                 return  EntityManager::getRepository(\App\Models\ResourceServer\ApiEndpoint::class);
         });
 
+        App::singleton(
+            'App\Models\ResourceServer\IEndpointRateLimitByIPRepository',
+            function(){
+                return  EntityManager::getRepository(\App\Models\ResourceServer\EndPointRateLimitByIP::class);
+            });
+
         App::singleton(
             'models\summit\ISummitRepository',
             function(){
@@ -132,5 +138,6 @@ final class RepositoriesProvider extends ServiceProvider
             function(){
                 return  EntityManager::getRepository(\models\summit\RSVP::class);
             });
+
     }
 }

app/Repositories/ResourceServer/DoctrineEndPointRateLimitByIPRepository.php

@@ -0,0 +1,53 @@
+<?php namespace repositories\resource_server;
+/**
+ * Copyright 2017 OpenStack Foundation
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ **/
+use App\Models\ResourceServer\EndPointRateLimitByIP;
+use App\Models\ResourceServer\IEndpointRateLimitByIPRepository;
+use repositories\DoctrineRepository;
+use Illuminate\Support\Facades\Log;
+/**
+ * Class DoctrineEndPointRateLimitByIPRepository
+ * @package repositories\resource_server
+ */
+final class DoctrineEndPointRateLimitByIPRepository
+    extends DoctrineRepository
+    implements IEndpointRateLimitByIPRepository
+{
+    /**
+     * @param string $ip
+     * @param string $route
+     * @param string $http_method
+     * @return EndPointRateLimitByIP
+     */
+    public function getByIPRouteMethod($ip, $route, $http_method)
+    {
+        try {
+            return $this->getEntityManager()->createQueryBuilder()
+                ->select("c")
+                ->from(\App\Models\ResourceServer\EndPointRateLimitByIP::class, "c")
+                ->where('c.route = :route')
+                ->andWhere('c.http_method = :http_method')
+                ->andWhere('c.ip = :ip')
+                ->andWhere('c.active = 1')
+                ->setParameter('ip', trim($ip))
+                ->setParameter('route', trim($route))
+                ->setParameter('http_method', trim($http_method))
+                ->getQuery()
+                ->getOneOrNullResult();
+        }
+        catch(\Exception $ex){
+            Log::error($ex);
+            return null;
+        }
+    }
+}

database/migrations/2017_06_01_143808_create_table_ip_rate_limit_route.php

@@ -0,0 +1,37 @@
+<?php
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class CreateTableIpRateLimitRoute extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::create('ip_rate_limit_routes', function(Blueprint $table)
+        {
+            $table->bigIncrements('id');
+            $table->string('ip',255);
+            $table->text("route");
+            $table->boolean('active')->default(true);
+            $table->enum('http_method', array('GET', 'HEAD','POST', 'PUT', 'DELETE', 'TRACE', 'CONNECT', 'OPTIONS', 'PATCH'));
+            $table->bigInteger("rate_limit")->unsigned()->default(0);
+            $table->bigInteger("rate_limit_decay")->unsigned()->default(0);
+            $table->timestamps();
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::drop('ip_rate_limit_routes');
+    }
+}

database/migrations/2017_06_01_144624_update_table_api_endpoint.php

@@ -0,0 +1,45 @@
+<?php
+
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Database\Migrations\Migration;
+
+class UpdateTableApiEndpoint extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        Schema::table('api_endpoints', function(Blueprint $table)
+        {
+            $table->dropColumn("rate_limit");
+        });
+
+        Schema::table('api_endpoints', function(Blueprint $table)
+        {
+            $table->bigInteger("rate_limit")->unsigned()->default(0);
+            $table->bigInteger("rate_limit_decay")->unsigned()->default(0);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+        Schema::table('api_endpoints', function(Blueprint $table)
+        {
+            $table->dropColumn('rate_limit_decay');
+            $table->dropColumn("rate_limit");
+        });
+
+        Schema::table('api_endpoints', function(Blueprint $table)
+        {
+            $table->bigInteger("rate_limit")->unsigned()->nullable();
+        });
+    }
+}

readme.md

@@ -21,6 +21,8 @@ run following commands on root folder
    * php artisan doctrine:clear:query:cache
    * php artisan doctrine:clear:result:cache
    * php artisan doctrine:ensure:production
+   * php artisan route:clear
+   * php artisan route:cache
    * give proper rights to storage folder (775 and proper users)
    * chmod 777 vendor/ezyang/htmlpurifier/library/HTMLPurifier/DefinitionCache/Serializer
    

update_doctrine.sh

@@ -2,4 +2,6 @@
 php artisan doctrine:generate:proxies
 php artisan doctrine:clear:metadata:cache
 php artisan doctrine:clear:query:cache
-php artisan doctrine:clear:result:cache
+php artisan doctrine:clear:result:cache
+php artisan route:clear
+php artisan route:cache