feat: Custom rate limit factor per user

Author: kumy

website/app-templates/smarty/admin/pages/api_rate_limit.tpl

@@ -2,7 +2,6 @@
 
 {block name=title}{t}Api Rate Limits{/t}{/block}
 
-{assign var=RATES_LIMITS value=constant('GK_RATE_LIMITS')}
 {block name=content}
     <h1>
         {t}Api Rate Limits{/t}
@@ -15,20 +14,24 @@
         <thead>
         <tr>
             <th>Group</th>
-            <th>ID</th>
-            <th class="text-right">Count</th>
+            <th>Key</th>
+            <th class="text-center">Tier</th>
+            <th class="text-right">Used</th>
             <th>Limit</th>
+            <th class="text-right">Left</th>
             <th class="text-right">Actions</th>
         </tr>
         </thead>
         <tbody>
-            {foreach from=$current key=group item=item}
-            {foreach from=$item key=key item=value}
+            {foreach from=$current key=group item=rows}
+            {foreach from=$rows key=wire_key item=row}
                 <tr>
-                    <td>{$group} ({$RATES_LIMITS[$group][1]})</td>
-                    <td>{$key}</td>
-                    <td class="text-right">{$value}</td>
-                    <td>/{$RATES_LIMITS[$group][0]}</td>
+                    <td>{$group} ({$row.period})</td>
+                    <td>{$wire_key}</td>
+                    <td class="text-center">{$row.tier|default:'-'}</td>
+                    <td class="text-right">{$row.used}</td>
+                    <td>/{$row.limit}</td>
+                    <td class="text-right">{$row.left}</td>
                     <td class="text-right">
                         {block user_actions}{/block}
                     </td>
@@ -41,7 +44,14 @@
 
 {block user_actions}
 <div class="btn-group" role="group" aria-label="...">
-    <button type="button" class="btn btn-warning btn-xs" title="{t}Reset{/t}" data-toggle="modal" data-target="#modal" data-type="admin-rate-limit-reset" data-key="{$key}" data-NAME="{$group}">
+    <button type="button"
+            class="btn btn-warning btn-xs"
+            title="{t}Reset{/t}"
+            data-toggle="modal"
+            data-target="#modal"
+            data-type="admin-rate-limit-reset"
+            data-key="{$wire_key}"
+            data-name="{$group}">
         {fa icon="refresh"}
     </button>
 </div>

website/app-templates/smarty/dialog/admin_dialog_rate_limit_reset.tpl

@@ -4,9 +4,11 @@
     <h4 class="modal-title" id="modalLabel">{t}Do you really want to reset this rate-limit?{/t}</h4>
 </div>
 
-<form name="ResetRateLimitForm" action="{'admin_rate_limit_reset'|alias:sprintf('name=%s,key=%s', $name, $key)}" method="post">
+<form name="ResetRateLimitForm" action="{'admin_rate_limit_reset'|alias}" method="post">
     <div class="modal-body">
-        {t escape=no name=$name key=$key}Name=%1 ID=%2{/t}
+        {t escape=no name=$name key=$key}Name=%1 KEY=%2{/t}
+        <input type="hidden" name="name" value="{$name|escape}">
+        <input type="hidden" name="key" value="{$key|escape}">
     </div>
     <div class="modal-footer">
         {call csrf}

website/app-templates/smarty/pages/help_api.tpl

@@ -459,39 +459,151 @@ var_dump($gk);
     </div>
 </div>
 
-<h3>Rate Limits</h3>
+<h3>Rate limits</h3>
 <a class="anchor" id="apiratelimit"></a>
+
 <div class="panel panel-default">
     <div class="panel-body">
-        <p>In order to protect our service from abuse/misbehaving client… We have rate limiting in place.</p>
-        <p>If you hit a rate limit, we wil respond with the usual http code <code>429</code>.</p>
-        <p>A first set of rule limit the request rate per minutes, for any pages.</p>
-        <p>A second set of rule limit the API calls over a period of time. We're using the <a href="https://en.wikipedia.org/wiki/Leaky_bucket" target="_blank">Leaky Bucket Algorithm</a>.</p>
-        <blockquote  cite="https://en.wikipedia.org/wiki/Leaky_bucket">
-            The leaky bucket analogy. Water can be added intermittently to the bucket, which leaks out at a constant
-            rate until empty, and will also overflow when full.
-            <img src="https://upload.wikimedia.org/wikipedia/commons/7/77/Leaky_bucket_analogy.svg" class="img-responsive" width="170" height="240">
+
+        <div class="alert alert-info" role="alert" style="margin-bottom:16px">
+            <b>Quick summary</b>
+            <ul style="margin-bottom:0">
+                <li><b>Two layers:</b> (1) a global <i>burst</i> limit per minute across endpoints; (2) per-endpoint <i>quota</i> over a longer period (leaky bucket).</li>
+                <li><b>Who’s counted:</b> anonymous traffic is counted per <b>IP address</b>; authenticated traffic is counted per <b>account</b>.</li>
+                <li><b>Exceeding a limit:</b> we return <code>HTTP 429</code> and standard rate-limit headers.</li>
+            </ul>
+        </div>
+
+        <p>To keep the service reliable and abuse-resistant, we apply rate limiting.</p>
+        <p>If you hit a limit, we respond with <code>429 Too Many Requests</code>.</p>
+
+        <h4>How it works</h4>
+        <ul>
+            <li><b>Layer 1 – Burst (RPM):</b> caps short spikes across all endpoints per minute.</li>
+            <li><b>Layer 2 – Endpoint quota:</b> each API endpoint also has a longer-period allowance enforced with the
+                <a href="https://en.wikipedia.org/wiki/Leaky_bucket" target="_blank" rel="noopener">Leaky Bucket algorithm</a>.
+            </li>
+        </ul>
+
+        <blockquote cite="https://en.wikipedia.org/wiki/Leaky_bucket">
+            The leaky bucket analogy: water can be added intermittently, leaks at a constant rate, and overflows when full.
+            <img src="{GK_CDN_IMAGES_URL}/help/Leaky_bucket_analogy.svg" class="img-responsive" width="170" height="240" alt="Leaky bucket analogy">
         </blockquote>
-        <p>The limits are set per IP or per secid depending if the call is authenticated or not.</p>
+
+        <h5>Check usage</h5>
         <p>
-            Your current API usage is available in the headers of each API call. You can also get your current Rate Limit usage using this endpoint:
+            Your current usage is exposed in the API response headers and via a dedicated endpoint:
         </p>
         <ul>
             <li>Anonymous: <a href="{'api_v1_rate_limit_usage'|alias}">{'api_v1_rate_limit_usage'|alias}</a></li>
             <li>Authenticated: <a href="{'api_v1_rate_limit_usage'|alias}?secid=&lt;secid_here&gt;">{'api_v1_rate_limit_usage'|alias}?secid=&lt;secid_here&gt;</a></li>
         </ul>
+
         <pre><code class="language-xml">{$rate_limit_usage}</code></pre>
+
+        <h5>Example: 429 response</h5>
+        <pre><code>HTTP/1.1 429 Too Many Requests
+Ratelimit-Limit: 750
+Ratelimit-Remaining: 0
+X-RateLimit-Limit: 750
+X-RateLimit-Remaining: 0
+X-RateLimit-Reset: 1725100800
+X-Ratelimit-Resource: API_V1_EXPORT2
+X-GK-Rate-Limit: API_V1_EXPORT2 0/750 (86400)
+X-GK-Rate-Limit-Exceeded: true
+Content-Type: application/xml</code></pre>
+         <pre><code class="language-xml">{$gk_xml_rate_limit_error}</code></pre>
+
+        <h4>Current API rate limits</h4>
+        <p>
+            Your effective limit is the base multiplied by your tier multiplier (the <b>period</b> does not change):
+            <code>effective_limit = floor(base_limit × tier_multiplier)</code>
+        </p>
+
+        <table class="table table-striped table-sm">
+            <caption class="text-muted" style="caption-side: bottom">
+                Endpoint quotas per tier (leaky bucket).
+            </caption>
+            <thead>
+            <tr>
+                <th>endpoint</th>
+                <th>period_s</th>
+                <th>base_limit</th>
+                {foreach from=$smarty.const.RATE_LIMIT_LEVEL_MULTIPLIER key=tier item=mult}
+                    <th>{$tier} (×{$mult})</th>
+                {/foreach}
+            </tr>
+            </thead>
+            <tbody>
+            {foreach GK_RATE_LIMITS_DEFAULT as $limit => $values}
+                {assign var=base value=$values[0]}
+                {assign var=period value=$values[1]}
+                <tr>
+                    <td>{$limit}</td>
+                    <td>{$period}</td>
+                    <td>{$base}</td>
+                    {foreach from=$smarty.const.RATE_LIMIT_LEVEL_MULTIPLIER key=tier item=mult}
+                        <td>{math equation="floor(x*y)" x=$base y=$mult}</td>
+                    {/foreach}
+                </tr>
+            {/foreach}
+            </tbody>
+        </table>
+
+        <p class="text-muted">
+            <em>Note: some endpoints (e.g. username changes) are account-only; anonymous rows are kept for consistency.</em>
+        </p>
+
+        <h4>Tiers and multipliers</h4>
+        <ul>
+            <li><b>Anonymous</b> — requests without an account or secure identifier; <b>counted by IP address</b> (×0.5).</li>
+            <li><b>User</b> — any logged-in account (×1).</li>
+            <li><b>Contributor</b> — merged code, documentation, translations, or issue triage (×2).</li>
+            <li><b>Donor</b> — active project sponsor (×2).</li>
+            <li><b>Recurring donor</b> — regular project sponsor (×3).</li>
+            <li><b>Maintainer</b> — core team and operational tasks (×3).</li>
+        </ul>
+
+        <h4>Tips to avoid 429s</h4>
+        <ul>
+            <li>Authenticate when possible to avoid sharing anonymous (IP-based) limits behind NAT.</li>
+            <li>Batch and paginate requests; add small client-side delays or jitter.</li>
+            <li>Honor <code>Retry-After</code> and <code>X-RateLimit-*</code> headers when present.</li>
+        </ul>
+
+        <h4>How to earn higher limits</h4>
+        <ul>
+            <li><b>Code &amp; QA:</b> bug fixes, features, security reports.</li>
+            <li><b>Community:</b> translations, documentation, user support, issue triage.</li>
+            <li><b>Sponsorship:</b> optional donations help cover hosting and maintenance.</li>
+        </ul>
+        <p>Once confirmed, your tier updates automatically (a short cache delay may apply). Non-financial contributions are valued equally.</p>
+
+        <h4>Transparency</h4>
+        <ul>
+            <li>The table lists <b>base limits</b> per endpoint and <b>multipliers</b> per tier.</li>
+            <li>Endpoints may expose your current allowance and tier so you can monitor usage.</li>
+        </ul>
+
+        <h4>Abuse &amp; fairness</h4>
+        <ul>
+            <li>Automated abuse, credential sharing, or bypass attempts may lead to temporary or permanent restrictions.</li>
+            <li>We aim to notify and work with users to resolve accidental overuse.</li>
+        </ul>
+
+        <h4>Privacy</h4>
+        <ul>
+            <li>We store minimal counters in Redis keyed to anonymized identifiers.</li>
+            <li>Anonymous usage is keyed by IP; authenticated usage is keyed to your account.</li>
+            <li>Short-lived caches keep the service responsive.</li>
+        </ul>
+
+        <h4>Change management</h4>
+        <ul>
+            <li>Base limits or multipliers may be adjusted to keep the service healthy; material changes are announced in the changelog/release notes.</li>
+            <li>If you believe your tier is incorrect, please contact us for a review.</li>
+        </ul>
+
     </div>
-    <p>Current API rate limits are:</p>
-    <ul>
-        {foreach GK_RATE_LIMITS as $limit => $values}
-            <li>{$limit}
-                <ul>
-                    <li>max requests: {$values[0]}</li>
-                    <li>period: {$values[1]}s</li>
-                </ul>
-            </li>
-        {/foreach}
-    </ul>
 </div>
 {/block}

website/app-templates/smarty/pages/privacy_statement.tpl

@@ -110,7 +110,7 @@
             <li>
                 <p>
                     {t}Rate limit counters are used to prevent service usage abuse. Activity is kept for:{/t}<br>
-                    {foreach from=GK_RATE_LIMITS key=key item=value}
+                    {foreach from=GK_RATE_LIMITS_DEFAULT key=key item=value}
                         {$key} => {'seconds'|print_interval_for_humans:$value[1]}
                         <br>
                     {/foreach}

website/app/GeoKrety/Controller/Admin/RateLimits.php

@@ -8,7 +8,7 @@
 
 class RateLimits extends Base {
     public function render_results(\Base $f3) {
-        $current = RateLimit::get_rates_limits_usage();
+        $current = RateLimit::get_rates_limits_usage_detailed();
         Smarty::assign('current', $current);
         Smarty::render('admin/pages/api_rate_limit.tpl');
     }

website/app/GeoKrety/Controller/Pages/HelpApi.php

@@ -65,14 +65,20 @@ public function get($f3) {
         Smarty::assign('modified_since', date('YmdHis', time() - (1 * 60 * 60)));
 
         $xml = new Xml\RateLimits(false);
-        foreach (GK_RATE_LIMITS as $name => $values) {
-            $xml->addLimit($name, $values[0], $values[1]);
-            $xml->addUsage('xxx', 0);
+        foreach (GK_RATE_LIMITS_DEFAULT as $name => $values) {
+            $xml->addLimit($name, $values[1]);
+            $xml->addUsage('xxx', 0, $values[0], RATE_LIMIT_LEVEL_ANONYMOUS);
             $xml->endElement();
         }
         $xml->end();
         Smarty::assign('rate_limit_usage', $xml->asXMLPretty());
 
+        // Render ratelimit error
+        $xml = new Xml\Error();
+        $xml->addError(_('Rate limit exceeded'));
+        $xml->end();
+        Smarty::assign('gk_xml_rate_limit_error', $xml->asXMLPretty());
+
         Smarty::render('pages/help_api.tpl');
     }
 

website/app/GeoKrety/Controller/Pages/RateLimitXML.php

@@ -2,6 +2,7 @@
 
 namespace GeoKrety\Controller;
 
+use GeoKrety\Model\User;
 use GeoKrety\Service\RateLimit;
 use GeoKrety\Service\StorageException;
 use GeoKrety\Service\Xml\RateLimits;
@@ -14,50 +15,30 @@ public function beforeRoute(\Base $f3) {
         $this->xml = new RateLimits(true, $f3->get('GET.compress'));
     }
 
-    public function authenticate(): array {
-        $keys = [];
-        if ($this->f3->exists('GET.secid')) {
-            $login = new Login();
-
-            // TODO can we use that to load user limits?
-            $user = $login->secidAuth($this->f3, $this->f3->get('GET.secid'), false);
-            if (!is_null($user)) {
-                $keys[] = $user->secid;
-                Login::disconnectUser($this->f3);
-                // return $keys;
-            }
-        }
-
-        $keys[] = \Base::instance()->get('IP');
-
-        return $keys;
-    }
-
     public function get(\Base $f3) {
-        $id = $this->f3->exists('GET.secid') ? $this->f3->get('GET.secid') : \Base::instance()->get('IP');
-        RateLimit::check_rate_limit_xml('API_V1_CHECK_RATE_LIMIT', $id);
+        // Pass secid if present; otherwise null and RateLimit will fallback to IP internally
+        $secid = $f3->exists('GET.secid') ? (string) $f3->get('GET.secid') : null;
+        RateLimit::check_rate_limit_xml('API_V1_CHECK_RATE_LIMIT', $secid);
 
         $xml = $this->xml;
         try {
-            $keys = $this->authenticate();
-
-            $usages = RateLimit::get_usage_for_identities($keys);
-            $originalByNorm = [];
-            foreach ($keys as $k) {
-                $originalByNorm[strtr($k, [':' => '_'])] = $k;
-            }
-            foreach (GK_RATE_LIMITS as $name => $values) {
-                $this->xml->addLimit($name, $values[0], $values[1]);
-                if (!isset($usages[$name])) {
-                    foreach ($keys as $k) {
-                        $this->xml->addUsage($k, 0);
-                    }
-                    $this->xml->endElement();
-                    continue;
+            $identities = [];
+            if ($secid !== null && !empty($secid)) {
+                $user = User::get_user_by_secid($secid);
+                if (!is_null($user)) {
+                    $identities[] = $user->id;
                 }
-                foreach ($usages[$name] as $normKey => $count) {
-                    $displayId = $originalByNorm[$normKey] ?? $normKey;
-                    $this->xml->addUsage($displayId, $count);
+            }
+            $ip = \Base::instance()->get('IP');
+            if ($ip) {
+                $identities[] = $ip;
+            }
+
+            $usages = RateLimit::get_usage_for_identities($identities);
+            foreach ($usages as $limitName => $info) {
+                $this->xml->addLimit($limitName, GK_RATE_LIMITS_DEFAULT[$limitName][1]);
+                foreach ($info as $key => $details) {
+                    $this->xml->addUsage($key, $details['usage'], $details['limit'], $details['tier']);
                 }
                 $this->xml->endElement();
             }

website/app/GeoKrety/Controller/Pages/UserUpdateUsername.php

@@ -34,7 +34,8 @@ public function post(\Base $f3) {
             RateLimit::incr('USERNAME_CHANGE', $this->currentUser->id);
         } catch (RateLimitExceeded $e) {
             register_shutdown_function('GeoKrety\Model\AuditPost::AmendAuditPostWithErrors', 'Rate limit exceeded');
-            \Flash::instance()->addMessage(sprintf(_('You can only change your username %d times per month'), GK_RATE_LIMITS['USERNAME_CHANGE'][0]), 'danger');
+            [$limit] = RateLimitPolicy::resolve('USERNAME_CHANGE', $this->currentUser->id);
+            \Flash::instance()->addMessage(sprintf(_('You can only change your username %d times per month'), (int) $limit), 'danger');
             $f3->get('DB')->rollback();
             $this->get($f3);
             exit;