|
| 1 | +# frozen_string_literal: true |
| 2 | + |
| 3 | +module Segment |
| 4 | + class Analytics |
| 5 | + # Tracks the two independent budgets one send may spend. |
| 6 | + # |
| 7 | + # A retryable status carrying Retry-After spends the rate-limit budget, which |
| 8 | + # is bounded by wall clock only. Anything else retryable spends the counted |
| 9 | + # backoff budget, bounded by both a retry count and wall clock. Keeping them |
| 10 | + # separate is what stops a rate-limited server from exhausting the retries |
| 11 | + # available to genuine failures. |
| 12 | + # |
| 13 | + # The caller performs the wait, so both methods return the delay in seconds, |
| 14 | + # or nil when the budget is spent and the batch should be abandoned. |
| 15 | + class RetryBudget |
| 16 | + attr_reader :retry_count |
| 17 | + |
| 18 | + # Keyword arguments would be cleaner but need Ruby 2.1; the gemspec still |
| 19 | + # declares >= 2.0, which is also what rubocop is configured to parse. |
| 20 | + def initialize(options = {}) |
| 21 | + @retries_remaining = options[:retries] |
| 22 | + @backoff_policy = options[:backoff_policy] |
| 23 | + @max_total_backoff_duration = options[:max_total_backoff_duration] |
| 24 | + @max_rate_limit_duration = options[:max_rate_limit_duration] |
| 25 | + @rate_limit_retry_after_cap = options[:rate_limit_retry_after_cap] |
| 26 | + @logger = options[:logger] |
| 27 | + @retry_count = 0 |
| 28 | + @backoff_start_time = nil |
| 29 | + @rate_limit_start_time = nil |
| 30 | + end |
| 31 | + |
| 32 | + def next_backoff_delay |
| 33 | + # Checked before the decrement: decrementing first spent one retry on the |
| 34 | + # exhaustion test itself, so a configured N only ever performed N-1, and |
| 35 | + # retries: 1 and retries: 0 were indistinguishable. |
| 36 | + return spent('Retries exhausted for batch') if @retries_remaining <= 0 |
| 37 | + |
| 38 | + @retries_remaining -= 1 |
| 39 | + |
| 40 | + @backoff_start_time ||= monotonic_now |
| 41 | + return spent('Max total backoff duration exceeded for batch') if elapsed?(@backoff_start_time, @max_total_backoff_duration) |
| 42 | + |
| 43 | + delay_ms = @backoff_policy.next_interval |
| 44 | + @logger.debug("Retrying request, #{@retries_remaining} retries left. Waiting #{delay_ms}ms") |
| 45 | + delay_ms.to_f / 1000 |
| 46 | + end |
| 47 | + |
| 48 | + def next_rate_limit_delay(retry_after, status_code) |
| 49 | + @rate_limit_start_time ||= monotonic_now |
| 50 | + return spent('Max rate limit duration exceeded for batch') if elapsed?(@rate_limit_start_time, @max_rate_limit_duration) |
| 51 | + |
| 52 | + delay = [retry_after, @rate_limit_retry_after_cap].min |
| 53 | + @logger.debug("Retry-After: #{delay}s on #{status_code}. Retrying after delay.") |
| 54 | + delay |
| 55 | + end |
| 56 | + |
| 57 | + def record_retry |
| 58 | + @retry_count += 1 |
| 59 | + end |
| 60 | + |
| 61 | + private |
| 62 | + |
| 63 | + def elapsed?(start_time, limit) |
| 64 | + (monotonic_now - start_time) >= limit |
| 65 | + end |
| 66 | + |
| 67 | + # Wall-clock time can jump; these budgets must not expire or stretch with it. |
| 68 | + def monotonic_now |
| 69 | + Process.clock_gettime(Process::CLOCK_MONOTONIC) |
| 70 | + end |
| 71 | + |
| 72 | + def spent(message) |
| 73 | + @logger.error(message) |
| 74 | + nil |
| 75 | + end |
| 76 | + end |
| 77 | + end |
| 78 | +end |
0 commit comments