-
1
require 'heroku'
-
-
1
module OurEelHacks
-
1
class NullLogger
-
1
def debug; end
-
1
def info; end
-
1
def warn; end
-
1
def fatal; end
-
end
-
-
1
class Autoscaler
-
1
class << self
-
1
def get_instance(flavor)
-
6
flavor = flavor.to_sym
-
7
@instances ||= Hash.new{ |h,k| h[k] = self.new }
-
6
return @instances[flavor]
-
end
-
-
1
def configure(flavor = :web, &block)
-
2
get_instance(flavor).configure(flavor, &block)
-
end
-
-
1
def instance_for(flavor = :web)
-
4
instance = get_instance(flavor)
-
4
instance.check_settings
-
4
return instance
-
end
-
end
-
-
1
class Limit
-
1
def initialize(soft, hard)
-
26
@soft = soft
-
26
@hard = hard
-
end
-
-
1
attr_accessor :hard, :soft
-
end
-
-
1
class UpperLimit < Limit
-
1
def includes?(value)
-
8
return (@soft < value && value <= @hard)
-
end
-
-
1
def >(value)
-
return @soft > value
-
end
-
-
1
def <(value)
-
17
return @hard < value
-
end
-
end
-
-
1
class LowerLimit < Limit
-
1
def includes?(value)
-
16
return (value >= @hard && value <= @soft)
-
end
-
-
1
def >(value)
-
18
return @hard > value
-
end
-
-
1
def <(value)
-
return @soft < value
-
end
-
end
-
-
1
def initialize()
-
13
@dynos = nil
-
13
@soft_side = nil
-
-
13
@memoed_dyno_info = nil
-
-
13
@last_scaled = Time.at(0)
-
13
@entered_soft = Time.at(0)
-
13
@last_reading = nil
-
-
13
@app_name = nil
-
13
@ps_type = nil
-
13
@heroku_api_key = nil
-
-
13
@min_dynos = 1
-
13
@max_dynos = 10
-
13
@lower_limits = LowerLimit.new(5, 1)
-
13
@upper_limits = UpperLimit.new(30, 50)
-
-
13
@soft_duration = 10000
-
13
@scaling_frequency = 5000
-
13
@heroku_rate_limit = 80_000
-
13
@heroku_rate_limit_margin = 0.1
-
-
13
@millis_til_next_scale = nil
-
-
13
@logger = NullLogger.new
-
end
-
-
1
def configure(flavor = nil)
-
14
yield self
-
14
check_settings
-
14
logger.info{ "Autoscaler configured for #{flavor || "{{unknown flavor}}"}"}
-
-
14
update_dynos(dyno_info.count, Time.now)
-
end
-
-
1
MILLIS_PER_DAY = 24 * 60 * 60 * 1000
-
1
def check_settings
-
18
errors = []
-
18
errors << "No heroku api key set" if @heroku_api_key.nil?
-
18
errors << "No app name set" if @app_name.nil?
-
18
errors << "No process type set" if @ps_type.nil?
-
18
if (MILLIS_PER_DAY / @heroku_rate_limit) *
-
18
(1.0 - @heroku_rate_limit_margin) *
-
18
API_CALLS_PER_SCALE > @scaling_frequency
-
errors << "Scaling frequency will lock up Heroku"
-
end
-
18
unless errors.empty?
-
logger.warn{ "Problems configuring Autoscaler: #{errors.inspect}" }
-
raise "OurEelHacks::Autoscaler, configuration problem: " + errors.join(", ")
-
end
-
end
-
-
1
attr_accessor :min_dynos, :max_dynos, :lower_limits, :upper_limits, :ps_type,
-
:soft_duration, :scaling_frequency, :logger, :heroku_api_key, :app_name
-
1
attr_reader :last_scaled, :dynos, :entered_soft, :last_reading, :soft_side, :millis_til_next_scale
-
-
1
def elapsed(start, finish)
-
35
seconds = finish.to_i - start.to_i
-
35
micros = finish.usec - start.usec
-
35
diff = seconds * 1000 + micros / 1000
-
35
logger.debug{ "Elapsed: #{start.to_s}:#{finish.to_s} : #{diff}ms" }
-
35
return diff
-
end
-
-
1
API_CALLS_PER_SCALE = 2
-
1
def scale(metric)
-
31
logger.debug{ "Scaling request for #{@ps_type}: metric is: #{metric}" }
-
31
moment = Time.now
-
31
if elapsed(last_scaled, moment) < millis_til_next_scale
-
13
logger.debug{ "Not scaling: elapsed #{elapsed(last_scaled, moment)} less than computed #{millis_til_next_scale}" }
-
13
return
-
end
-
-
18
clear_dyno_info
-
-
18
starting_wait = millis_til_next_scale
-
-
18
update_dynos(dyno_info.count, moment)
-
-
18
target_dynos = target_scale(metric, moment)
-
-
18
target_dynos = [[target_dynos, max_dynos].min, min_dynos].max
-
18
logger.debug{ "Target dynos at: #{min_dynos}/#{target_dynos}/#{max_dynos} (vs. current: #{@dynos})" }
-
-
18
set_dynos(target_dynos, moment)
-
-
18
break_cadence(starting_wait)
-
rescue => ex
-
logger.warn{ "Problem scaling: #{ex.inspect}" }
-
end
-
-
1
def target_scale(metric, moment)
-
18
if lower_limits > metric
-
1
return dynos - 1
-
17
elsif upper_limits < metric
-
1
return dynos + 1
-
elsif
-
16
result = (dynos + soft_limit(metric, moment))
-
16
return result
-
end
-
end
-
-
1
def soft_limit(metric, moment)
-
40
hit_limit = [lower_limits, upper_limits].find{|lim| lim.includes? metric}
-
-
16
if soft_side == hit_limit
-
4
if elapsed(entered_soft, moment) > soft_duration
-
2
entered_soft = moment
-
2
case hit_limit
-
when upper_limits
-
1
return +1
-
when lower_limits
-
1
return -1
-
else
-
return 0
-
end
-
else
-
2
return 0
-
end
-
else
-
12
@entered_soft = moment
-
end
-
-
12
@soft_side = hit_limit
-
12
return 0
-
end
-
-
1
def break_cadence(starting_wait)
-
18
if starting_wait > millis_til_next_scale
-
2
@millis_til_next_scale = rand((starting_wait..@millis_til_next_scale))
-
end
-
end
-
-
1
def update_dynos(new_value, moment)
-
36
if new_value != dynos
-
17
@millis_til_next_scale = scaling_frequency * new_value
-
17
@last_scaled = moment
-
17
@entered_soft = moment
-
end
-
36
@dynos = new_value
-
36
@last_reading = moment
-
end
-
-
1
def clear_dyno_info
-
18
@memoed_dyno_info = nil
-
end
-
-
1
def dyno_info
-
return @memoed_dyno_info ||=
-
begin
-
31
regexp = /^#{ps_type}[.].*/
-
31
heroku.ps(app_name).find_all do |dyno|
-
186
dyno["process"] =~ regexp
-
end
-
36
end
-
end
-
-
1
def dynos_stable?
-
4
return dyno_info.all? do |dyno|
-
12
dyno["state"] == "up"
-
end
-
end
-
-
1
def heroku
-
@heroku ||= Heroku::Client.new("", heroku_api_key).tap do |client|
-
13
unless client.info(app_name)[:stack] == "cedar"
-
raise "#{self.class.name} built against cedar stack"
-
end
-
47
end
-
end
-
-
1
def set_dynos(count,moment)
-
18
if count == dynos
-
14
logger.debug{ "Not scaling: #{count} ?= #{dynos}" }
-
14
return
-
end
-
-
4
if not (stable = dynos_stable?)
-
logger.debug{ "Not scaling: dynos not stable (iow: not all #{ps_type} dynos are up)" }
-
return
-
end
-
4
logger.info{ "Scaling from #{dynos} to #{count} dynos for #{ps_type}" }
-
4
heroku.ps_scale(app_name, :type => ps_type, :qty => count)
-
4
update_dynos(count, moment)
-
end
-
end
-
end
-
1
require 'our-eel-hacks/autoscaler'
-
-
1
module OurEelHacks
-
1
class Middleware
-
1
def initialize(flavor)
-
2
@flavor = flavor
-
2
@canary_string = "Canary: #{Time.now.to_s}"
-
end
-
-
1
protected
-
-
1
def autoscale(metric)
-
2
now = Time.now
-
2
canary = @canary_string.dup
-
2
if @scaling_at.nil? or (now - @scaling_at) > 60
-
2
@scaling_at = now
-
2
trigger_scaling(metric, canary)
-
end
-
end
-
-
1
def trigger_scaling(metric, canary)
-
2
unless @canary_string == canary
-
raise "Canary died: #{@canary_string} != #{canary}"
-
end
-
2
Autoscaler.instance_for(@flavor).scale(metric)
-
2
@scaling_at = nil
-
end
-
end
-
end
-
1
require 'our-eel-hacks/middleware'
-
1
require 'our-eel-hacks/defer/event-machine'
-
1
module OurEelHacks
-
1
class Rack < Middleware
-
1
include Defer::EventMachine
-
-
1
def initialize(app, env_field, flavor = :web)
-
1
super(flavor)
-
1
@env_field = env_field
-
1
@app = app
-
end
-
-
1
def call(env)
-
1
begin
-
1
autoscale(metric_from(env))
-
rescue => ex
-
puts "Problem in autoscaling: #{ex.inspect}"
-
end
-
-
1
@app.call(env)
-
end
-
-
1
def metric_from(env)
-
1
(Integer(env[@env_field]) rescue 0).tap{|val|
-
1
puts "#{@env_field} => #{env[@env_field]} : #{val}"
-
}
-
end
-
end
-
-
1
class ScaleOnRoutingQueue < Rack
-
1
def initialize(app, flavor = :web)
-
super(app, "HTTP_X_HEROKU_QUEUE_DEPTH", flavor)
-
end
-
end
-
end
-
1
require 'our-eel-hacks/middleware'
-
1
require 'our-eel-hacks/defer/celluloid'
-
-
1
module OurEelHacks
-
1
class Sidekiq < Middleware
-
1
include Defer::Celluloid
-
1
def initialize(flavor=:sidekiq)
-
1
super
-
end
-
-
1
def call(worker_class, item, queue)
-
1
begin
-
2
autoscale(get_queue_length(queue).tap{|length| puts "Queue length: #{length}"})
-
rescue => ex
-
puts "Problem in autoscaling: #{ex.inspect}"
-
end
-
1
yield
-
end
-
-
1
def get_queue_length(queue)
-
1
::Sidekiq.redis do |conn|
-
conn.llen("queue:#{queue}") || 0
-
end
-
end
-
end
-
end