Class: Gizzard::CreateTableCommand

Inherits:
Command
  • Object
show all
Defined in:
lib/gizzard/commands.rb

Constant Summary collapse

DEFAULT_NUM_SHARDS = 
1024
DEFAULT_BASE_NAME = 
"shard"
FORWARDING_SPACE = 
2 ** 60
FORWARDING_SPACE_MIN = 
0
FORWARDING_SPACE_MAX = 
2 ** 60 - 1

Instance Attribute Summary

Attributes inherited from Command

#argv, #buffer, #command_options, #global_options, #job_injector, #manager

Instance Method Summary collapse

Methods inherited from Command

classify, #confirm!, #get_base_name, #help!, #initialize, make_job_injector, make_manager, #output, #require_tables, #require_template_options, run

Constructor Details

This class inherits a constructor from Gizzard::Command

Instance Method Details

#generate_base_ids(num_shards, min_id, max_id) ⇒ Object 



1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
 
# File 'lib/gizzard/commands.rb', line 1062

def generate_base_ids(num_shards, min_id, max_id)
  srand(42) # consistent randomization

  id_space  = max_id - min_id + 1
  step_size = id_space / num_shards

  enums = (0...num_shards).to_a
  ids   = enums.map {|i| min_id + (i * step_size) }.sort_by { rand }

  enums.zip(ids)
end

#parse_templates_and_weights(arr) ⇒ Object 



1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
 
# File 'lib/gizzard/commands.rb', line 1074

def parse_templates_and_weights(arr)
  templates_and_weights = {}

  arr.each_slice(2) do |(weight_s, to_template_s)|
    to     = ShardTemplate.parse(to_template_s)
    weight = weight_s.to_i

    templates_and_weights[to] = weight
  end

  templates_and_weights
end

#run ⇒ Object

This is all super hacky but I don’t have time to generalize right now



1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
 
# File 'lib/gizzard/commands.rb', line 1089

def run
  help!("must have an even number of arguments") unless @argv.length % 2 == 0
  require_tables
  require_template_options

  base_name    = command_options.base_name || DEFAULT_BASE_NAME
  num_shards   = (command_options.shards || DEFAULT_NUM_SHARDS).to_i
  max_id       = (command_options.max_id || FORWARDING_SPACE_MAX).to_i
  min_id       = (command_options.min_id || FORWARDING_SPACE_MIN).to_i

  be_quiet     = global_options.force && command_options.quiet

  templates_and_weights = parse_templates_and_weights(@argv)
  total_weight = templates_and_weights.values.inject {|a,b| a + b }
  templates = templates_and_weights.keys

  base_ids = generate_base_ids(num_shards, min_id, max_id)

  templates_and_base_ids = templates_and_weights.inject({}) do |h, (template, weight)|
    share = (weight.to_f / total_weight * num_shards).floor
    ids   = []
    share.times { ids << base_ids.pop }

    h.update template => ids
  end

  # divvy up the remainder across all templates.
  base_ids.each_with_index do |base, idx|
    templates_and_base_ids.values[idx % templates_and_base_ids.length] << base
  end

  proto     = templates.first
  transform = Transformation.new(proto, proto)

  op_sets = templates.inject({}) do |h, template|
    ops = transform.create_tree(template).sort
    h.update template => ops
  end

  unless be_quiet
    puts "Create tables #{global_options.tables.join(", ")}:"
    templates_and_base_ids.each do |template, base_ids|
      puts "  #{template.inspect}"
      puts "  for #{base_ids.length} base ids:"
      base_ids.each {|(enum, base_id)| puts "    #{base_id}" }
    end
    puts ""
  end

  confirm!

  global_options.tables.each do |table_id|
    templates_and_base_ids.each do |template, base_ids|
      ops = op_sets[template]

      base_ids.each do |(enum, base_id)|
        table_prefix = Shard.canonical_table_prefix(enum, table_id, base_name)
        ops.each do |op|
          puts "#{op.inspect}: #{table_prefix}"
          op.apply(manager, table_id, base_id, table_prefix, {})
        end
      end
    end
  end
end