Module: Cotcube::Bardata

Defined in:
lib/cotcube-bardata/eods.rb,
lib/cotcube-bardata.rb,
lib/cotcube-bardata/init.rb,
lib/cotcube-bardata/daily.rb,
lib/cotcube-bardata/cached.rb,
lib/cotcube-bardata/helpers.rb,
lib/cotcube-bardata/provide.rb,
lib/cotcube-bardata/suggest.rb,
lib/cotcube-bardata/quarters.rb,
lib/cotcube-bardata/constants.rb,
lib/cotcube-bardata/trade_dates.rb,
lib/cotcube-bardata/range_matrix.rb,
lib/cotcube-bardata/trading_hours.rb

Overview

Missing top level comment

Constant Summary collapse

SYMBOL_EXAMPLES = 
[
  { id: '13874U', symbol: 'ET', ticksize: 0.25, power: 1.25, months: 'HMUZ', bcf: 1.0, reports: 'LF',
    name: 'S&P 500 MICRO' },
  { id: '209747', symbol: 'NM', ticksize: 0.25, power: 0.5,  months: 'HMUZ', bcf: 1.0, reports: 'LF',
    name: 'NASDAQ 100 MICRO' }
].freeze
MONTH_COLOURS = 
{ 'F' => :cyan, 'G' => :green, 'H' => :light_green,
'J' => :blue,  'K' => :yellow,  'M' => :light_yellow,
'N' => :cyan,  'Q' => :magenta, 'U' => :light_magenta,
'V' => :blue,  'X' => :red,     'Z' => :light_red }.freeze

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.compare ⇒ Object

.config_path ⇒ Object

.config_prefix ⇒ Object

.continuous ⇒ Object

.continuous_actual_ml ⇒ Object

.continuous_ml ⇒ Object

.continuous_overview ⇒ Object

.continuous_table ⇒ Object

.extended_select_for_range ⇒ Object

.filter_series ⇒ Object

.holidays ⇒ Object

.init ⇒ Object

.last_trade_date ⇒ Object

.most_liquid_for ⇒ Object

.provide ⇒ Object

.provide_cached ⇒ Object

.provide_daily ⇒ Object

.provide_eods ⇒ Object

.provide_most_liquids_by_eod ⇒ Object

.provide_quarters ⇒ Object

.range_matrix ⇒ Object

.select_specific_date ⇒ Object

.suggest_contract_for ⇒ Object

.trading_hours ⇒ Object

Instance Method Details

#compare(contract:, format: '%5.2f') ⇒ Object 



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/cotcube-bardata/helpers.rb', line 48

def compare(contract:, format: '%5.2f')
  format = "%#{format}" unless format[0] == '%'
  daily = provide(contract: contract, interval: :daily)
  full  = provide(contract: contract, interval: :days, filter: :full)
  rth   = provide(contract: contract, interval: :days, filter: :rth)
  rth_dates = rth.map { |x| x[:datetime] }
  daily.select! { |x| rth_dates.include? x[:datetime].to_datetime }
  full.select! {  |x| rth_dates.include? x[:datetime].to_datetime }

  printer = lambda { |z|
       # rubocop:disable Layout/ClosingParenthesisIndentation
       "#{z[:datetime].strftime('%m-%d') # rubocop:disable Layout/IndentationWidth
     }\t#{format format, z[:open]
     }\t#{format format, z[:high]
     }\t#{format format, z[:low]
     }\t#{format format, z[:close]
     }\t#{format '%7d', z[:volume]}"
    # rubocop:enable Layout/ClosingParenthesisIndentation
  }
  daily.each_with_index do |_x, i|
    puts "DAILY #{printer.call daily[i]}"
    puts "FULL  #{printer.call full[i]}"
    puts "RTH   #{printer.call rth[i]}"
    puts ' '
  end
end

#config_path ⇒ Object 



32
33
34
 
# File 'lib/cotcube-bardata/init.rb', line 32

def config_path
  "#{config_prefix}/etc/cotcube"
end

#config_prefix ⇒ Object 



20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/cotcube-bardata/init.rb', line 20

def config_prefix
  os = Gem::Platform.local.os
  case os
  when 'linux'
    ''
  when 'freebsd'
    '/usr/local'
  else
    raise 'unknown architecture'
  end
end

#continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false, selector: nil, debug: false, add_eods: true, indicators: nil) ⇒ Object

reads all files in bardata/daily/<id> and aggregates by date (what is a pre-stage of a continuous based on daily bars)

Raises:

  • (ArgumentError) 


100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
 
# File 'lib/cotcube-bardata/daily.rb', line 100

def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false, selector: nil, debug: false, add_eods: true, indicators: nil)
  raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
  measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }

  measuring.call("Starting")
  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
  id  = sym[:id]
  symbol = sym[:symbol]
  ticksize = sym[:ticksize] 
  effective_selector = selector || :volume
  raise ArgumentError, 'selector must be in %i[ nil :volume ;oi].' unless [ nil, :volume, :oi ].include? selector
  id_path = "#{config[:data_path]}/daily/#{id}"
  c_file  = "#{id_path}/continuous.csv"
  puts "Using file #{c_file}" if debug

  # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
  # this boosts from 4.5sec to 0.3sec
  rewriting = (force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days))
  if rewriting
    puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow if debug
    `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v ',0,' | grep -v ',0$'| sort -t, -k2 | cut -d, -f1-8 | grep ',.*,' | uniq > #{c_file}`
  end
  loading = lambda do
    data = CSV.read(c_file).map do |row|
      r = { contract: row[0],
            date:     row[1],
            open:     row[2],
            high:     row[3],
            low:      row[4],
            close:    row[5],
            volume:   row[6].to_i,
            oi:       row[7].to_i,
            type:     :cont
      }
    end
    if add_eods
      today = Date.today
      eods = [ ]
      while today.strftime('%Y-%m-%d') > data.last[:date]
        eods << provide_eods(symbol: symbol, dates: today, contracts_only: false, quiet: true)
        today -= 1
      end
      eods.flatten!.map!{|x| x.tap {|y| i[ volume_part oi_part ].map{|z| y.delete(z)} } }
      eods.delete_if { |elem|  elem.flatten.empty? }
      data += eods.reverse
    end

    measuring.call("Finished retrieving dailies.")
    result = []
    rounding =  8
    indicators ||= {
      tr:   Cotcube::Indicators.true_range,
      atr5: Cotcube::Indicators.ema(key: :tr, length: 5),
      dist: Cotcube::Indicators.calc(a: :high, b: :low, finalize: :to_i) {|high, low| ((high-low) / ticksize) }
    }

    data.group_by { |x| x[:date] }.map do |k, v|
      v.map { |x| x.delete(:date) }
      avg_bar = { 
        date: k,
        contract: v.max_by{|x| x[:oi] }[:contract],
        open: nil, high: nil, low: nil, close: nil,
        volume:   v.map { |x| x[:volume] }.reduce(:+),
        oi:       v.map { |x| x[:oi] }.reduce(:+),
        type:     :cont_eod
      }

      i[ open high low close ].each do |ohlc|
        avg_bar[ohlc] = (v.map{|x| x[ohlc].to_f * x[effective_selector] }.reduce(:+) / avg_bar[effective_selector]).round(rounding)
        avg_bar[ohlc] = (avg_bar[ohlc] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0

      end
      p avg_bar if debug
      indicators.each do |k,v|
        print format('%12s:  ', k.to_s) if debug
        tmp = v.call(avg_bar)
        avg_bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
        puts avg_bar[k] if debug
      end
      #%i[tr atr5].each { |ind|
      #  avg_bar[ind]     = (avg_bar[ind] / sym[:ticksize]).round.to_i unless avg_bar[ind].nil?
      #}
      result << avg_bar
      result.last[:contracts] = v
    end
    result
  end
  constname = "CONTINUOUS_#{symbol}#{selector.nil? ? '' : ('_' + selector.to_s)}".to_sym
  if rewriting or not  Cotcube::Bardata.const_defined?( constname)
    old = $VERBOSE; $VERBOSE = nil
    Cotcube::Bardata.const_set constname, loading.call
    $VERBOSE = old
  end
  measuring.call("Finished processing")
  date.nil? ? Cotcube::Bardata.const_get(constname).map{|z| z.dup } : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
end

#continuous_actual_ml(symbol: nil, id: nil) ⇒ Object

the method above delivers the most_liquid as it is found at the end of the day. D during trading, the work is done with data that is already one day old. This is is fixed here:



258
259
260
261
262
263
264
265
266
267
268
 
# File 'lib/cotcube-bardata/daily.rb', line 258

def continuous_actual_ml(symbol: nil, id: nil)
  continuous =    Cotcube::Bardata.continuous    symbol: symbol, id: id
  continuous_ml = Cotcube::Bardata.continuous_ml base: continuous
  continuous_hash = continuous.to_h { |x| [x[:date], x[:contracts]] }
  actual_ml = continuous_ml.pairwise { |a, b| { date: b[:date], ml: a[:ml] } }
  actual_ml.map do |x|
    r = continuous_hash[x[:date]].select { |z| x[:ml] == z[:contract] }.first
    r = continuous_hash[x[:date]].min_by { |z| -z[:volume] } if r.nil?
    r
  end
end

#continuous_ml(symbol: nil, id: nil, base: nil) ⇒ Object 



248
249
250
251
252
253
 
# File 'lib/cotcube-bardata/daily.rb', line 248

def continuous_ml(symbol: nil, id: nil, base: nil)
  (base.nil? ? Cotcube::Bardata.continuous(symbol: symbol, id: id) : base).map do |x|
    x[:ml] = x[:contracts].max_by { |z| z[:volume] }[:contract]
    { date: x[:date], ml: x[:ml] }
  end
end

#continuous_overview(symbol: nil, id: nil, config: init, selector: :volume, human: false, measure: nil, filter: nil) ⇒ Object

based on .continuous, this methods sorts the prepared dailies continuous for each date on either :volume (default) or :oi with this job done, it can provide the period for which a past contract was the most liquid

Raises:

  • (ArgumentError) 


274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
 
# File 'lib/cotcube-bardata/daily.rb', line 274

def continuous_overview(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
                        config: init,
                        selector: :volume,
                        human: false,
                        measure: nil,
                        filter: nil)

  raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
  measuring = lambda {|c| puts "[continuous_overview] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }

  raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
    i[volume oi].include?(selector)

  measuring.call("Starting")
  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
  id  = sym[:id]
  # noinspection RubyNilAnalysis
  data = continuous(id: id, config: config, measure: measure).map do |x|
    {
      date: x[:date],
      volume: x[:contracts].sort_by { |z| - z[:volume] }[0..4].compact.reject { |z| z[:volume].zero? },
      oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
    }
  end
  measuring.call("Retrieved continuous for #{sym[:symbol]}")
  data.reject! { |x| x[selector].empty? }
  result = data.group_by { |x| x[selector].first[:contract] }
  result.each_key do |key|
    result[key].map! do |x|
      x[:volume].select! { |z| z[:contract] == key }
      x[:oi].select!     { |z| z[:contract] == key }
      x
    end
  end
  if human
    result.each do |k, v|
      next unless filter.nil? || v.first[selector].first[:contract][2..4] =~ (/#{filter}/)

      # rubocop:disable Layout/ClosingParenthesisIndentation
      puts "#{k
         }\t#{v.first[:date]
         }\t#{v.last[:date]
         }\t#{format('%4d', (Date.parse(v.last[:date]) - Date.parse(v.first[:date])))
         }\t#{result[k].map do |x|
        x[:volume].select do
          x[:contract] == k
        end
      end.size
         }"
      # rubocop:enable Layout/ClosingParenthesisIndentation
    end
  end
  measuring.call("Finished processing")
  result
end

#continuous_table(symbol: nil, id: nil, selector: :volume, filter: nil, date: Date.today, short: true, silent: false, measure: nil, debuglevel: 1, debug: false) ⇒ Object

Raises:

  • (ArgumentError) 


330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
 
# File 'lib/cotcube-bardata/daily.rb', line 330

def continuous_table(symbol: nil, id: nil,
                     selector: :volume,
                     filter: nil,
                     date: Date.today,
                     short: true,
                     silent: false,
                     measure: nil,
                     debuglevel: 1,
                     debug: false)
  if debug.is_a?(Integer)
    debuglevel = debug
    debug = debuglevel > 0 ? true : false
  end
  silent = false if debug

  raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
  measuring = lambda {|c| puts "[continuous_table] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }

  raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
    i[volume oi].include?(selector)

  measuring.call("Entering function")
  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
  if %w[R6 BJ GE].include? sym[:symbol]
    puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
    return []
  end
  id  = sym[:id]
  dfm = lambda do |x, y = date.year|
    k = Date.strptime("#{y} #{x.negative? ? x + 366 : x}", '%Y %j')
    k -= 1 while [0, 6].include?(k.wday)
    k.strftime('%a, %Y-%m-%d')
  rescue StandardError
    puts "#{sym[:symbol]}\t#{x}\t#{y}"
  end

  ytoday = date.yday
  data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
    .reject { |k, _| k[-2..].to_i >= date.year % 2000 }
    .select { |k, _| sym[:months].chars.include? k[2] }
    .group_by { |k, _| k[2] }
  measuring.call("Retrieved continous_overview")
  long_output = []

  toydate = -> (z,y=2021) { str = "#{z>365 ? y+1 : y} #{z>365 ? z-365 : z}";  DateTime.strptime(str, '%Y %j').strftime('%Y-%m-%d') }

  data.keys.sort.each do |month|
    puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
    v0 = data[month]

    # ldays is the list of 'last days'
    ldays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
    # fdays is the list of 'first days'
    fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
    # if the last ml day nears the end of the year, we must fix
    ldays.map! { |x| x > 350 ? x - 366 : x } if ldays.min < 50
    fday  = fdays[fdays.size / 2]
    lavg  = ldays.reduce(:+) / ldays.size

    # rubocop:disable Layout/ClosingParenthesisIndentation
    current = {
      month:    month,
      contract: "#{sym[:symbol]}#{month}",
      first_ml: fday,
      last_min: ldays.min,
      last_avg: lavg,
      last_max: ldays.max,
      until_start: fday - ytoday,
      until_end:   lavg - ytoday
    }
    current[:until_end] += 365 if current[:until_end] - current[:until_start] < 0
    current[:until_end] -= 365 if current[:until_end] > 365

    long_output << current

    # a contract is proposed to use after fday - 1, but before ldays.min (green)
    # it is warned to user after fday - 1 but before lavg - 1            (red)
    # it is warned red >= lavg - 1 and <= lavg + 1
    color = if (ytoday >= lavg - 1) && (ytoday <= lavg + 1)
              :light_red
            elsif (ytoday > ldays.min) && (ytoday < lavg - 1)
              :light_yellow
            elsif (ytoday >= (fday > lavg ? 0 : fday - 5)) && (ytoday <= ldays.min)
              :light_green
            else
              :white
            end

    output = "#{sym[:symbol]
      }#{month
      }\t#{format '%12s', sym[:type]
      }\ttoday is #{ytoday
      } -- median of first is #{fday
      }\tlast ranges from #{format '%5d', ldays.min
      }: #{dfm.call(ldays.min)
      }\t#{format '%5d', lavg
      }: #{dfm.call(lavg)
      }\tto #{format '%5d', ldays.max
      }: #{dfm.call(ldays.max)}".colorize(color)

      if debug || (color != :white)
        puts output unless silent
      end
      next if silent or not (debug and debuglevel >= 2)

      v0.each do |contract, v1|
        puts "\t#{contract
         }\t#{v1.first[:date]
         } (#{format '%3d', Date.parse(v1.first[:date]).yday
        })\t#{Date.parse(v1.last[:date]).strftime('%a, %Y-%m-%d')
         } (#{Date.parse(v1.last[:date]).yday})" unless silent
         # rubocop:enable Layout/ClosingParenthesisIndentation
      end

  end
  long_output.sort_by!{|z| z[:until_end] + (z[:until_end].negative? ? 365 : 0)}

  if short
    return ([long_output.first] + long_output.select{|z| z[:until_start].positive? and z[:until_start] < 10 }).map{|z| z[:contract] }.uniq
  end

  measuring.call("Finished processing")
  return long_output
end

#determine_significant_volume(base:, contract:) ⇒ Object 



58
59
60
61
62
 
# File 'lib/cotcube-bardata/provide.rb', line 58

def determine_significant_volume(base: , contract: )
  set  = Cotcube::Bardata.trading_hours(symbol: contract[0..1], filter: :rth)
  prod = base - base.select_within(ranges: set ,attr: :datetime) {|x| x.to_datetime.to_sssm }
  prod.group_by{|x| x[:volume] / 500 }
end

#extended_select_for_range(base:, range: ('1900-01-01'...'2100-01-01'), timezone: Time.find_zone('America/Chicago'), quiet: false) ⇒ Object

diminishes a given base of bars to fit into a given range (DO NOT CONFUSE with trading_hours) note that the last bar is simply required to start within the given range, not to end withing



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
 
# File 'lib/cotcube-bardata/helpers.rb', line 23

def extended_select_for_range(base:,
                              range: ('1900-01-01'...'2100-01-01'),
                              timezone: Time.find_zone('America/Chicago'),
                              quiet: false)

  starting = range.begin
  starting = timezone.parse(starting) if starting.is_a? String
  ending   = range.end
  ending   = timezone.parse(ending) if ending.is_a? String
  puts "#{starting}\t#{ending}" unless quiet
  if starting.hour.zero? && starting.min.zero? && ending.hour.zero? && ending.min.zero?
    unless quiet
      puts 'WARNING: When sending midnight, full trading day'\
        ' is assumed (starting 5 pm CT yesterday, ending 4 pm CT today)'.colorize(:light_yellow)
    end
    result = select_specific_date(date: starting, base: base)
    result += base.select { |d| d[:datetime] >= starting and d[:datetime] < ending.to_date }
    result += select_specific_date(date: ending, base: base)
    result.uniq!
  else
    result = base.select { |x| x[:datetime] >= starting and x[:datetime] <= ending }
  end
  result
end

#filter_series(ema_length: 50, symbol:, print_range: nil) ⇒ Object

the filter series is an indicator based on the Cotcube::Bardata.continuous of the asset price.

current default filter is the ema50



199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
 
# File 'lib/cotcube-bardata/daily.rb', line 199

def filter_series(ema_length: 50, symbol: , print_range: nil)
  ema_high_n = "ema#{ema_length}_high".to_sym
  ema_low_n  = "ema#{ema_length}_low".to_sym
  ema_filter = "ema#{ema_length}_filter".to_sym
  indicators = {
    ema_high_n => Cotcube::Indicators.ema(key: :high,     length: ema_length,  smoothing: 2),
    ema_low_n  => Cotcube::Indicators.ema(key: :low,      length: ema_length,  smoothing: 2),
    # NOTE: TR / ATR5 are in default set of continuous
    :tr        => Cotcube::Indicators.true_range,
    :atr5      => Cotcube::Indicators.ema(key: :tr,       length: 5,           smoothing: 2),
    ema_filter => Cotcube::Indicators.calc(a: :high,      b: :low,             c: :close,
                                           d: ema_high_n, e: ema_low_n,        f: :atr5,
                                           finalize: :to_i)  do |high, low, close, ema_high, ema_low, atr5|

                                             if    close >  ema_high and (low - ema_high).abs <= atr5 / 5.0; 3 # :bullish_tipped
                                             elsif   low >  ema_high and (low - ema_high).abs >= atr5 * 3.0; 5 # :bullish_away
                                             elsif   low >  ema_high and (low - ema_high).abs <= atr5 / 1.5; 2 # :bullish_nearby
                                             elsif   low >  ema_high;                                        4 # :bullish

                                             elsif close <  ema_low and (high - ema_low).abs <= atr5 / 5.0; -3 # :bearish_tipped
                                             elsif  high <  ema_low and (high - ema_low).abs >= atr5 * 3.0; -5 # :bearish_away
                                             elsif  high <  ema_low and (high - ema_low).abs <= atr5 / 1.5; -2 # :bearish_nearby
                                             elsif  high <  ema_low;                                        -4 # :bearish

                                             elsif close >= ema_high and (close - ema_high).abs > atr5 ;     2 # :bullish_closed
                                             elsif close <= ema_low  and (close - ema_low ).abs > atr5 ;    -2 # :bearish_closed
                                             elsif close >= ema_high;                                        1 # :bullish_weak
                                             elsif close <= ema_low;                                        -1 # :bearish_weak
                                             elsif close >  ema_low and close < ema_high;                    0 # :ambigue
                                             else
                                               raise RuntimeError, "Unconsidered Indicator value with #{high}, #{low}, #{close}, #{ema_high}, #{ema_low}, #{atr5}"

                                             end
                                           end
  }
  filter = Cotcube::Bardata.continuous(symbol: symbol, indicators: indicators).
    map{ |z| z[:datetime] = DateTime.parse(z[:date]); z[:datetime] += z[:datetime].wday == 5 ? 3 : 1; z.slice(:datetime, ema_filter) }.
    group_by{ |z| z[:datetime] }.
    map{ |k,v| [ k, v[0][ema_filter] ] }.
    to_h.
    tap{ |z|
      z.to_a[print_range].each { |v|
        puts "#{symbol} #{v[0].strftime('%Y-%m-%d')
           }  #{format '%2d', v[1]
           }".colorize(v[1] > 3 ? :light_green : v[1] > 1 ? :green : v[1] < -3 ? :light_red : v[1] < -1 ? :red : :white )
      } if print_range.is_a? Range
    }
end

#holidays(config: init) ⇒ Object 



49
50
51
 
# File 'lib/cotcube-bardata/trade_dates.rb', line 49

def holidays(config: init)
  CSV.read("#{config[:data_path]}/holidays.csv").map{|x| DateTime.parse(x[0])}
end

#init(config_file_name: 'bardata.yml') ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/cotcube-bardata/init.rb', line 36

def init(config_file_name: 'bardata.yml')
  name = 'bardata'
  config_file = config_path + "/#{config_file_name}"

  config = if File.exist?(config_file)
             YAML.safe_load(File.read(config_file)).transform_keys(&:to_sym)
           else
             {}
           end

  defaults = {
    data_path: "#{config_prefix}/var/cotcube/#{name}"
  }

  config = defaults.merge(config)

  # part 2 of init process: Prepare directories

  save_create_directory = lambda do |directory_name|
    unless Dir.exist?(directory_name)
      begin
        `mkdir -p #{directory_name}`
        unless $CHILD_STATUS.exitstatus.zero?
          puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
          exit 1 unless defined?(IRB)
        end
      rescue StandardError
        puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
        exit 1 unless defined?(IRB)
      end
    end
  end
  ['', :daily, :quarters, :eods, :trading_hours, :cached].each do |path|
    dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path}"
    save_create_directory.call(dir)
  end

  # eventually return config
  config
end

#last_trade_date(force_update: false) ⇒ Object

fetching official trade dates from CME it returns the current trade date or, if today isn’t a trading day, the last trade date.



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
 
# File 'lib/cotcube-bardata/trade_dates.rb', line 8

def last_trade_date(force_update: false)
  const_LTD = :LAST_TRADE_DATE
  const_LTDU = :LAST_TRADE_DATE_UPDATE
  if force_update or not Object.const_defined?(const_LTD) or Object.const_get(const_LTD).nil? or Time.now - Object.const_get(const_LTDU) > 2.hours
    result = nil
    uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
    headers = { "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
              "Accept-Encoding" => "gzip, deflate, br",
              "Accept-Language" => "en-US,en;q=0.9",
              "Cache-Control" => "max-age=0",
              "Connection" => "keep-alive",
             # Cookie: ak_bmsc=602078F6DE40954BAA8C7E7D3815102CACE82AAFD237000084B5A460F4FBCA68~pluz010T49Xag3sXquUZtVJmFX701dzEgt5v6Ht1EZSLKE4HL+bgg1L9ePnL5I0mm7QWXe1qaLhUbX1IPrL/f20trRMMRlkC3UWXk27DY/EBCP4mRno8QQygLCwgs2B2AQHJyb63WwRihCko8UYUiIhb89ArPZM5OPraoKy3JU9oE9e+iERdARNZHLHqRiB1GnmbKUvQqos3sXaEe3GpoiTszzk8sHZs4ZKuoO/rvFHko=",
              "Host" => "www.cmegroup.com",
              "sec-ch-ua" => %q[" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"],
              "sec-ch-ua-mobile" => "?0",
              "Sec-Fetch-Dest" => "document",
              "Sec-Fetch-Mode" => "navigate",
              "Sec-Fetch-Site" => "none",
              "Sec-Fetch-User" => "?1",
              "Upgrade-Insecure-Requests" => "1",
              "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" }
    begin
#          HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
       result = HTTParty.get(uri, headers: headers)
              .parsed_response
              .map do |x|
                a = x['tradeDate'].chars.each_slice(2).map(&:join)
                "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
              end
              .first
    rescue StandardError
      result = nil
    end
    oldverbose = $VERBOSE; $VERBOSE = nil
    Object.const_set(const_LTD, result)
    Object.const_set(const_LTDU, Time.now) unless result.nil?
    $VERBOSE = oldverbose
  end
  Object.const_get(const_LTD)
end

#most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init) ⇒ Object 



6
7
8
9
 
# File 'lib/cotcube-bardata/eods.rb', line 6

def most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init)
  id = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config)[:id]
  provide_eods(id: id, dates: date, contracts_only: true).first
end

#provide(contract:, range: nil, symbol: nil, id: nil, config: init, interval: :days, filter: :full, force_update: false, force_recent: false) ⇒ Object

rubocop:disable Metrics/ParameterLists



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/cotcube-bardata/provide.rb', line 6

def provide(contract:, # rubocop:disable Metrics/ParameterLists
            # Can be like ("2020-12-01 12:00"..."2020-12-14 11:00")
            range: nil,
            symbol: nil, id: nil,
            config: init,
            # supported types are :quarters, :hours, :days, :rth, :dailies, :weeks, :months
            interval: :days,
            # supported filters are :full and :rth (and custom named, if provided as file)
            filter: :full,
            # TODO: for future compatibility and suggestion: planning to include a function to update
            #       with live data from broker
            force_update: false,
            force_recent: false)

  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract, config: config)

  case interval
  when :quarters, :hours, :quarter, :hour
    base = provide_quarters(contract: contract, symbol: symbol, id: id, config: config)
    base = extended_select_for_range(range: range, base: base) if range
    requested_set = trading_hours(symbol: sym[:symbol], filter: filter)

    base = base.select_within(ranges: requested_set, attr: :datetime) { |x| x.to_datetime.to_sssm }
    return base if i[quarters quarter].include? interval

    Cotcube::Helpers.reduce(bars: base, to: :hours) do |c, b|
      c[:day] == b[:day] and c[:datetime].hour == b[:datetime].hour
    end

  when :days, :weeks, :months
    base = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
                          range: range, force_recent: force_recent, force_update: force_update
    return base if i[day days].include? interval

    # TODO: Missing implementation to reduce cached days to weeks or months
    raise 'Missing implementation to reduce cached days to weeks or months'
  when :dailies, :daily
    provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
  when :synth, :synthetic, :synthetic_days
    days = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
                          range: range, force_recent: force_recent, force_update: force_update
    dailies = provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
    if ((days.last[:datetime] > dailies.last[:datetime]) rescue false)
      dailies[..-2] + days.select { |d| d[:datetime] > dailies[-2][:datetime] }
    else
      dailies
    end
  else
    raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
  end
end

#provide_cached(contract:, symbol: nil, id: nil, range: nil, config: init, debug: false, timezone: Time.find_zone('America/Chicago'), filter: :full, force_update: false, force_recent: false) ⇒ Object

send pre-created days based on quarters



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
 
# File 'lib/cotcube-bardata/cached.rb', line 7

def provide_cached(contract:, # rubocop:disable Metrics/ParameterLists
                   symbol: nil, id: nil,
                   range: nil,
                   config: init,
                   debug: false,
                   timezone: Time.find_zone('America/Chicago'),
                   filter: :full, # most probably either :full or :rth
                   force_update: false, # force reloading via provide_quarters
                   force_recent: false) # provide base.last even if dayswitch hasn't happen yet

  unless range.nil? ||
         range.is_a?(Range) &&
         [Date, DateTime, ActiveSupport::TimeWithZone].map do |cl|
           (range.begin.nil? || range.begin.is_a?(cl)) &&
           (range.end.nil? || range.end.is_a?(cl))
         end.reduce(:|)
    raise ArgumentError, 'Range, if given, must be either (Integer..Integer) or (Timelike..Timelike)'
  end

  unless range.nil?
    range_begin = range.begin.nil? ? nil : timezone.parse(range.begin.to_s)
    range_end   = range.end.nil? ? nil : timezone.parse(range.end.to_s)
    range = (range_begin..range_end)
  end

  headers = i[contract datetime open high low close volume]
  sym      = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
  contract = contract[-3..]
  dir      = "#{config[:data_path]}/cached/#{sym[:id]}_#{filter.to_s.downcase}"
  symlink  = "#{config[:data_path]}/cached/#{sym[:symbol]}_#{filter.to_s.downcase}"
  `mkdir -p #{dir}` unless Dir.exist? dir
  `ln -s #{dir} #{symlink}` unless File.exist? symlink
  file = "#{dir}/#{contract}.csv"
  quarters_file = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract[-3..]}.csv"
  if File.exist?(file) && (not force_update)
    puts "Working with existing #{file}, no update was forced" if debug
    puts "    Using quarters from #{quarters_file}" if debug
    base = CSV.read(file, headers: headers).map do |x|
      x = x.to_h
      x[:datetime] = timezone.parse(x[:datetime])
      i[open high low close].each { |z| x[z] = x[z].to_f.round(9) }
      x[:volume] = x[:volume].to_i
      x[:dist]     = ((x[:high] - x[:low]) / sym[:ticksize] ).to_i
      x[:type]   = "#{filter.to_s.downcase}_day".to_sym
      x
    end
    if base.last[:high].zero?
      # contract exists but is closed (has the CLOSED marker)
      base.pop 
      # rubocop:disable Metrics/BlockNesting
      result = if range.nil?
                 base
               else
                 base.select do |x|
                   (range.begin.nil? ? true : x[:datetime] >= range.begin) and
                     (range.end.nil? ? true : x[:datetime] <= range.end)
                 end
               end
      return result
    elsif File.mtime(file) - Time.now.beginning_of_day >= 0
      puts "CACHE #{File.mtime(file)}\t#{file}" if debug
      puts "QUART #{File.mtime(quarters_file)}\t#{quarters_file}" if debug
      result = if range.nil?
                 base
               else
                 base.select do |x|
                   (range.begin.nil? ? true : x[:datetime] >= range.begin) and
                     (range.end.nil? ? true : x[:datetime] <= range.end)
                 end
               end
      # rubocop:enable Metrics/BlockNesting
      return result
    else
      # write a (positive warning, that the cache needs to be updated, as cached value is older
      #   than one day but not closed
      puts "File #{file} exists, but is neither closed nor current. Running update...".colorize(:light_green)
    end
  end
  begin
    data = provide_quarters(contract: contract, id: sym[:id], keep_marker: true)
  rescue StandardError
    puts "Cannot provide quarters for requested contract #{sym[:symbol]}:#{contract},"\
        "returning '[ ]'".colorize(:light_red)
    return []
  end

  # removing marker if existing
  contract_is_marked = data.last[:high].zero?
  data.pop if contract_is_marked
  unless (filter == :full) || (data.size < 3)
    requested_set = trading_hours(symbol: sym[:symbol], filter: filter)
    data = data.select_within(ranges: requested_set, attr: :datetime) { |x| x.to_datetime.to_sssm }
  end

  base = Cotcube::Helpers.reduce(bars: data, to: :days)
  puts "Reduced base ends at #{bast.last[:datetime].strftime('%Y-%m-%d')}" if debug

  # remove last day of result if suspecting incomplete last base 
  base.pop if base.last[:datetime].to_date == timezone.now.to_date and not force_recent

  base.map do |x|
    x[:date] = x[:datetime].to_date
    x[:type] = "#{filter}_day".to_sym
    x.delete(:day)
  end
  CSV.open(file, 'w') do |csv|
    base.each { |b| csv << b.values_at(*headers) }
    if contract_is_marked
      marker = ["#{sym[:symbol]}#{contract}", base.last[:date] + 1.day, 0, 0, 0, 0, 0]
      csv << marker
    end
  end
  if range.nil?
    base
  else
    base.select do |x|
      (range.begin.nil? ? true : x[:date] >= range.begin) and
        (range.end.nil? ? true : x[:date] <= range.end)
    end
  end
end

#provide_daily(contract:, symbol: nil, id: nil, range: nil, timezone: Cotcube::Helpers::CHICAGO, keep_last: false, add_eods: true, indicators: {}, config: init) ⇒ Object

just reads bardata/daily/<id>/<contract>.csv



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
 
# File 'lib/cotcube-bardata/daily.rb', line 7

def provide_daily(contract:, # rubocop:disable Metrics/ParameterLists
                  symbol: nil, id: nil,
                  range: nil,
                  timezone: Cotcube::Helpers::CHICAGO,
                  keep_last: false,
                  add_eods: true,
                  indicators: {},
                  config: init)
  contract = contract.to_s.upcase
  rounding = 8
  unless contract.is_a?(String) && [3, 5].include?(contract.size)
    raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
  end
  unless range.nil? ||
         (range.is_a?(Range) &&
         [Date, DateTime, ActiveSupport::TimeWithZone].map do |cl|
           (range.begin.nil? || range.begin.is_a?(cl)) &&
           (range.end.nil?   || range.end.is_a?(cl))
         end.reduce(:|))

    raise ArgumentError, 'Range, if given, must be either (Integer..Integer) or (Timelike..Timelike)'
  end

  unless range.nil?
    range_begin = range.begin.nil? ? nil : timezone.parse(range.begin.to_s)
    range_end   = range.end.nil?   ? nil : timezone.parse(range.  end.to_s)
    range = (range_begin..range_end)
  end

  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
  contract = contract[2..4] if contract.to_s.size == 5
  id = sym[:id]
  id_path   = "#{config[:data_path]}/daily/#{id}"
  data_file = "#{id_path}/#{contract}.csv"
  raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)

  raise "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)

  data = CSV.read(data_file, headers: i[contract date open high low close volume oi]).map do |row|
    row = row.to_h
    row.each do |k, _|
      if i[open high low close].include? k
        row[k] = row[k].to_f
        row[k] = (row[k] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
      end
      row[k] = row[k].to_i if i[volume oi].include? k
    end
    row[:datetime] = timezone.parse(row[:date])
    row[:dist]     = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
    row[:type]     = :daily
    row
  end
  contract_expired = data.last[:high].zero?
  data.pop if contract_expired and not keep_last
  if not contract_expired and add_eods
    today = Date.today
    eods = [ ]
    while today.strftime('%Y-%m-%d') > data.last[:date]
      eods << provide_eods(symbol: sym[:symbol], dates: today, contracts_only: false, quiet: true)
      today -= 1
    end
    eods.flatten!.map!{|x| x.tap {|y| i[ volume_part oi_part ].map{|z| y.delete(z)} } }
    eods.select!{|x| x[:contract] == "#{sym[:symbol]}#{contract}" } 
    eods.map!{|x| x.tap{|y| 
      if sym[:bcf] != 1.0
        i[open high low close].map{|k|
           y[k] = (y[k] * sym[:bcf]).round(8)
        }
      end
      y[:datetime] = timezone.parse(y[:date])
      y[:dist]     = ((y[:high] - y[:low]) / sym[:ticksize] ).to_i
      y[:type]     = :eod
    } }
    data += eods.reverse
  end
  data.map do |bar| 
    indicators.each do |k,v|
      tmp = v.call(bar)
      bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
    end
  end unless indicators.empty?
  if range.nil?
    data
  else
    data.select do |x|
      (range.begin.nil? ? true : x[:datetime] >= range.begin) and
        (range.end.nil? ? true : x[:datetime] <= range.end)
    end
  end
end

#provide_eods(symbol: nil, id: nil, contract: nil, config: init, dates: last_trade_date, threshold: 0.05, filter: :volume_part, contracts_only: true, quiet: false) ⇒ Object

provide a list of all eods for id/symbol or all symbols (default) for an

array of dates (default: [last_trade_date])

filter by :threshold*100% share on entire volume(default) or oi

return full data or just the contract name (default)

Raises:

  • (ArgumentError) 


43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/cotcube-bardata/eods.rb', line 43

def provide_eods(symbol: nil, # rubocop:disable Metrics/ParameterLists
                 id: nil,
                 contract: nil,
                 config: init,
                 # should accept either a date or date_alike or date string OR a range of 2 dates alike
                 # if omitted returns the eods of last trading date
                 dates: last_trade_date,
                 # set threshold to 0 to disable filtering at all.
                 # otherwise only contracts with partial of >= threshold are returned
                 threshold: 0.05,
                 # filter can be set to volume_part and oi_part.
                 # determines, which property is used for filtering.
                 filter: :volume_part,
                 # set to false to return the complete row instead
                 # of just the contracts matching filter and threshold
                 contracts_only: true,
                 quiet: false)
  unless contract.nil? || (contract.is_a?(String) && [3, 5].include?(contract.size))
    raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
  end

  symbol = contract[0..1] if contract.to_s.size == 5
  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config) if symbol || id
  # if no id can be clarified from given arguments, return all matching contracts from all available symbols
  # raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
  raise ArgumentError, ':filter must be in [:volume_part, :oi_part]' unless i[volume_part oi_part].include? filter

  # noinspection RubyScope
  ids = sym.nil? ? Cotcube::Helpers.symbols.map { |x| x[:id] } : [sym[:id]]
  dates = [dates] unless dates.is_a?(Array) || dates.nil?

  id_path_get = ->(local_id) { "#{config[:data_path]}/eods/#{local_id}" }

  process_date_for_id = lambda do |d, i|
    # l_sym        = symbols.select { |s| s[:id] == i }.first
    # l_symbol     = l_sym[:symbol]
    id_path    = id_path_get.call(i)
    data_file  = "#{id_path}/#{d}.csv"
    current_sym =  Cotcube::Helpers.get_id_set(id: i, config: config)
    raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)

    unless File.exist?(data_file)
      unless quiet
        puts 'WARNING: No data found for requested symbol'\
          " #{current_sym[:symbol]} in #{id_path} for #{d}.".colorize(:light_yellow)

      end
      return []
    end
    data = CSV.read(data_file, headers: i[contract date open high low close volume oi]).map do |row|
      row = row.to_h
      row.each do |k, _|
        row[k] = row[k].to_f if i[open high low close].include? k
        row[k] = row[k].to_i if i[volume oi].include? k
      end
      row
    end
    all_volume = data.map { |x| x[:volume] }.reduce(:+)
    all_oi     = data.map { |x| x[:oi]     }.reduce(:+)
    data.map do |x|
      x[:volume_part] = (x[:volume] / all_volume.to_f).round(4)
      x[:oi_part]     = (x[:oi] / all_oi.to_f).round(4)
    end
    data.select { |x| x[filter] >= threshold }.sort_by { |x| -x[filter] }.tap do |x|
      if contracts_only
        x.map! do |y|
          y[:contract]
        end
      end
    end
  end
  if dates
    dates.map do |date|
      ids.map { |local_id| process_date_for_id.call(date, local_id) }
    end.flatten
  else
    raise ArgumentError,
          'Sorry, support for unlimited dates is not implemented yet. Please send array of dates or single date'
  end
end

#provide_most_liquids_by_eod(symbol: nil, id: nil, config: init, date: last_trade_date, filter: :volume_part, age: 1.hour) ⇒ Object

the following method seems to be garbage. It is not used anywhere. It seems it’s purpose was to retrieve a list of quarters that have not been fetched recently (–> :age)



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/cotcube-bardata/eods.rb', line 13

def provide_most_liquids_by_eod(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
                                config: init,
                                date: last_trade_date,
                                filter: :volume_part,
                                age: 1.hour)
  sym  = Cotcube::Helpers.get_id_set(symbol: symbol, id: id) if symbol || id
  # noinspection RubyScope
  eods = provide_eods(id: sym.nil? ? nil : sym[:id], config: config, dates: date, filter: filter)
  result = []
  eods.map do |eod|
    symbol   = eod[0..1]
    contract = eod[2..4]
    sym      = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
    quarter  = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract}.csv"
    if File.exist?(quarter)
      # puts "#{quarter}: #{ Time.now } - #{File.mtime(quarter)} > #{age} : #{Time.now - File.mtime(quarter) > age}"
      result << eod if Time.now - File.mtime(quarter) > age
    else
      result << eod
    end
  end
  result
end

#provide_quarters(contract:, symbol: nil, id: nil, timezone: Time.find_zone('America/Chicago'), config: init, keep_marker: false) ⇒ Object

the following method loads the quarterly bars (15-min bars) from the directory tree also note that a former version of this method allowed to provide range or date parameters. this has been moved to #provide itself.



9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'lib/cotcube-bardata/quarters.rb', line 9

def provide_quarters(contract:, # rubocop:disable Metrics/ParameterLists
                     symbol: nil, id: nil,
                     timezone: Time.find_zone('America/Chicago'),
                     config: init,
                     keep_marker: false)

  unless contract.is_a?(String) && [3, 5].include?(contract.size)
    raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
  end

  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)

  contract = contract[2..4] if contract.to_s.size == 5
  id = sym[:id]
  symbol = sym[:symbol]

  id_path   = "#{config[:data_path]}/quarters/#{id}"
  data_file = "#{id_path}/#{contract}.csv"
  raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)

  raise "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)

  data = CSV.read(data_file, headers: i[contract datetime day open high low close volume]).map do |row|
    row = row.to_h
    i[open high low close].map { |x| row[x] = row[x].to_f }
    i[volume day].map { |x| row[x] = row[x].to_i }
    row[:datetime] = timezone.parse(row[:datetime])
    row[:dist]     = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
    row[:type]     = :quarter
    row
  end
  data.pop if data.last[:high].zero? && (not keep_marker)
  data
end

#range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block) ⇒ Object

this is an analysis tool to investigate actual ranges of an underlying symbol it is in particular no true range or average true range, as a ‘true range’ can only be applied to

The result printed / returned is a table, containing a matrix of rows:

1. size: the amount of values evaluated
2. avg:
3. lower: like median, but not at 1/2 but 1/4
4. median:
5. upper: like median, but not at 1/2 but 3/4
6. max:

and columns:

1.a) all days os the series
1.b) all days of the series, diminished by 2* :dim*100% extreme values (i.e. at both ends)
1.c) the last 200 days
2.a-c) same with days reduced to weeks (c: 52 weeks)
3.a-c) same with days reduced to months (c: 12 months)

NOTE: there is now a new method Cotcube::Helpers.simple_series_stats, that should be used in favor.



24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
 
# File 'lib/cotcube-bardata/range_matrix.rb', line 24

def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
  # rubocop:disable Style/MultilineBlockChain
  symbol ||= base.last[:contract][0..1] if id.nil?
  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
  source = {}
  target = {}
  if base.nil?
    ml = (Cotcube::Bardata.continuous_actual_ml symbol: symbol)&.last[:contract]
    source[:days]   = Cotcube::Bardata.provide contract: ml
  else
    source[:days]   = base
  end
  source[:weeks]  = Cotcube::Helpers.reduce bars: source[:days], to: :weeks
  source[:months] = Cotcube::Helpers.reduce bars: source[:days], to: :months

  i[days weeks months].each do |period|
    next if days_only and i[weeks months].include? period
    source[period].map! do |x|
      x[:range] = block_given? ? yield(x) : (((x[:high] - x[:low]) / sym[:ticksize]).round)
      x
    end
    target[period] = {}
    target[period][:all_size]   =  source[period].size
    target[period][:all_avg]    = (source[period].map { |x| x[:range] }.reduce(:+) / source[period].size).round
    target[period][:all_lower]  =  source[period].sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (source[period].size * 1 / 4).round ]

    target[period][:all_median] = source[period].sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (source[period].size * 2 / 4).round ]

    target[period][:all_upper] = source[period].sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (source[period].size * 3 / 4).round ]

    target[period][:all_max] = source[period].map { |x| x[:range] }.max
    target[period][:all_records] = source[period].sort_by do |x|
      -x[:range]
    end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)

    tenth = (source[period].size * dim).round
    custom = source[period].sort_by { |x| x[:range] }[tenth..source[period].size - tenth]

    target[period][:dim_size]   =  custom.size
    target[period][:dim_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
    target[period][:dim_lower]  =  custom.sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (custom.size * 1 / 4).round ]

    target[period][:dim_median] = custom.sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (custom.size * 2 / 4).round ]

    target[period][:dim_upper] = custom.sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (custom.size * 3 / 4).round ]

    target[period][:dim_max] = custom.map { |x| x[:range] }.max
    target[period][:dim_records] = custom.sort_by do |x|
      -x[:range]
    end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)

    range = case period
            when :months
              -(last_n/15)..-2
            when :weeks
              -(last_n/4)..-2
            when :days
              -last_n..-1
            else
              raise ArgumentError, "Unsupported period: '#{period}'"
            end
    if range.begin.abs > source[period].size
      puts "WARNING: requested last_n = #{last_n} exceeds actual size (#{source[period].size}), adjusting...".light_yellow
      range = (-source[period].size..range.end)
    end
    custom = source[period][range]
    target[period][:rec_size]   =  custom.size
    target[period][:rec_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
    target[period][:rec_lower]  =  custom.sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (custom.size * 1 / 4).round ]
    target[period][:rec_median] = custom.sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (custom.size * 2 / 4).round ]
    target[period][:rec_upper] = custom.sort_by do |x|
      x[:range]
    end.map { |x| x[:range] }[ (custom.size * 3 / 4).round ]
    target[period][:rec_max] = custom.map { |x| x[:range] }.max
    target[period][:rec_records] = custom.sort_by do |x|
      -x[:range]
    end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
  end

  if print
    %w[size avg lower median upper max].each do |a|
      print "#{'%10s' % a} | " # rubocop:disable Style/FormatString
      i[days weeks months].each do |b|
        next if days_only and i[weeks months].include? b
        %w[all dim rec].each do |c|
          print ('%8d' % target[b]["#{c}_#{a}".to_sym]).to_s # rubocop:disable Style/FormatString
        end
        print ' | '
      end
      puts ''
    end
  end

  target
  # rubocop:enable Style/MultilineBlockChain
end

#select_specific_date(date:, base:) ⇒ Object

small helper to select a specific full trading day from quarters (or reduced)

this special handling is needed, as full trading days start '5pm CT yesterday'



8
9
10
11
12
13
14
15
16
17
18
19
 
# File 'lib/cotcube-bardata/helpers.rb', line 8

def select_specific_date(date:, base:)
  base.select do |d|
    d[:day] == date.day and date.year == d[:datetime].year and (
    if date.day > 1
      date.month == d[:datetime].month
    else
      ((date.month == d[:datetime].month     and d[:datetime].day == 1) or
       (date.month == d[:datetime].month + 1 and d[:datetime].day > 25))
    end
  )
  end
end

#suggest_contract_for(symbol:, date: Date.today, warnings: true) ⇒ Object

based on day(of year) and symbol, suggest best fitting contract



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
 
# File 'lib/cotcube-bardata/suggest.rb', line 7

def suggest_contract_for symbol:, date: Date.today, warnings: true
  ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date, silent: true
  if ml.size != 1 
  puts "WARNING: No or no unique most liquid found for #{date}. please give :contract parameter".light_yellow if warnings
  if ml.size > 1
    puts "\tUsing #{ml.last}. Consider breaking here, if that is not acceptable.".light_yellow if warnings
    sleep 1
  else
    puts "\tERROR: No suggestible contract found for #{symbol} and #{date}.".light_red
    return
  end
  end
  year = date.year % 100
  if ml.last[2] < "K" and date.month > 9
    "#{ml.last}#{year + 1}"
  else
    "#{ml.last}#{year}"
  end
end

#symbols(config: init, type: nil, symbol: nil) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
 
# File 'lib/cotcube-bardata/init.rb', line 6

def symbols(config: init, type: nil, symbol: nil)
  if config[:symbols_file].nil?
    SYMBOL_EXAMPLES
  else
    CSV
      .read(config[:symbols_file], headers: i[id symbol ticksize power months type bcf reports format name])
      .map(&:to_h)
      .map { |row| i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row[:format] = "%#{row[:format]}f"; row } # rubocop:disable Style/Semicolon
      .reject { |row| row[:id].nil? }
      .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
      .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }
  end
end

#trading_hours(symbol: nil, id: nil, filter:, force_filter: false, headers_only: false, config: init, debug: false) ⇒ Object

returns an Array of ranges containing a week of trading hours, specified by seconds since monday morning

(as sunday is wday:0)

according files are located in config/trading_hours and picked either by the symbol itself or by the assigned type commonly there are two filter for each symbol: :full and :rth, exceptions are e.g. meats



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/cotcube-bardata/trading_hours.rb', line 11

def trading_hours(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
                  filter: ,
                  force_filter: false,  # with force_filter one would avoid falling back
                                        # to the contract_type based range set
                  headers_only: false,  # return only headers instead of ranges
                  config: init, debug: false)
  return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/

  prepare = lambda do |f|
    if headers_only
      CSV.read(f)
        .first
    else
      CSV.read(f, converters: :numeric)
         .map(&:to_a)
         .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
         .map { |x| (x.first...x.last) }
    end
  end

  sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)

  file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_#{filter}.csv"
  puts "Trying to use #{file} for #{symbol} + #{filter}" if debug
  return prepare.call(file) if File.exist? file

  file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_full.csv"
  puts "Failed. Trying to use #{file} now" if debug
  return prepare.call(file) if File.exist?(file) && (not force_filter)

  file = "#{config[:data_path]}/trading_hours/#{sym[:type]}_#{filter}.csv"
  puts "Failed. Trying to use #{file} now." if debug
  return prepare.call(file) if File.exist? file

  file = "#{config[:data_path]}/trading_hours/#{sym[:type]}_full.csv"
  puts "Failed. Trying to use #{file} now." if debug
  return prepare.call(file) if File.exist?(file) && (not force_filter)

  puts "Finally failed to find range filter for #{symbol} + #{filter}, returning 24x7".colorize(:light_yellow)
  (0...24 * 7 * 3600)
end