Module: Cotcube::CftcSource

Includes:
Helpers
Defined in:
lib/cotcube-cftcsource.rb,
lib/cotcube-cftcsource/init.rb,
lib/cotcube-cftcsource/dates.rb,
lib/cotcube-cftcsource/fetch.rb,
lib/cotcube-cftcsource/series.rb,
lib/cotcube-cftcsource/provide.rb,
lib/cotcube-cftcsource/constants.rb,
lib/cotcube-cftcsource/distribute.rb,
lib/cotcube-cftcsource/decommission.rb

Constant Summary collapse 

{
  disagg: {
    fut: {
      current: 'https://www.cftc.gov/dea/newcot/f_disagg.txt',
      hist: 'https://www.cftc.gov/files/dea/history/fut_disagg_txt_'
    },
    com: {
      current: 'https://www.cftc.gov/dea/newcot/c_disagg.txt',
      hist: 'https://www.cftc.gov/files/dea/history/com_disagg_txt_'
    }
  },
  legacy: {
    fut: {
      current: 'https://www.cftc.gov/dea/newcot/deafut.txt',
      hist: 'https://www.cftc.gov/files/dea/history/deacot'
    },
    com: {
      current: 'https://www.cftc.gov/dea/newcot/deacom.txt',
      hist: 'https://www.cftc.gov/files/dea/history/deahistfo'
    }
  },
  financial: {
    fut: {
      current: 'https://www.cftc.gov/dea/newcot/FinFutWk.txt',
      hist: 'https://www.cftc.gov/files/dea/history/fut_fin_txt_'
    },
    com: {
      current: 'https://www.cftc.gov/dea/newcot/FinComWk.txt',
      hist: 'https://www.cftc.gov/files/dea/history/com_fin_txt_'
    }
  },
  cit: {
    com: {
      current: 'https://www.cftc.gov/dea/newcot/deacit.txt',
      hist: 'https://www.cftc.gov/files/dea/history/dea_cit_txt_'
    }
  }
}.freeze
CFTC_HEADERS = 
{ 
  legacy: i[ name date date2 cftcid cftcid2 cftcid3 cftcid3] +
  i[ std_count_oi_all std_count_ncom_long std_count_ncom_short std_count_ncom_spread std_count_com_long std_count_com_short] + 
  i[ std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short] +
  i[ old_count_oi_all old_count_ncom_long old_count_ncom_short old_count_ncom_spread old_count_com_long old_count_com_short] +
  i[ old_count_rept_long old_count_rept_short old_count_nrept_long old_count_nrept_short] +
  i[ other_count_oi_all other_count_ncom_long other_count_ncom_short other_count_ncom_spread other_count_com_long other_count_com_short] +
  i[ other_count_rept_long other_count_rept_short other_count_nrept_long other_count_nrept_short] +
  i[ std_change_oi_all std_change_ncom_long std_change_ncom_short std_change_ncom_spread std_change_com_long std_change_com_short] +
  i[ std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short] +
  i[ std_pct_oi_all std_pct_ncom_long std_pct_ncom_short std_pct_ncom_spread std_pct_com_long std_pct_com_short] +
  i[ std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short] +
  i[ old_pct_oi_all old_pct_ncom_long old_pct_ncom_short old_pct_ncom_spread old_pct_com_long old_pct_com_short] +
  i[ old_pct_rept_long old_pct_rept_short old_pct_nrept_long old_pct_nrept_short] +
  i[ other_pct_oi_all other_pct_ncom_long other_pct_ncom_short other_pct_ncom_spread other_pct_com_long other_pct_com_short] +
  i[ other_pct_rept_long other_pct_rept_short other_pct_nrept_long other_pct_nrept_short] +
  i[ std_traders_oi_all std_traders_ncom_long std_traders_ncom_short std_traders_ncom_spread std_traders_com_long std_traders_com_short] +
  i[ std_traders_rept_long std_traders_rept_short] + 
  i[ old_traders_oi_all old_traders_ncom_long old_traders_ncom_short old_traders_ncom_spread old_traders_com_long old_traders_com_short] +
  i[ old_traders_rept_long old_traders_rept_short] +
  i[ other_traders_oi_all other_traders_ncom_long other_traders_ncom_short other_traders_ncom_spread other_traders_com_long] +
  i[ other_traders_com_short other_traders_rept_long other_traders_rept_short] +
  i[ std_conc_gross4_long std_conc_gross4_short std_conc_gross8_long std_conc_gross8_short std_conc_net4_long std_conc_net4_short] +
  i[ std_conc_net8_long std_conc_net8_short old_conc_gross4_long old_conc_gross4_short old_conc_gross8_long old_conc_gross8_short] +
  i[ old_conc_net4_long old_conc_net4_short old_conc_net8_long old_conc_net8_short other_conc_gross4_long other_conc_gross4_short] +
  i[ other_conc_gross8_long other_conc_gross8_short other_conc_net4_long other_conc_net4_short other_conc_net8_long other_conc_net8_short] +
  i[ units CFTCContractMarketCode(Quotes) CFTCMarketCodeinInitials(Quotes) CFTCCommodityCode(Quotes) supplement],

  disagg: i[name date date2 cftcid cftcid2 cftcid3 cftcid3] +
  i[ std_count_oi_all std_count_prod_long std_count_prod_short std_count_swap_long std_count_swap_short std_count_swap_spread] + 
  i[ std_count_money_long std_count_money_short std_count_money_spread std_count_other_long std_count_other_short std_count_other_spread] + 
  i[ std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short] +
  i[ old_count_oi_all old_count_prod_long old_count_prod_short old_count_swap_long old_count_swap_short old_count_swap_spread] +
  i[ old_count_money_long old_count_money_short old_count_money_spread old_count_other_long old_count_other_short old_count_other_spread] +
  i[ old_count_rept_long old_count_rept_short old_count_nrept_long old_count_nrept_short] +
  i[ other_count_oi_all other_count_prod_long other_count_prod_short other_count_swap_long other_count_swap_short other_count_swap_spread] +
  i[ other_count_money_long other_count_money_short other_count_money_spread other_count_other_long other_count_other_short other_count_other_spread] +
  i[ other_count_rept_long other_count_rept_short other_count_nrept_long other_count_nrept_short] +
  i[ std_change_oi_all std_change_prod_long std_change_prod_short std_change_swap_long std_change_swap_short std_change_swap_spread] +
  i[ std_change_money_long std_change_money_short std_change_money_spread std_change_other_long std_change_other_short std_change_other_spread] +
  i[ std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short] +
  i[ std_pct_oi_all std_pct_prod_long std_pct_prod_short std_pct_swap_long std_pct_swap_short std_pct_swap_spread] +
  i[ std_pct_money_long std_pct_money_short std_pct_money_spread std_pct_other_long std_pct_other_short std_pct_other_spread] +
  i[ std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short] +
  i[ old_pct_oi_all old_pct_prod_long old_pct_prod_short old_pct_swap_long old_pct_swap_short old_pct_swap_spread old_pct_money_long] +
  i[ old_pct_money_short old_pct_money_spread old_pct_other_long old_pct_other_short old_pct_other_spread] +
  i[ old_pct_rept_long old_pct_rept_short old_pct_nrept_long old_pct_nrept_short] + 
  i[ other_pct_oi_all other_pct_prod_long other_pct_prod_short other_pct_swap_long other_pct_swap_short other_pct_swap_spread] +
  i[ other_pct_money_long other_pct_money_short other_pct_money_spread other_pct_other_long other_pct_other_short other_pct_other_spread] +
  i[ other_pct_rept_long other_pct_rept_short other_pct_nrept_long other_pct_nrept_short] + 
  i[ std_traders_oi_all std_traders_prod_long std_traders_prod_short std_traders_swap_long std_traders_swap_short std_traders_swap_spread] +
  i[ std_traders_money_long std_traders_money_short std_traders_money_spread std_traders_other_long std_traders_other_short std_traders_other_spread] +
  i[ std_traders_rept_long std_traders_rept_short] +
  i[ old_traders_oi_all old_traders_prod_long old_traders_prod_short old_traders_swap_long old_traders_swap_short old_traders_swap_spread] +
  i[ old_traders_money_long old_traders_money_short old_traders_money_spread old_traders_other_long old_traders_other_short old_traders_other_spread] +
  i[ old_traders_rept_long old_traders_rept_short] +
  i[ other_traders_oi_all other_traders_prod_long other_traders_prod_short other_traders_swap_long other_traders_swap_short] +
  i[ other_traders_swap_spread other_traders_money_long other_traders_money_short other_traders_money_spread other_traders_other_long] +
  i[ other_traders_other_short other_traders_other_spread other_traders_rept_long other_traders_rept_short] + 
  i[ std_conc_gross4_long std_conc_gross4_short std_conc_gross8_long std_conc_gross8_short std_conc_net4_long std_conc_net4_short std_conc_net8_long] +
  i[ std_conc_net8_short old_conc_gross4_long old_conc_gross4_short old_conc_gross8_long old_conc_gross8_short old_conc_net4_long old_conc_net4_short] +
  i[ old_conc_net8_long old_conc_net8_short other_conc_gross4_long other_conc_gross4_short other_conc_gross8_long other_conc_gross8_short] +
  i[ other_conc_net4_long other_conc_net4_short other_conc_net8_long other_conc_net8_short] + 
  i[ units CFTC_Contract_Market_Code_Quotes CFTC_Market_Code_Quotes CFTC_Commodity_Code_Quotes CFTC_SubGroup_Code Fut_Combined supplement],

  financial: i[ name date date2 cftcid cftcid2 cftcid3 cftcid3] +
  i[ std_count_oi_all std_count_dealers_long std_count_dealers_short std_count_dealers_spread std_count_asset_long std_count_asset_short] +
  i[ std_count_asset_spread std_count_lev_long std_count_lev_short std_count_lev_spread std_count_other_long std_count_other_short] + 
  i[ std_count_other_spread std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short] +
  i[ std_change_oi_all std_change_dealers_long std_change_dealers_short std_change_dealers_spread std_change_asset_long std_change_asset_short] +
  i[ std_change_asset_spread std_change_lev_long std_change_lev_short std_change_lev_spread std_change_other_long std_change_other_short] +
  i[ std_change_other_spread std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short] +
  i[ std_pct_oi_all std_pct_dealers_long std_pct_dealers_short std_pct_dealers_spread std_pct_asset_long std_pct_asset_short] +
  i[ std_pct_asset_spread std_pct_lev_long std_pct_lev_short std_pct_lev_spread std_pct_other_long std_pct_other_short] +
  i[ std_pct_other_spread std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short] +
  i[ std_traders_oi_all std_traders_dealers_long std_traders_dealers_short std_traders_dealers_spread std_traders_asset_long std_traders_asset_short] +
  i[ std_traders_asset_spread std_traders_lev_long std_traders_lev_short std_traders_lev_spread std_traders_other_long std_traders_other_short std] +
  i[_traders_other_spread std_traders_rept_long std_traders_rept_short] +
  i[ std_conc_gross4_long std_conc_gross4_short std_conc_gross8_long std_conc_gross8_short std_conc_net4_long std_conc_net4_short] +
  i[ std_conc_net8_long std_conc_net8_short] +
  i[ units CFTC_Contract_Market_Code_Quotes CFTC_Market_Code_Quotes CFTC_Commodity_Code_Quotes CFTC_SubGroup_Code type supplement],


  cit: i[name date date2 cftcid cftcid2 cftcid3 cftcid3] + 
  i[std_count_oi_all std_count_ncom_long std_count_ncom_short std_count_ncom_spread std_count_com_long std_count_com_short] + 
  i[std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short std_count_cit_long std_count_cit_short] + 
  i[std_change_oi_all std_change_ncom_long std_change_ncom_short std_change_ncom_spread std_change_com_long std_change_com_short] +
  i[std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short std_change_cit_long std_change_cit_short] +
  i[std_pct_oi_all std_pct_ncom_long std_pct_ncom_short std_pct_ncom_spread std_pct_com_long std_pct_com_short] +
  i[std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short std_pct_cit_long std_pct_cit_short] +
  i[std_traders_oi_all std_traders_ncom_long std_traders_ncom_short std_traders_ncom_spread std_traders_com_long std_traders_com_short] +
  i[std_traders_rept_long std_traders_rept_short std_traders_cit_long std_traders_cit_short] +
  i[units supplement]
}.freeze
CFTC_SYMBOL_EXAMPLES = 
[
  { id: "13874U", symbol: "ET", ticksize: 0.25, power: 1.25, months: "HMUZ", bcf: 1.0, reports: "LF", name: "S&P 500 MICRO" },
  { id: "209747", symbol: "NM", ticksize: 0.25, power: 0.5,  monhts: "HMUZ", bcf: 1.0, reports: "LF", name: "NASDAQ 100 MICRO" }
].freeze

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.available_cftc_ids ⇒ Object

.config_path ⇒ Object

.config_prefix ⇒ Object

.current_cot_date_on_website ⇒ Object

.date_of_last_report ⇒ Object

.distribute ⇒ Object

.fetch ⇒ Object

.init ⇒ Object

.provide ⇒ Object

.series ⇒ Object

Instance Method Details

#available_cftc_ids(config: init, print: true, sort: :exchange) ⇒ Object

print list



55
56
57
58
59
60
61
62
63
 
# File 'lib/cotcube-cftcsource/distribute.rb', line 55

def available_cftc_ids(config: init, print: true, sort: :exchange)
  command = %Q[find #{init[:data_path]}/cot/* | grep legacy | xargs head -n1  | grep , | awk -v FS=, -v OFS=, '{print $5,$4,$1}' | sed 's/"//g' | sort | uniq]
  result = CSV.parse(`#{command}`.chomp, headers: i{ exchange id name }).map{|x| x.to_h}
  result = result.uniq{|row| row[:id] }
  if print
    result.sort_by{|x| x[sort]}.each {|row| puts "#{row[:exchange]}\t#{row[:id]}\t#{row[:name]}" }
  end
  result
end

#config_path ⇒ Object 



33
34
35
 
# File 'lib/cotcube-cftcsource/init.rb', line 33

def config_path
  config_prefix + '/etc/cotcube'
end

#config_prefix ⇒ Object

def symbols(config: init, type: nil, symbol: nil)

  type = [type] unless [NilClass, Array].include?(type.class)
  if config[:symbols_file].nil?
    SYMBOL_EXAMPLES
  else
    CSV.read(config[:symbols_file], headers: i{ id symbol ticksize power months type bcf reports format name}).
      map{|row| row.to_h }.
      map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }.
      reject{|row| row[:id].nil? }.
      tap{|all| all.select!{|x| type.include?(x[:type])} unless type.nil? }.
      tap{|all| all.select!{|x| symbol.include?(x[:symbol])} unless symbol.nil? }
  end
end



21
22
23
24
25
26
27
28
29
30
31
 
# File 'lib/cotcube-cftcsource/init.rb', line 21

def config_prefix
  os       = Gem::Platform.local.os
  case os
  when 'linux'
    ''
  when 'freebsd'
    '/usr/local'
  else
    raise RuntimeError, 'unknown architecture'
  end
end

#current_cot_date_on_website ⇒ DateTime

method to extract the currently available most recent COT report from the according CFTC website

Returns:

  • (DateTime) —

    a DateTime object containing the extracted date



17
18
19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/cotcube-cftcsource/dates.rb', line 17

def current_cot_date_on_website
  uri = 'https://www.cftc.gov/MarketReports/CommitmentsofTraders/index.htm'
  raw = HTTParty.get(uri).to_s
  encoding_options = {
    :invalid           => :replace,  # Replace invalid byte sequences
    :undef             => :replace,  # Replace anything not defined in ASCII
    :replace           => '',        # Use a blank for those replacements
    :universal_newline => true       # Always break lines with \n
  }
  line = raw.each_line.select{|x| x if x =~ /Reports Dated/ }[0].encode(Encoding.find('ASCII'), encoding_options)
  date_string = line.split('Dated ').last.split('- Current').first
  DateTime.strptime(date_string, '%B %d,%Y')
end

#date_of_last_report(config: init_config) ⇒ DateTime

method to parse current years raw CFTC cot_files and return the latest available date

Returns:

  • (DateTime) —

    DateTime object containing the date found



9
10
11
12
 
# File 'lib/cotcube-cftcsource/dates.rb', line 9

def date_of_last_report(config: init_config)
  files = Dir["#{config[:data_path]}/raw/*#{Time.now.strftime('%Y')}.csv"]
  DateTime.strptime(CSV.read(files[0]).tap { |x| x.shift if x[0][0] =~ /Market/ }.map { |x| x[2] }.max, '%Y-%m-%d') rescue nil
end

#distribute(year: Time.now.year, debug: false, config: init, quiet: false) ⇒ Object

Goes through CFTCLINKS to iterate over all types of reports

Parameters:

  • opts (Hash)
  • [Integer, (Hash) —

    a customizable set of options



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/cotcube-cftcsource/distribute.rb', line 11

def distribute(year: Time.now.year, debug: false, config: init, quiet: false)
  raw_path = "#{config[:data_path]}/raw"
  cot_path = "#{config[:data_path]}/cot"
  CFTC_LINKS.each do |report, a|
  a.each do |combined, _b|
    puts ("Processing #{report}\t#{combined}\t#{year}") unless quiet

    infile  = "#{raw_path}/#{report}_#{combined}_#{year}.csv"
    outfile = ->(symbol) { "#{cot_path}/#{symbol}/#{report}_#{combined}.csv" }

    last_report = lambda do |symbol|
 CSV.parse(`tail -n1 #{cot_path}/#{symbol}/#{report}_#{combined}.csv`).last[1]
    rescue StandardError
 '1900-01-01'
    end

    cache = {}
    csv_data = CSV.read(infile).tap { |lines| lines.shift if lines[0][0] =~ /Market/ }
    csv_data.sort_by { |x| x[2] }.each do |line|
        puts "processing #{line.take(5)}" if debug
 next if line[3].length != 6

 sym = line[3]
 cache[sym] ||= last_report.call(sym)
        puts "#{cache[sym]} >= #{line[2]}" if debug
 next if cache[sym] >= line[1]

 line << "#{report} #{combined}"
 line.map! { |x| x&.strip }
 begin
          puts "Writing to #{outfile.call(sym)}: #{line.take(5)}" unless quiet
   CSV.open(outfile.call(sym), 'a+') { |f| f << line }
 rescue StandardError
   puts ("Found new id: #{sym}").colorize(:light_green)
   `mkdir #{cot_path}/#{sym}`
   CSV.open(outfile.call(sym), 'a+') { |f| f << line }
 end
    end
  end
  end
end

#fetch(year: Time.now.year, debug: false, silent: false, config: init) ⇒ Object

method to download all available historical cot report files

Parameters:

  • year (Integer, String) (defaults to: Time.now.year)
  • debug (Boolean) (defaults to: false)
  • config (Hash) (defaults to: init) 


12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/cotcube-cftcsource/fetch.rb', line 12

def fetch(year: Time.now.year, debug: false, silent: false, config: init)
  recent_report_date = `tail -n1 #{config[:data_path]}/cot/13874A/legacy_com.csv  | cut -d ',' -f 3`.chomp

  CFTC_LINKS.each do |report, a|
    a.each do |combined, _b|
      puts ("====> working on #{report}\t#{combined}") if debug
      raw_dir   = "#{config[:data_path]}/raw"

      uri = "#{CFTC_LINKS[report][combined][:hist]}#{year}.zip"
      file_data = nil
      input = HTTParty.get(uri).body
      Zip::InputStream.open(StringIO.new(input)) do |io|
        while entry = io.get_next_entry
          file_data = io.read
        end
      end
      file = "#{raw_dir}/#{report}_#{combined}_#{year}.csv"
      File.write(file, file_data)

      puts "Contents have been written to '#{file}'." if debug
      current_report_date = `head -n2 #{file} | tail -n1 | cut -d, -f3`.chomp
      if recent_report_date == current_report_date || current_report_date==''
        puts "#{report}:#{combined}\tLast date '#{current_report_date}' is same as recent_date '#{recent_report_date}', please re_run".colorize(:light_yellow)
      end
    end
  end
end

#init(config_file_name: 'cftcsource.yml', debug: false) ⇒ Object 



37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/cotcube-cftcsource/init.rb', line 37

def init(config_file_name: 'cftcsource.yml', debug: false)
  name = 'cftcsource'
  config_file = config_path + "/#{config_file_name}"

  if File.exist?(config_file)
    config      = YAML.load(File.read config_file).transform_keys(&:to_sym)
  else
    config      = {} 
  end

  defaults = { 
    data_path: config_prefix + '/var/cotcube/' + name,
  }


  config = defaults.merge(config)


  # part 2 of init process: Prepare directories

  save_create_directory = lambda do |directory_name|
    unless Dir.exist?(directory_name)
      begin
        `mkdir -p #{directory_name}`
        unless $?.exitstatus.zero?
          puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
          exit 1 unless defined?(IRB)
        end
      rescue 
        puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
        exit 1 unless defined?(IRB)
      end
    end
  end
  ['','raw','cot','series'].each do |path| 
    dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path}"
    save_create_directory.call(dir)
  end

  # eventually return config
  config
end

#move_expired_reports ⇒ Object

move expired



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/cotcube-cftcsource/decommission.rb', line 7

def move_expired_reports
  Dir['/var/cotcube/cftc/cot/*'].each do |dir|
    target = "/var/cotcube/cftc/expired/#{dir.split('/').last}/"
    Dir["#{dir}/*.csv"].each do |file|
      next unless CSV.read(file, converters: :all).last[2] < Date.today - 30

      xdebug "Moving #{file} to #{target}"
      `mkdir -p #{target}` unless File.exist? target
      `mv #{file} #{target}`
    end
    if Dir["#{dir}/*"].empty?
      xdebug "Removing #{dir}"
      `rm -rf #{dir}`
    end
  end
end

#series(symbol: nil, report: :legacy, combined: :com, config: init, after: '1900-01-01', lookback: nil, set: nil, force: false, measure: false, debug: false) ⇒ Object

  • there will be configfile, containing the indicators

a call hence

1. will try to load from the series directory
2. will try to update the series already loaded (or starting from empty)
3. provides the series (containing only the lookback period requested)\q

Raises:

  • (ArgumentError) 


12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
 
# File 'lib/cotcube-cftcsource/series.rb', line 12

def series(symbol: nil, 
           report: :legacy, 
           combined: :com,
           config: init,
           after: '1900-01-01',
           lookback: nil,
           set: nil,
           force: false,
           measure: false,
           debug: false
          )
  raise ArgumentError, "Can only build series with given symbol" if symbol.nil?
  symbol_config = Cotcube::Helpers.symbols(symbol: symbol).first
  raise ArgumentError, "Can only build series with known symbol, '#{symbol}' is unknown." if symbol_config.nil?
  puts "Using symbol_config '#{symbol_config}'" if debug

  lookback = config[:default_lookback] if lookback.nil? 
  raise ArgumentError, "Lookback may not be nil, either provide option or set :default_lookback in configfile" if lookback.nil?

  set      = config[:default_set]      if set.nil?
  set      = 'default'                 if set.nil?              

  series_dir  = "#{config[:data_path]}/series/#{symbol}"
  series_file = "#{series_dir}/#{report}_#{combined}_#{set}_#{lookback}.csv"
  puts "Guessing series_file '#{series_file}'" if debug

  `mkdir -p #{series_dir}` unless Dir.exist?(series_dir)

  header_keys = CFTC_HEADERS[report]
    .select   { |key| key.to_s =~ /std_count/ }
    .group_by { |key| key.to_s.split('_')[2] }
    .keys
    .tap      { |k|   k.delete('oi') }

  puts "Using header_keys '#{header_keys}'" if debug


  if set == 'default' or set.nil?
  # load default indicatorset
  indicators = {}
  prefix     = "std_count"
  header_keys.map do |key|
    indicators["#{prefix}_#{key}_net"] = Cotcube::Indicators.calc(a: "#{prefix}_#{key}_long", b: "#{prefix}_#{key}_short"){|a,b| a - b }
  end
  else
  # load custom indicatorset
  indicators_file = "#{config_path}/indicators-#{set}.rb"
  raise ArgumentError, "Indicatorsfile #{indicators_file} does not exist" unless File.exist? indicators_file
  load indicators_file
  indicators = Cotcube::Indicators.build(lookback: lookback, keys: header_keys)
  end

  if File.exist?(series_file) and (Time.now - File.stat(series_file).mtime).to_i / 86400.0 < 5.0 and not force and not config[:always_force]
  headers_file = "#{config[:headers_path]}/#{report}_#{combined}.json"
  if File.exist? headers_file
    provide_headers = JSON.parse(File.read(headers_file)).map{|x| x.to_sym}
  else
    provide_headers = (Cotcube::CftcSource.provide symbol: symbol, report: report, combined: combined, after: (Time.now - 20 * 86000).strftime("%Y-%m-%d")).first.keys
    File.open(headers_file, 'w'){|f| f << provide_headers.to_json }
  end
  headers         = provide_headers + indicators.keys
  puts 'Returning saved data' if debug
  puts "#{Time.now - measure}: SERIES: returning series from file" if measure

  return CSV.read(series_file, headers: headers, converters: :all).map(&:to_h)
  end

  puts "#{Time.now - measure}: SERIES: before source" if measure

  source = provide(symbol: symbol, report: report, combined: combined, config: config, after: after, measure: measure)

  puts "#{Time.now - measure}: SERIES: after source" if measure

  puts "First source: #{source.first}" if debug
  puts "#{config}" if debug

  source.each do |dataset| 
  indicators.each do |key, lambada|
    dataset[key] = lambada.call(dataset)
  end
  end
  CSV.open(series_file, 'w') { |csv| source.map{|x| csv << x.values } }

  puts "#{Time.now - measure}: SERIES: returning series from source" if measure
  source

end