Class: Geoptima::Data

Inherits:
Object
  • Object
show all
Includes:
ErrorCounter
Defined in:
lib/geoptima/data.rb

Overview

The Geoptima::Data is an entire JSON file of events

Instance Attribute Summary collapse

Attributes included from ErrorCounter

#errors

Class Method Summary collapse

Instance Method Summary collapse

Methods included from ErrorCounter

#combine_errors, #report_errors

Constructor Details

#initialize(path) ⇒ Data

Returns a new instance of Data.



679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
 
# File 'lib/geoptima/data.rb', line 679

def initialize(path)
  @path = path
  @name = File.basename(path)
#      @json = JSON.parse(File.read(path))
  @json = MultiJson.decode(File.read(path))
  @fields = {}
  @errors = {}
  if $debug
    puts "Read Geoptima: #{geoptima.to_json}"
    puts "\tSubscriber: #{subscriber.to_json}"
    puts "\tIMSI: #{self['imsi']}"
    puts "\tIMEI: #{self['imei']}"
    puts "\tMCC: #{self['MCC']}"
    puts "\tMNC: #{self['MNC']}"
    puts "\tStart: #{start}"
  end
end

Instance Attribute Details

#count ⇒ Object (readonly)

Returns the value of attribute count.



678
679
680
 
# File 'lib/geoptima/data.rb', line 678

def count
  @count
end

#json ⇒ Object (readonly)

Returns the value of attribute json.



678
679
680
 
# File 'lib/geoptima/data.rb', line 678

def json
  @json
end

#name ⇒ Object (readonly)

Returns the value of attribute name.



678
679
680
 
# File 'lib/geoptima/data.rb', line 678

def name
  @name
end

#path ⇒ Object (readonly)

Returns the value of attribute path.



678
679
680
 
# File 'lib/geoptima/data.rb', line 678

def path
  @path
end

Class Method Details

.max_start ⇒ Object 



733
734
735
 
# File 'lib/geoptima/data.rb', line 733

def self.max_start
  @@max_start ||= MAX_VALID_DATETIME
end

.min_start ⇒ Object 



730
731
732
 
# File 'lib/geoptima/data.rb', line 730

def self.min_start
  @@min_start ||= MIN_VALID_DATETIME
end

Instance Method Details

#[](key) ⇒ Object 



721
722
723
 
# File 'lib/geoptima/data.rb', line 721

def [](key)
  @fields[key] ||= subscriber[key] || subscriber[key.downcase]
end

#events ⇒ Object 



736
737
738
 
# File 'lib/geoptima/data.rb', line 736

def events
  @events ||= make_events
end

#events_names ⇒ Object 



745
746
747
 
# File 'lib/geoptima/data.rb', line 745

def events_names
  events.keys.sort
end

#find_first_and_last(events_data) ⇒ Object 



839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
 
# File 'lib/geoptima/data.rb', line 839

def find_first_and_last(events_data)
  @first = nil
  @last = nil
  events_data.each do |event_type,data|
    if data.length > 0
      @first ||= data[0]
      @last ||= data[-1]
      @first = data[0] if(@first && @first.time > data[0].time)
      @last = data[-1] if(@last && @last.time < data[-1].time)
    end
  end
  if $debug
    puts "For data: #{self}"
    puts "\tFirst event: #{@first}"
    puts "\tLast event:  #{@last}"
  end
end

#first ⇒ Object 



739
740
741
 
# File 'lib/geoptima/data.rb', line 739

def first
  events && @first
end

#geoptima ⇒ Object 



706
707
708
 
# File 'lib/geoptima/data.rb', line 706

def geoptima
  @geoptima ||= json['geoptima']
end

#id ⇒ Object 



718
719
720
 
# File 'lib/geoptima/data.rb', line 718

def id
  @id ||= self['id'] || self['udid'] || self['imei']
end

#imei ⇒ Object 



715
716
717
 
# File 'lib/geoptima/data.rb', line 715

def imei
  @imei ||= self['imei']
end

#incr_error(name) ⇒ Object 



696
697
698
699
 
# File 'lib/geoptima/data.rb', line 696

def incr_error(name)
  @errors[name] ||= 0
  @errors[name] += 1
end

#last ⇒ Object 



742
743
744
 
# File 'lib/geoptima/data.rb', line 742

def last
  events && @last
end

#make_events ⇒ Object 



760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
 
# File 'lib/geoptima/data.rb', line 760

def make_events
  @count = 0
  @events_metadata = make_hash('events-metadata')
  events_data = {}
  geoptima['events'].each do |data|
    events = data['values']
    event_type = data.keys.reject{|k| k=~/values/}[0]
    event_count = data[event_type].to_i
    header = @events_metadata[event_type]
    # If the JSON is broken (known bug on some releases of the iPhone app)
    # Then get the header information from a list of known headers
    unless header
      puts "No header found for '#{event_type}', trying known Geoptima headers"
      header = Event::KNOWN_HEADERS[event_type]
      puts "Found known header '#{event_type}' => #{header.inspect}" if(header)
    end
    # Double-check the header length matches a multiple of the data length
    if header
      mismatch_records = events.length - header.length * event_count
      if mismatch_records != 0
        puts "'#{event_type}' header length #{header.length} incompatible with data length #{events.length} and record count #{event_count}"
        proposed_header = header
        header = nil
        incr_error "Metadata mismatch"
        if events.length == proposed_header.length * event_count * 2 && event_type == 'roundtrip'
          incr_error "#4593 iPhone roundtrip event counts"
          event_count *= 2
          header = proposed_header
        elsif Event::ALT_HEADERS.keys.grep(event_type).length>0
          incr_error "#{Event::HEADER_BUGS[event_type]} #{event_type}"
          [Event::KNOWN_HEADERS[event_type],*(Event::ALT_HEADERS[event_type])].each do |alt_header|
            puts "Trying alternative header: #{alt_header.inspect}" if($debug)
            if alt_header && (events.length == alt_header.length * event_count)
              puts "\tAlternative header length matches: #{alt_header.inspect}" if($debug)
              records_valid = (0...[10,event_count].min).inject(true) do |vt,ri|
                timeoffset = events[ri*alt_header.length]
                vt &&= timeoffset.is_a?(Fixnum)
              end
              if records_valid
                header = alt_header
                puts "Found alternative header that matches #{event_type}: #{header.join(',')}"
                break
              end
            end
          end
        end
      end
    else
      puts "No header found for event type: #{event_type}"
    end
    # Now process the single long data array into a list of events with timestamps
    if header
      events_data[event_type] = (0...event_count).inject([]) do |a,block|
        index = header.length * block
        record = events[index...(index+header.length)]
        if record && record.length == header.length
          @count += 1
          event = Event.new(self,start,event_type,header,record,a[-1])
          combine_errors event
          puts "About to add new event #{event} to existing list of #{a.length} events (previous: #{a[-1] && a[-1].time})" if($debug)
          a << event
        else
          puts "Invalid '#{event_type}' data block #{block}: #{record.inspect}"
          incr_error "Invalid data block"
          break a
        end
      end
      if $debug
        puts "Have '#{event_type}' event data:"
        puts "\t#{header.join("\t")}"
        events_data[event_type].each do |d|
          puts "\t#{d.data.join("\t")}"
        end
      end
    end
  end
  find_first_and_last(events_data)
  events_data
end

#make_hash(name) ⇒ Object 



748
749
750
751
752
753
754
755
756
757
758
759
 
# File 'lib/geoptima/data.rb', line 748

def make_hash(name)
  puts "About to process json hash: #{geoptima[name].to_json}" if($debug)
  geoptima[name].inject({}) do |a,md|
    if md.respond_to? 'keys'
      key = md.keys[0]
      a[key]=md[key]
    else
      puts "Invalid hash format for '#{name}': #{md.to_json[0..70]}..."
    end
    a
  end
end

#puts(line) ⇒ Object 



703
704
705
 
# File 'lib/geoptima/data.rb', line 703

def puts line
  Kernel.puts "#{name}: #{line}"
end

#start ⇒ Object 



724
725
726
 
# File 'lib/geoptima/data.rb', line 724

def start
  @start ||= subscriber['start'] && DateTime.parse(subscriber['start'].gsub(/Asia\/Bangkok/,'GMT+7'))#.gsub(/Mar 17 2044/,'Feb 14 2012'))
end

#subscriber ⇒ Object 



712
713
714
 
# File 'lib/geoptima/data.rb', line 712

def subscriber
  @subscriber ||= geoptima['subscriber']
end

#to_s ⇒ Object 



700
701
702
 
# File 'lib/geoptima/data.rb', line 700

def to_s
  json.to_json[0..100]
end

#valid? ⇒ Boolean

Returns:

  • (Boolean) 


727
728
729
 
# File 'lib/geoptima/data.rb', line 727

def valid?
  start && start >= (Data.min_start-1) && start < Data.max_start
end

#version ⇒ Object 



709
710
711
 
# File 'lib/geoptima/data.rb', line 709

def version
  @version ||= geoptima['Version'] || geoptima['version']
end