Class: Thm::DataServices::Trafviz

Inherits:
Object
  • Object
show all
Includes:
TextProcessing
Defined in:
lib/thm/dataservices/trafviz/trafviz.rb

Defined Under Namespace

Classes: FilterManager, Main

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from TextProcessing

#text_highlighter

Constructor Details

#initialize ⇒ Trafviz

Returns a new instance of Trafviz.



40
41
42
43
44
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 40

def initialize
  @debug = false
  @reqtable, @reqtableua = String.new, String.new
  @makeurl_last = String.new
end

Instance Attribute Details

#debug=(value) ⇒ Object (writeonly)

Sets the attribute debug

Parameters:

  • value —

    the value to set the attribute debug to.



34
35
36
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 34

def debug=(value)
  @debug = value
end

#makeurl_last ⇒ Object (readonly)

Returns the value of attribute makeurl_last.



35
36
37
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 35

def makeurl_last
  @makeurl_last
end

#reqtable=(value) ⇒ Object (writeonly)

Sets the attribute reqtable

Parameters:

  • value —

    the value to set the attribute reqtable to.



34
35
36
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 34

def reqtable=(value)
  @reqtable = value
end

#reqtableua=(value) ⇒ Object (writeonly)

Sets the attribute reqtableua

Parameters:

  • value —

    the value to set the attribute reqtableua to.



34
35
36
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 34

def reqtableua=(value)
  @reqtableua = value
end

Instance Method Details

#catch_header(hdrs, comment = "") ⇒ Object

This is just an informal function when in debug mode



78
79
80
81
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 78

def catch_header(hdrs, comment="")
  print "Caught: #{hdrs} "
  puts "Header comment: #{comment}" unless comment == ""
end

#filter_header?(lkey) ⇒ Boolean

Cookie ommit as we don’t want to steal cookie data and pointless to store. Other useless headers / slight issues You can now add a comment to catch_header if you like

Returns:

  • (Boolean) 


86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 86

def filter_header?(lkey)
  puts "MY LKEY: |#{lkey}|" if @debug == true
  case 
  when lkey == "cookie"
    catch_header(lkey) if @debug == true
    return true
  when lkey == "range"
    catch_header(lkey) if @debug == true
    return true
  when lkey =~ %r=^get |^post |^head =
    catch_header(lkey, "Seen this unsure why it even occurs yet !") if @debug == true
    return true
  else
    return false
  end
end

#lkey_strip(hdrs) ⇒ Object

Filter lkey = header, rkey = requestdata



109
110
111
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 109

def lkey_strip(hdrs)
  hdrs.split(": ")[0].downcase.gsub("-", "").to_s.strip
end

#makeurl(data) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 46

def makeurl(data)
  if !request_valid?(data)
    return false
  end 
  hostn, requestn = ""
  data.each_line {|n|
    if n.split(":")[0] == "Host"
      hostn = n.split(":")[1].strip
    elsif n.split(" ")[0] =~ /^GET|^HEAD/
      requestn = n.split(" ")[1]
    end
  }
  @makeurl_last = "http://#{hostn}#{requestn}"
  puts "\e[1;37mURL: http://#{hostn}#{requestn} \e[0m\ "
end

#request_filter(data, keysamples = 2000) ⇒ Object

Filter request data and build query



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 118

def request_filter(data, keysamples=2000)
  flt = Stopwatch.new
  flt.watch('start')
  if !request_valid?(data)
    sql = "SELECT 1;"
    return sql
  end
  guid = Tools::guid
  cols, vals = String.new, String.new
  lkey, rkey = String.new, String.new
  sql_ua = String.new
  json_data_pieces = String.new
  t = 0
  json_data_hdr = "@json_template = { 'http' => { "
  json_data_ftr = " } }"
  sql = "INSERT INTO #{@reqtable} (recv_time,recv_date,guid,json_data) "
  data.each_line {|n|
    unless n.strip == ""
      if t > 0 # Don't processes GET / POST Line
        lkey, rkey = lkey_strip(n), rkey_strip(n)
        puts "LKEY: #{lkey} RKEY: #{rkey}" if @debug == true
        rkeyenc = filter_header?(lkey)
        if rkeyenc == false
          rkeyenc = rkey_decode(rkey)
          if lkey == "useragent"
            ua = Tools::ua_parser(rkeyenc)
            sql_ua = "INSERT INTO #{@reqtableua} (family, "
            sql_ua << "major, minor, " unless ua.version == nil
            sql_ua << "os, guid) "
            sql_ua << "VALUES ('#{ua.family}', "
            sql_ua << "'#{ua.version.major}', '#{ua.version.minor}', " unless ua.version == nil
            sql_ua << "'#{ua.os.to_s}', '#{guid}');"
          end
        else 
          rkey = "ommited"
        end
        if rkey != "" or lkey != ""
          prerkeyins = rkey.gsub('"', '') # Strip Quotes
          prerkeyins = "blank" if prerkeyins.strip == "" # Seems JSON values can't be "accept":""
          puts "Found Blank Value!!!" if prerkeyins == "blank"
          json_data_pieces << "'#{lkey}' => \"#{prerkeyins}\",\n" if lkey != "useragent"
        end
      end
      t += 1
    end
  }
  # Store the URL in the JSON unless its blank
  # Build JSON Manually as i bet its faster than using some JSON encoder where it has to convert from Array etc.
  json_data_pieces << "'url' => \"#{@makeurl_last}\",\n" unless @makeurl_last == ""
  # SQL for Datastore
  begin
    # Remove last , to fix hash table
    json_data_pieces.sub!(%r{,\n$}, "")
    json_eval = %Q{#{json_data_hdr}#{json_data_pieces}#{json_data_ftr}}
    puts "\e[4;36mJSON Data:\e[0m\ \n#{json_eval}"
    eval(json_eval) # Unsure why a local variable works for this in IRB
    json_data = @json_template.to_json
    remove_instance_variable("@json_template") # Hence remove instance variable here
    # Added GUID as i could extend TCP/IP capture suites in the future for HTTP traffic 
    sql = "#{sql}VALUES (NOW(), NOW(), '#{guid}', '#{json_data}');"
    flt.watch('stop')
    print "\e[4;36mFilter Time Taken:\e[0m\ "
    flt.print_stats
    return [sql, sql_ua]
  rescue => e
    pp e
  end
end

#request_valid?(data) ⇒ Boolean

Check if a request isn’t just a GET line without headers / single line Not sure if this is valid HTTP

Returns:

  • (Boolean) 


64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 64

def request_valid?(data)
  ln = 0
  data.each_line {|l|
    ln += 1
  }
  if ln > 1
    return true
  else
    puts "\e[1;31mCatch GET's without header information / Other \e[0m\ "
    return false # Due to single GET Requests to no headers 
  end
end

#rkey_decode(rkey) ⇒ Object

Right Cell eval



104
105
106
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 104

def rkey_decode(rkey)
  rkeyenc = URI.decode(rkey)
end

#rkey_strip(data) ⇒ Object 



113
114
115
 
# File 'lib/thm/dataservices/trafviz/trafviz.rb', line 113

def rkey_strip(data)
  data.split(": ")[1].to_s.strip #to_s.gsub(",", "").gsub(";", "").gsub("=", "").strip
end