Module: DoroParser

Defined in:

lib/dorothy2/DEM.rb,
lib/doroParser.rb

Overview

include Pcap

Defined Under Namespace

Classes: DoroDNS, DoroFile, DoroHttp, Doroxtractr, Geoinfo, Parser

Constant Summary

CCIRC =

Host roles

1

CCDROP =

3

CCSUPPORT =

5

NONETBIOS =

true

Class Method Summary

• .start(daemon) ⇒ Object

Instance Method Summary

• #analyze_bintraffic(pcaps) ⇒ Object
• #search_irc(streamdata) ⇒ Object

Class Method Details

.start(daemon) ⇒ Object

# File 'lib/doroParser.rb', line 417

def self.start(daemon)
  daemon ||= false

  LOGGER_PARSER.info "PARSER", "Started, looking for network dumps into Dorothive.."


  infinite = true

  @insertdb = Insertdb.new


  begin
    while infinite
      begin
        pcaps = @insertdb.find_pcap
        analyze_bintraffic(pcaps)
      rescue SignalException #, RuntimeError
        LOGGER.warn "PARSER", "SIGINT".red + " Catched [1], exiting gracefully."
      end
      infinite = daemon
      LOGGER.debug "PARSER", "SLEEPING" if daemon && DEBUG
      sleep DoroSettings.env[:sleeptime].to_i if daemon # Sleeping a while if -d wasn't set, then quit.
    end
    LOGGER_PARSER.info "PARSER" , "There are no more pcaps to analyze.".yellow if DEBUG
    @insertdb.close
    exit(0)

  rescue SignalException #, RuntimeError
    LOGGER.warn "PARSER", "SIGINT".red + " Catched [2], exiting gracefully."
  end

end

Instance Method Details

#analyze_bintraffic(pcaps) ⇒ Object

# File 'lib/doroParser.rb', line 52

def analyze_bintraffic(pcaps)

  dns_list = Hash.new
  hosts = []

  pcaps.each do |dump|
    #RETRIVE MALWARE FILE INFO

    !dump['sample'].nil? && !dump['sha256'].nil? && !dump['pcapr_id'].nil? or next

    LOGGER_PARSER.info "PARSER", "Analyzing file: ".yellow + dump['sample']
    LOGGER_PARSER.info "PARSER", "Analyzing pcaprid: ".yellow + dump['pcapr_id'].gsub(/\s+/, "")


    LOGGER_PARSER.debug "PARSER", "Analyzing dump: ".yellow + dump['sha256'].gsub(/\s+/, "") if VERBOSE

    downloadir = "#{DoroSettings.env[:analysis_dir]}/#{dump['anal_id']}/downloads"


    begin

      #check if the pcap has been correctly indexed by pcapr
      xtractr = Doroxtractr.create "http://#{DoroSettings.pcapr[:host]}:#{DoroSettings.pcapr[:port]}/pcaps/1/pcap/#{dump['pcapr_id'].rstrip}"

    rescue => e
      LOGGER_PARSER.fatal "PARSER", "Can't connect to the PCAPR server."
      LOGGER_PARSER.debug "PARSER", "#{e.inspect}"
      LOGGER_PARSER.debug "PARSER", e.backtrace if VERBOSE
      return false
    end

    #it may happen that Pcapr has created an instance, but it is still indexing the pcap.
    #The following section is to avoid a crash while quering such (still-empty instance)
    #In addition, an added check is inserted, to see if the pcapr instance really match the pcap filename
    begin
      pcapr_query = URI.parse "http://#{DoroSettings.pcapr[:host]}:#{DoroSettings.pcapr[:port]}/pcaps/1/about/#{dump['pcapr_id'].rstrip}"
      pcapr_response = Net::HTTP.get_response(pcapr_query)
      pcapname = File.basename(JSON.parse(pcapr_response.body)["filename"], ".pcap")

      t ||= $1 if pcapname =~ /[0-9]*\-(.*)$/
      raise NameError.new if  t != dump['sample'].rstrip

    rescue NameError
      LOGGER_PARSER.error "PARSER", "The pcapr filename mismatchs the one present in Dorothive!. Skipping."
      next

    rescue
      LOGGER_PARSER.error "PARSER", "Can't find the PCAP into Pcapr, maybe it has not been indexed yet. Skipping."
      next
    end


    LOGGER_PARSER.info "PARSER", "Scanning network flows and searching for unknown host IPs".yellow

    @insertdb.begin_t


    xtractr.flows.each { |flow|

      begin

        flowdeep = xtractr.flows("flow.id:#{flow.id}")

        #Skipping if NETBIOS spreading activity:
        if flow.dport == 135 or flow.dport == 445
          LOGGER_PARSER.info "PARSER", "Netbios connections, skipping flow" unless NONETBIOS
          next
        end

        title = flow.title[0..200].gsub(/'/,"") #xtool bug ->')

        #insert hosts (geo) info into db
        #TODO: check if is a localaddress
        localip = xtractr.flows.first.src.address
        localnet = IPAddr.new(localip)
        multicast = IPAddr.new("224.0.0.0/4")

        #check if already present in DB
        unless(@insertdb.select("host_ips", "ip", flow.dst.address).one? || hosts.include?(flow.dst.address))
          LOGGER_PARSER.info "PARSER", "Analyzing #{flow.dst.address}".yellow
          hosts << flow.dst.address
          dest = flow.dst.address


          #insert Geoinfo
          unless localnet.include?(flow.dst.address) || multicast.include?(flow.dst.address)

            geo = Geoinfo.new(flow.dst.address.to_s)
            geoval = ["default", geo.coord, geo.country, geo.city, geo.updated, geo.asn]
            LOGGER_PARSER.debug "GEO", "Geo-values for #{flow.dst.address.to_s}: " + geo.country + " " + geo.city + " " + geo.coord if VERBOSE

            if geo.coord != "null"
              LOGGER_PARSER.debug "DB", " Inserting geo values for #{flow.dst.address.to_s} : #{geo.country}".blue  if VERBOSE
              @insertdb.insert("geoinfo",geoval)
              geoval = "currval('dorothy.geoinfo_id_seq')"
            else
              LOGGER_PARSER.warn "DB", " No Geovalues found for #{flow.dst.address.to_s}".red  if VERBOSE
              geoval = "null"
            end

          else
            LOGGER_PARSER.warn "PARSER", "#{flow.dst.address} skipped while searching for GeoInfo (it's a local network))".yellow
            geoval = 'null'
            dest = localip
          end

          #Insert host info
          #ip - geoinfo -  sbl - uptime - is_online - whois - zone - last-update - id - dns_name
          hostname = (dns_list[dest].nil? ? "null" : dns_list[dest])
          hostval = [dest, geoval, "null", "null", true, "null", "null", Util.get_time, "default", hostname]

          unless @insertdb.insert("host_ips",hostval)
            LOGGER_PARSER.debug "DB", " Skipping flow #{flow.id}: #{flow.src.address} > #{flow.dst.address}"  if VERBOSE
            next
          end

        else
          LOGGER_PARSER.debug "PARSER", "Host already #{flow.dst.address} known, skipping..." if VERBOSE
          #puts ".:: Geo info host #{flow.dst.address} already present in geodatabase, skipping.." if @insertdb.select("host_ips", "ip", flow.dst.address)
        end

        #case TCP xtractr.flows('flow.service:SMTP').first.proto = 6

        flowvals = [flow.src.address, flow.dst.address, flow.sport, flow.dport, flow.bytes, dump['sha256'], flow.packets, "default", flow.proto, flow.service.name, title, "null", flow.duration, flow.time, flow.id ]

        unless @insertdb.insert("flows",flowvals)
          LOGGER_PARSER.info "PARSER", "Skipping flow #{flow.id}: #{flow.src.address} > #{flow.dst.address}"
          next
        end

        LOGGER_PARSER.debug("DB", "Inserting flow #{flow.id} - #{flow.title}".blue) if VERBOSE

        flowid = "currval('dorothy.connections_id_seq')"

        #Layer 3 analysis
        service = flow.service.name

        case flow.proto
          when 6 then
            #check if HTTP,IRC, MAIL
            #xtractr.flows('flow.service:SMTP').first.service.name == "TCP" when unknow

            #Layer 4 analysis
            streamdata = xtractr.streamdata(flow.id)

            case service  #TODO: don't trust service field: it's based on default-port definition, do a packet inspection instead.

              #case HTTP
              when "HTTP" then
                http = DoroHttp.new(flowdeep)

                if http.method =~ /GET|POST/
                  LOGGER_PARSER.info "HTTP", "FOUND an HTTP request".white
                  LOGGER_PARSER.info "HTTP", "HTTP #{http.method}".white + " #{http.uri}".yellow

                  t = http.uri.split('/')
                  filename = (t[t.length - 1].nil? ? "noname-#{flow.id}" :  t[t.length - 1])

                  if http.method =~ /POST/
                    role_values = [CCDROP, flow.dst.address]
                    @insertdb.insert("host_roles", role_values ) unless @insertdb.select("host_roles", "role", role_values[0], "host_ip", role_values[1]).one?
                    http.data = xtractr.flowcontent(flow.id)
                    LOGGER_PARSER.debug "DB", "POST DATA SAVED IN THE DB"
                  end

                  if http.contype =~ /application/     # STORING ONLY application* type GET DATA (avoid html pages, etc)
                    LOGGER_PARSER.info "HTTP", "FOUND an Application Type".white
                    LOGGER_PARSER.debug "DB", " Inserting #{filename} downloaded file info" if VERBOSE

                    #download
                    flowdeep.each do |flow|
                      flow.contents.each do |c|

                        LOGGER_PARSER.debug("DB", "Inserting downloaded http file info from #{flow.dst.address.to_s}".blue) if VERBOSE

                        downvalues = [ DoroFile.sha2(c.body), flowid, downloadir, filename ]
                        @insertdb.insert("downloads", downvalues )

                        role_values = [CCSUPPORT, flow.dst.address]
                        @insertdb.insert("host_roles", role_values ) unless @insertdb.select("host_roles", "role", role_values[0], "host_ip", role_values[1]).one?

                        LOGGER_PARSER.debug "HTTP", "Saving downloaded file into #{downloadir}".white if VERBOSE
                        c.save("#{downloadir}/#{filename}")
                      end

                    end


                  end

                  httpvalues = "default, '#{http.method.downcase}', '#{http.uri}', #{http.size}, #{http.ssl}, #{flowid}, E'#{Insertdb.escape_bytea(http.data)}' "

                  LOGGER_PARSER.debug "DB", " Inserting http data info from #{flow.dst.address.to_s}".blue if VERBOSE
                  @insertdb.raw_insert("http_data", httpvalues)

                else
                  LOGGER_PARSER.warn "HTTP", "Not a regular HTTP traffic on flow #{flow.id}".yellow
                  LOGGER_PARSER.info "PARSER", "Trying to guess if it is IRC".white

                  if Parser.guess(streamdata.inspect).class.inspect =~ /IRC/
                    ircvalues = search_irc(streamdata)
                    ircvalues.each do |ircvalue|
                      LOGGER_PARSER.debug "DB", " Inserting IRC DATA info from #{flow.dst.address.to_s}".blue if VERBOSE
                      @insertdb.raw_insert("irc_data", ircvalue )
                      role_values = [CCIRC, flow.dst.address]
                      @insertdb.insert("host_roles", role_values ) unless @insertdb.select("host_roles", "role", role_values[0], "host_ip", role_values[1]).one?
                    end

                  else
                    LOGGER_PARSER.info "PARSER", "NO-IRC".red
                    #TODO, store UNKNOWN communication data

                  end

                end

              #case MAIL
              when "SMTP" then
                LOGGER_PARSER.info "SMTP", "FOUND an SMTP request..".white
                #insert mail
                #by from to subject data id time connection
                streamdata.each do |m|
                  mailfrom = 'null'
                  mailto = 'null'
                  mailcontent = 'null'
                  mailsubject = 'null'
                  mailhcmd = 'null'
                  mailhcont = 'null'
                  rdata = ['null', 'null']

                  case m[1]
                    when 0
                      if Parser::SMTP.header?(m[0])
                        @email = Parser::SMTP.new(m[0])
                        LOGGER_PARSER.info "SMTP", "[A]".white + @email.hcmd + " " + @email.hcont
                        if Parser::SMTP.hasbody?(m[0])
                          @email.body = Parser::SMTP.body(m[0])
                          mailto = @email.body.to
                          mailfrom = @email.body.from
                          mailsubject = @email.body.subject.gsub(/'/,"") #xtool bug ->')
                        end
                      end
                    when 1
                      rdata = Parser::SMTP.response(m[0]) if Parser::SMTP.response(m[0])
                      LOGGER_PARSER.info "SMTP", "[R]".white + rdata[0] + " " + rdata[1]
                      rdata[0] = 'null' if rdata[0].empty?
                      rdata[1] = 'null' if rdata[1].empty?

                  end
                  mailvalues = [mailfrom, mailto, mailsubject, mailcontent, "default", flowid, mailhcmd, mailhcont, rdata[0], rdata[1].gsub(/'/,"")] #xtool bug ->')
                  @insertdb.insert("emails", mailvalues )
                end

              #case FTP
              when "FTP" then
                LOGGER_PARSER.info "FTP", "FOUND an FTP request".white
              #TODO
              when "TCP" then

                LOGGER_PARSER.info "TCP", "FOUND GENERIC TCP TRAFFIC - may be a netbios scan".white
                LOGGER_PARSER.info "PARSER", "Trying see if it is IRC traffic".white

                if Parser.guess(streamdata.inspect).class.inspect =~ /IRC/
                  ircvalues = search_irc(streamdata)
                  ircvalues.each do |ircvalue|
                    LOGGER_PARSER.debug "DB", " Inserting IRC DATA info from #{flow.dst.address.to_s}".blue if VERBOSE
                    @insertdb.raw_insert("irc_data", ircvalue )
                    role_values = [CCIRC, flow.dst.address]
                    @insertdb.insert("host_roles", role_values ) unless @insertdb.select("host_roles", "role", CCIRC, "host_ip", flow.dst.address).one?
                  end
                end

              else
                LOGGER_PARSER.info "PARSER", "Unknown traffic, try see if it is IRC traffic"

                if Parser.guess(streamdata.inspect).class.inspect =~ /IRC/
                  ircvalues = search_irc(streamdata)
                  ircvalues.each do |ircvalue|
                    LOGGER_PARSER.debug "DB", " Inserting IRC DATA info from #{flow.dst.address.to_s}".blue if VERBOSE
                    @insertdb.raw_insert("irc_data", ircvalue )
                    role_values = [CCIRC, flow.dst.address]
                    @insertdb.insert("host_roles", role_values ) unless @insertdb.select("host_roles", "role", CCIRC, "host_ip", flow.dst.address).one?
                  end
                end

            end

          when 17 then
            #check if DNS

            #Layer 4 analysis
            case service
              when "DNS" then

                @i = 0
                @p = []

                flowdeep.each  do |flow|
                  flow.each do |pkt|
                    @p[@i] = pkt.payload
                    @i = @i + 1
                  end
                end

                @p.each do |d|

                  begin

                    dns = DoroDNS.new(d)

                    dnsvalues = ["default", dns.name, dns.cls_i.inspect, dns.qry?, dns.ttl, flowid, dns.address.to_s, dns.data, dns.type_i.inspect]

                    LOGGER_PARSER.debug "DB", " Inserting DNS data from #{flow.dst.address.to_s}".blue if VERBOSE
                    unless @insertdb.insert("dns_data", dnsvalues )
                      LOGGER_PARSER.error "DB", " Error while Inserting DNS data".blue
                      nex
                    end
                    dnsid = @insertdb.find_seq("dns_id_seq").first['currval']

                    if dns.qry?
                      LOGGER_PARSER.info "DNS", "DNS Query:".white + "  #{dns.name}".yellow + "  class #{dns.cls_i} type #{dns.type_i}"
                    else
                      dns_list.merge!( dns.address.to_s => dnsid)
                      LOGGER_PARSER.info "DNS", "DNS Answer:".white + "  #{dns.name}".yellow + " class #{dns.cls} type #{dns.type} ttl #{dns.ttl} " + "#{dns.address}".yellow
                    end

                  rescue => e

                    LOGGER_PARSER.error "DB", "Something went wrong while adding a DNS entry into the DB (packet malformed?) - The packet will be skipped"
                    LOGGER_PARSER.debug "DB", "#{e.inspect}" if VERBOSE
                    LOGGER_PARSER.debug "DB", e  if VERBOSE
                  end


                end
            end

          when 1 then
            #TODO: ICMP data
            #case ICMP xtractr.flows('flow.service:SMTP').first.proto = 1
          else

            LOGGER_PARSER.warn "PARSER", "Unknown protocol: #{flow.id}  -- Proto #{flow.proto}".yellow

        end

      rescue => e

        LOGGER_PARSER.error "PARSER", "Error while analyzing flow #{flow.id}: #{e.inspect}"
        LOGGER_PARSER.debug "PARSER", "#{e.backtrace}" if VERBOSE
        LOGGER_PARSER.info "PARSER", "Flow #{flow.id} will be skipped"
        next
      end

    }

    #DEBUG
    #puts "save?" 
    #gets
    @insertdb.set_analyzed(dump['sha256'])
    @insertdb.commit
  end
end

#search_irc(streamdata) ⇒ Object

# File 'lib/doroParser.rb', line 38

def search_irc(streamdata)

  ircvalues = []
  streamdata.each do |m|
    #	if m[1] == 0  #we fetch only outgoing traffic
    direction_bool = (m[1] == 0 ? false : true)
    LOGGER_PARSER.info "PARSER", "FOUND IRC DATA".white
    LOGGER_PARSER.info "IRC", "#{m[0]}".yellow

    ircvalues.push "default, currval('dorothy.connections_id_seq'), E'#{Insertdb.escape_bytea(m[0])}', #{direction_bool}"
  end
  ircvalues
end