Module: Google::SafeBrowsingHelper

Extended by:
SafeBrowsingHelper
Included in:
SafeBrowsingHelper
Defined in:
lib/rails/generators/google/helper/templates/safe_browsing_helper.rb

Instance Method Summary collapse

Instance Method Details

#can_request_full_hash?(hash_prefix) ⇒ Boolean

Returns:

  • (Boolean) 


63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 63

def can_request_full_hash? hash_prefix
  now = Time.now
  request = Google::SafeBrowsingFullHashRequest.where(prefix: hash_prefix).first
  if request.nil?
    true
  else
    if request.state == SafeBrowsingFullHashRequest::COMPLETED
      if request.requested_at < now - Google::CONFIG['full_length_hash_expires']
        true
      else
        Rails.logger.warn "Full hash [#{hash_prefix}] requested successfully recently, skip this time."
        false
      end
    else
      attempts = request.attempts.nil?? 1 : request.attempts
      max_delay = Google::CONFIG['full_length_hash_backoff_delay_max']
      delay = attempts * Google::CONFIG['full_length_hash_backoff_delay']
      delay = max_delay if delay > max_delay
      if request.requested_at > now - delay
        Rails.logger.warn "Full hash [#{hash_prefix}] request in backoff mode. Wait time is #{delay} seconds"
        false
      else
        true
      end
    end
  end
end

#find_host_key_hits(prefixes) ⇒ Object 



141
142
143
144
145
146
147
148
149
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 141

def find_host_key_hits prefixes
  ret = []
  SafeBrowsingShavar.add_host_keys(prefixes).each do |add|
    if SafeBrowsingShavar.find_subs_for_add(add.chunk_num, add.host_key, add.prefix).empty?
      ret << add
    end
  end
  ret
end

#find_prefix_key_hits(host_keys, prefixes) ⇒ Object 



151
152
153
154
155
156
157
158
159
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 151

def find_prefix_key_hits host_keys, prefixes
  ret = []
  SafeBrowsingShavar.add_host_prefixes(host_keys, prefixes).each do |add|
    if SafeBrowsingShavar.find_subs_for_add(add.chunk_num, add.host_key, add.prefix).empty?
      ret << add
    end
  end
  ret
end

#full_hash_cache_hits(full_hashes) ⇒ Object 



161
162
163
164
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 161

def full_hash_cache_hits full_hashes
  SafeBrowsingFullHash.includes(:list).where(value: full_hashes)
                      .where('updated_at > ?', Time.now - Google::CONFIG['full_length_hash_expires'])
end

#get_full_hash_list(hash_prefix_arr) ⇒ Object

hash_prefix_arr: [‘5b3583c0’, ‘b3e357a6’]



20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 20

def get_full_hash_list hash_prefix_arr
  ret_full_hashes = []
  api =  SafeBrowsingClient.new

  return ret_full_hashes if hash_prefix_arr.empty?
  
  begin
    full_hash_objs = api.full_hash(hash_prefix_arr) || {}
    save_full_hash_requests(hash_prefix_arr, SafeBrowsingFullHashRequest::COMPLETED)
  rescue Google::Error::NoContent => e
    Rails.logger.warn "NoContent Error for hash prefixes [#{hash_prefix_arr.join(', ')}]"
    save_full_hash_requests(hash_prefix_arr, SafeBrowsingFullHashRequest::COMPLETED)
    return ret_full_hashes
  rescue Exception => e
    Rails.logger.error "Error backtrace #{e.backtrace.join(%Q(\n))}"
    Rails.logger.warn "Error (#{e.inspect}) for hash prefixes [#{hash_prefix_arr.join(', ')}], continue..."
    save_full_hash_requests(hash_prefix_arr, e.message)
    return ret_full_hashes
  end
    
  full_hash_objs.keys.each do |list|
    full_hash_objs[list].keys.each do |add_chunk_num|
      full_hash_objs[list][add_chunk_num].each do |full_hash|
        unless (list_obj = SafeBrowsingList.list_by_name(list)).nil?
          Rails.logger.info "Updating full hash data with #{list}:#{add_chunk_num}:#{full_hash}"
          local = SafeBrowsingFullHash.where(value: full_hash).first

          if local
            local.touch
          else
            local = SafeBrowsingFullHash.create(value: full_hash, add_chunk_num: add_chunk_num, 
                                    google_safe_browsing_list_id: list_obj.id)
          end
          
          ret_full_hashes << full_hash
        end
      end
    end
  end
  
  ret_full_hashes
end

#hits_to_category(url, full_hits) ⇒ Object 



132
133
134
135
136
137
138
139
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 132

def hits_to_category url, full_hits
  (ret = {}).tap do
    ret[url] ||= []
    full_hits.each do |hit|
      ret[url] << hit.list.name
    end
  end
end

#lookup_url(url) ⇒ Object 



91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 91

def lookup_url url
  canon_url = Google::UrlCanonicalizer.apply(url)
  hosts_set, urls_set = Google::UrlScramble.gen(canon_url)
  full_hits = []
  
  unless hosts_set.empty?
    host_shas = hosts_set.map {|x| ShaUtil.sha256_hex(x, 8)}
    host_hits = find_host_key_hits(host_shas).map(&:host_key)
    
    if host_hits.empty?
      Rails.logger.info "No host key prefix found. Return."
      return hits_to_category(url, full_hits)
    else
      url_shas = urls_set.map {|x| ShaUtil.sha256_hex(x, 8)}
      prefix_hits = find_prefix_key_hits(host_shas, url_shas).map(&:prefix)
      
      candidate_prefixes = (host_hits + prefix_hits).uniq
      full_hash_expressions = urls_set.map {|x| ShaUtil.sha256_hex(x)}.select {|x| candidate_prefixes.include?(x.first(8))}
      full_hits = full_hash_cache_hits(full_hash_expressions)

      if full_hits.empty?
        warm_prefixes = candidate_prefixes.select {|x| !can_request_full_hash?(x)}
        candidate_prefixes -= warm_prefixes
        unless candidate_prefixes.empty?
          Rails.logger.info "Asking Google for full length hash #{candidate_prefixes}"
          get_full_hash_list(candidate_prefixes) 
          full_hits = full_hash_cache_hits(full_hash_expressions)
        end
      end
    end
  end

  if full_hits.empty?
    Rails.logger.info "Return no hits..."
  else
    Rails.logger.info "Return hits #{full_hits.inspect}..."
  end

  hits_to_category(url, full_hits)
end

#save_full_hash_requests(prefixes, state, requested_at = Time.now) ⇒ Object 



5
6
7
8
9
10
11
12
13
14
 
# File 'lib/rails/generators/google/helper/templates/safe_browsing_helper.rb', line 5

def save_full_hash_requests prefixes, state, requested_at = Time.now
  prefixes.each do |pre|
    obj = SafeBrowsingFullHashRequest.where(prefix: pre).first
    if obj.nil?
      obj = SafeBrowsingFullHashRequest.create(prefix: pre, requested_at: requested_at, state: state)
    else
      obj.update_attributes(requested_at: requested_at, state: state)
    end
  end
end