Module: Rbkb::Extends::String

Included in:
RbkbString, String
Defined in:
lib/rbkb/extends/string.rb

Instance Method Summary collapse

Instance Method Details

#^(other) ⇒ Object

convert bytes to number then xor against another byte-string or number



224
225
226
227
 
# File 'lib/rbkb/extends/string.rb', line 224

def ^(other)
  other = other.dat_to_num unless other.is_a? Numeric
  (dat_to_num ^ other) # .to_bytes
end

#b64(len = nil) ⇒ Object

Base64 encode



40
41
42
43
44
45
46
47
 
# File 'lib/rbkb/extends/string.rb', line 40

def b64(len = nil)
  ret = [self].pack('m').gsub("\n", '')
  if len and len.is_a?(Numeric)
    ret.scan(/.{1,#{len}}/).join("\n") + "\n"
  else
    ret
  end
end

#bgrep(find, align = nil) ⇒ Object

Binary grep

Parameters:

find  : A Regexp or string to search for in self
align : nil | numeric alignment (matches only made if aligned)


477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
 
# File 'lib/rbkb/extends/string.rb', line 477

def bgrep(find, align = nil)
  raise 'alignment must be a integer >= 0' if align and (!align.is_a?(Integer) or align < 0)

  dat = self
  search = if find.is_a? Regexp
             lambda do |mf, buf|
               if m = mf.match(buf)
                 mtch = m[0]
                 off, endoff = m.offset(0)
                 [off, endoff, mtch]
               end
             end
           else
             lambda do |s, buf|
               if off = buf.index(s)
                 [off, off + s.size, s]
               end
             end
           end

  ret = []
  pos = 0
  while (res = search.call(find, dat[pos..-1].force_to_binary))
    off, endoff, match = res
    if align and (pad = (pos + off).pad(align)) != 0
      pos += pad
    else
      hit = [pos + off, pos + endoff, match]
      ret << hit if !block_given? or yield([pos + off, pos + endoff, match])
      pos += endoff
    end
  end
  ret
end

#camelize ⇒ Object

Converts a '_' delimited string to CamelCase like 'foo_class' into 'FooClass'. See also: camelize_meth, decamelize



319
320
321
 
# File 'lib/rbkb/extends/string.rb', line 319

def camelize
  gsub(/(^|_)([a-z])/) { ::Regexp.last_match(2).upcase }
end

#camelize_meth ⇒ Object

Converts a '_' delimited string to method style camelCase like 'foo_method' into 'fooMethod'. See also: camelize, decamelize



326
327
328
 
# File 'lib/rbkb/extends/string.rb', line 326

def camelize_meth
  gsub(/_([a-z])/) { ::Regexp.last_match(1).upcase }
end

#char_frequency ⇒ Object

Produces a character frequency distribution histogram in descending order. Example:

pp some_english_text.char_frequency()

[[" ", 690],
["e", 354],
["t", 242],
["o", 233],
["i", 218],
...
]


190
191
192
193
194
195
196
197
 
# File 'lib/rbkb/extends/string.rb', line 190

def char_frequency
  hits = {}
  each_byte do |c|
    hits[c.chr] ||= 0
    hits[c.chr] += 1
  end
  hits.to_a.sort { |a, b| b[1] <=> a[1] }
end

#class_name ⇒ Object

convert a string to its idiomatic ruby class name



344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
 
# File 'lib/rbkb/extends/string.rb', line 344

def class_name
  r = ''
  up = true
  each_byte do |c|
    if c == 95
      if up
        r << '::'
      else
        up = true
      end
    else
      m = up ? :upcase : :to_s
      r << c.chr.send(m)
      up = false
    end
  end
  r
end

#const_lookup(ns = Object) ⇒ Object

Returns a reference to actual constant for a given name in namespace can be used to lookup classes from enums and such



365
366
367
368
369
 
# File 'lib/rbkb/extends/string.rb', line 365

def const_lookup(ns = Object)
  if c = ns.constants.select { |n| n == class_name } and !c.empty?
    ns.const_get(c.first)
  end
end

#crc32 ⇒ Object

returns CRC32 checksum for the string object



263
264
265
266
267
268
269
270
271
272
273
274
275
 
# File 'lib/rbkb/extends/string.rb', line 263

def crc32
  ## pure ruby version. slower, but here for reference (found on some forum)
  #  r = 0xFFFFFFFF
  #  self.each_byte do |b|
  #    r ^= b
  #    8.times do
  #      r = (r>>1) ^ (0xEDB88320 * (r & 1))
  #    end
  #  end
  #  r ^ 0xFFFFFFFF
  ## or... we can just use:
  Zlib.crc32 self
end

#cstring(off = 0) ⇒ Object

Returns a single null-terminated ascii string from beginning of self. This will return the entire string if no null is encountered.

Parameters:

off = specify an optional beggining offset


258
259
260
 
# File 'lib/rbkb/extends/string.rb', line 258

def cstring(off = 0)
  self[off, (index("\x00") || size)]
end

#d64 ⇒ Object

Base64 decode



50
51
52
 
# File 'lib/rbkb/extends/string.rb', line 50

def d64
  unpack1('m')
end

#dat_to_num(order = Rbkb::DEFAULT_BYTE_ORDER) ⇒ Object

A "generalized" lazy bytestring -> numeric converter.

Bonus: should work seamlessly with really large strings.

>> ("\xFF"*10).dat_to_num
=> 1208925819614629174706175
>> ("\xFF"*20).dat_to_num
=> 1461501637330902918203684832716283019655932542975

Parameters:

  • order (Symbol) (defaults to: Rbkb::DEFAULT_BYTE_ORDER) —

    :big or :little endian (default is :big)



151
152
153
154
155
156
157
 
# File 'lib/rbkb/extends/string.rb', line 151

def dat_to_num(order = Rbkb::DEFAULT_BYTE_ORDER)
  b = bytes
  b = b.to_a.reverse if order == :little
  r = 0
  b.each { |c| r = ((r << 8) | c) }
  r
end

#decamelize ⇒ Object

Converts a CamelCase or camelCase string into '_' delimited form like 'FooBar' or 'fooBar' into 'foo_bar'.

Note: This method only handles camel humps. Strings with consecutive uppercase chars like 'FooBAR' will be converted to 'foo_bar'

See also: camelize, camelize_meth



337
338
339
340
341
 
# File 'lib/rbkb/extends/string.rb', line 337

def decamelize
  gsub(/(^|[a-z])([A-Z])/) do
    ::Regexp.last_match(1).empty? ? ::Regexp.last_match(2) : "#{::Regexp.last_match(1)}_#{::Regexp.last_match(2)}"
  end.downcase
end

#dehexdump(opt = {}) ⇒ Object Also known as: dedump, undump, unhexdump

Converts a hexdump back to binary - takes the same options as hexdump(). Fairly flexible. Should work both with 'xxd' and 'hexdump -C' style dumps.



441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
 
# File 'lib/rbkb/extends/string.rb', line 441

def dehexdump(opt = {})
  s = self
  out = opt[:out] || StringIO.new
  l = opt[:len]
  len = l && l.positive? ? opt[:len] : 16

  hcrx = /[A-Fa-f0-9]/
  dumprx = /^(#{hcrx}+):?\s*((?:#{hcrx}{2}\s*){0,#{len}})/
  off = opt[:start_addr] || 0

  i = 1
  # iterate each line of hexdump
  s.split(/\r?\n/).each do |hl|
    # match and check offset
    raise "Hexdump parse error on line #{i} #{s}" unless dumprx.match(hl) && ::Regexp.last_match(1).hex == off

    i += 1
    # take the data chunk and unhexify it
    raw = ::Regexp.last_match(2).unhexify
    off += out.write(raw)
  end

  return unless out.instance_of?(StringIO)

  out.string.force_to_binary
end

#entropy ⇒ Object

calculates entropy in string

TQBF's description: "I also added a chi-squared test to quickly figure out entropy of a string, in "bits of randomness per byte". This is useful, so..."



166
167
168
169
170
171
172
173
174
175
 
# File 'lib/rbkb/extends/string.rb', line 166

def entropy
  e = 0
  sz = bytesize.to_f
  b = bytes
  0.upto(255) do |i|
    x = b.count(i) / sz
    e += - x * (Math.log(x) / Math.log(2)) if x > 0
  end
  e
end

#force_to_binary ⇒ Object

This is so disgusting... but str.encode('BINARY') fails hard whenever certain utf-8 characters present. Try "\xca\xfe\xba\xbe".encode('BINARY') for kicks.



12
13
14
 
# File 'lib/rbkb/extends/string.rb', line 12

def force_to_binary
  dup.force_encoding('binary')
end

#hex_to_num(order = Rbkb::DEFAULT_BYTE_ORDER) ⇒ Object

Converts a hex value to numeric.

Parameters:

order => :big or :little endian (default is :big)


136
137
138
 
# File 'lib/rbkb/extends/string.rb', line 136

def hex_to_num(order = Rbkb::DEFAULT_BYTE_ORDER)
  unhexify.dat_to_num(order)
end

#hexdump(opt = {}) ⇒ Object

Returns or prints a hexdump in the style of 'hexdump -C'

:len => optionally specify a length other than 16 for a wider or thinner dump. If length is an odd number, it will be rounded up.

:out => optionally specify an alternate IO object for output. By default, hexdump will output to STDOUT. Pass a StringIO object and it will return it as a string.

Example:

Here's the default behavior done explicitely:

>> xxd = dat.hexdump(:len => 16, :out => StringIO.new)
=> <a string containing hexdump>

Here's how to change it to STDERR

>> xxd = dat.hexdump(:len => 16, :out => STDERR)
<prints hexdump on STDERR>
-> nil # return value is nil!


398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
 
# File 'lib/rbkb/extends/string.rb', line 398

def hexdump(opt = {})
  s = self
  out = opt[:out] || StringIO.new
  len = (opt[:len] and opt[:len] > 0) ? opt[:len] + (opt[:len] % 2) : 16

  off = opt[:start_addr] || 0
  offlen = opt[:start_len] || 8

  hlen = len / 2

  s.bytes.each_slice(len) do |m|
    out.write(off.to_s(16).rjust(offlen, '0') + '  ')

    i = 0
    m.each do |c|
      out.write '%0.2x ' % c
      out.write(' ') if (i += 1) == hlen
    end

    out.write('   ' * (len - i)) # pad
    out.write(' ') if i < hlen

    out.write(' |')
    m.each do |c|
      if c > 0x19 and c < 0x7f
        out.write(c.chr)
      else
        out.write('.')
      end
    end
    out.write("|\n")
    off += m.length
  end

  out.write(off.to_s(16).rjust(offlen, '0') + "\n")

  return unless out.class == StringIO

  out.string
end

#hexify(opts = {}) ⇒ Object

Convert a string to ASCII hex string. Supports a few options for format:

Parameters:

  • opts (Hash) (defaults to: {})

Options Hash (opts):

  • :delim (String) —

    delimter between each hex byte

  • :prefix (String) —

    prefix before eaech hex byte

  • :suffix (String) —

    suffix after each hex byte



98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
 
# File 'lib/rbkb/extends/string.rb', line 98

def hexify(opts = {})
  delim = opts[:delim]
  pre = opts[:prefix] || ''
  suf = opts[:suffix] || ''

  if (rx = opts[:rx]) and !rx.is_a? Regexp
    raise 'rx must be a regular expression for a character class'
  end

  hx = Rbkb::HEXCHARS

  out = []

  each_byte do |c|
    hc = if rx and !rx.match c.chr
           c.chr
         else
           pre + (hx[(c >> 4)] + hx[(c & 0xf)]) + suf
         end
    out << hc
  end
  out.join(delim)
end

#ishex? ⇒ Boolean

Returns:

  • (Boolean) 


16
17
18
 
# File 'lib/rbkb/extends/string.rb', line 16

def ishex?
  (self =~ /\A([a-f0-9]{2}|\n)+\Z/i) == 0
end

#lalign(a, p = ' ') ⇒ Object

left-align to 'a' alignment padded with 'p'



63
64
65
66
67
68
 
# File 'lib/rbkb/extends/string.rb', line 63

def lalign(a, p = ' ')
  p ||= ' '
  l = bytesize
  pad = l.pad(a)
  ljust(pad + l, p)
end

#md5 ⇒ Digest::MD5 Also known as: md5sum

Returns the MD5 digest/checksum for this string.

Returns:

  • (Digest::MD5) —

    the MD5 digest/checksum for this string.



278
279
280
281
282
 
# File 'lib/rbkb/extends/string.rb', line 278

def md5
  d = Digest::MD5.new
  d.update(self)
  d
end

#pipe_magick(arg = '') ⇒ Object

This attempts to identify a blob of data using 'file(1)' via popen3 (using popen3 because IO.popen blows) Tried doing this with a fmagic ruby extention to libmagic, but it was a whole lot slower.



304
305
306
307
308
309
310
311
312
313
314
 
# File 'lib/rbkb/extends/string.rb', line 304

def pipe_magick(arg = '')
  ret = ''
  Open3.popen3("file #{arg} -") do |w, r, _e|
    w.write self
    w.close
    ret = r.read
    r.close
    ret.sub!(%r{^/dev/stdin: }, '')
  end
  ret
end

#ralign(a, p = ' ') ⇒ Object

right-align to 'a' alignment padded with 'p'



55
56
57
58
59
60
 
# File 'lib/rbkb/extends/string.rb', line 55

def ralign(a, p = ' ')
  p ||= ' '
  l = bytesize
  pad = l.pad(a)
  rjust(pad + l, p)
end

#randomize ⇒ Object

String randomizer



236
237
238
 
# File 'lib/rbkb/extends/string.rb', line 236

def randomize
  split('').randomize.to_s
end

#randomize! ⇒ Object

In-place string randomizer



241
242
243
 
# File 'lib/rbkb/extends/string.rb', line 241

def randomize!
  replace(randomize)
end

#rotate_bytes(k = 0) ⇒ Object

Byte rotation as found in lame ciphers.



230
231
232
233
 
# File 'lib/rbkb/extends/string.rb', line 230

def rotate_bytes(k = 0)
  k = (256 + k) if k < 0
  bytes.map { |c| ((c + k) & 0xff).chr }.join
end

#sha1 ⇒ Digest::SHA1

Returns the SHA1 digest for this string.

Returns:

  • (Digest::SHA1) —

    the SHA1 digest for this string.



286
287
288
289
290
 
# File 'lib/rbkb/extends/string.rb', line 286

def sha1
  d = Digest::SHA1.new
  d.update(self)
  d
end

#sha2 ⇒ Digest::SHA2 Also known as: sha256

Returns the SHA2 digest for this string.

Returns:

  • (Digest::SHA2) —

    the SHA2 digest for this string.



293
294
295
296
297
 
# File 'lib/rbkb/extends/string.rb', line 293

def sha2
  d = Digest::SHA2.new
  d.update(self)
  d
end

#starts_with?(dat) ⇒ Boolean

Does string "start with" dat? No clue whether/when this is faster than a regex, but it is easier to type.

Returns:

  • (Boolean) 


247
248
249
 
# File 'lib/rbkb/extends/string.rb', line 247

def starts_with?(dat)
  index(dat) == 0
end

#strings(opts = {}) ⇒ Object

A 'strings' method a-la unix strings utility. Finds printable strings in a binary blob. Supports ASCII and little endian unicode (though only for ASCII printable character.)

Parameters and options:

  • Use the :minimum parameter to specify minimum number of characters to match. (default = 6)

  • Use the :encoding parameter as one of :ascii, :unicode, or :both (default = :ascii)

  • The 'strings' method uses Regexp under the hood. Therefore you can pass a character class for "valid characters" with :valid (default = /[\r\n [:print:]]/)

  • Supports an optional block, which will be passed |offset, type, string| for each match. The block's boolean return value also determines whether the match passes or fails (true or false/nil) and gets returned by the function.

Return Value:

Returns an array consisting of matches with the following elements:

[[start_offset, end_offset, string_type, string], ...]
  • string_type will be one of :ascii or :unicode
  • end_offset will include the terminating null character
  • end_offset will include all null bytes in unicode strings (including
  • both terminating nulls)

If strings are null terminated, the trailing null IS included in the end_offset. Unicode matches will also include null bytes.

Todos?

  • better unicode support (i.e. not using half-assed unicode)
  • support other encodings such as all those the binutils strings does?


551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
 
# File 'lib/rbkb/extends/string.rb', line 551

def strings(opts = {})
  opts[:encoding] ||= :both
  min = opts[:minimum] || 6

  raise 'Minimum must be numeric and > 0' unless min.is_a? Numeric and min > 0

  acc = /[\s[:print:]]/
  ucc = /(?:#{acc}\x00)/

  arx = /(#{acc}{#{min}}#{acc}*\x00?)/
  urx = /(#{ucc}{#{min}}#{ucc}*(?:\x00\x00)?)/

  rx = case (opts[:encoding] || :both).to_sym
       when :ascii
         mtype_blk = ->(_x) { :ascii }
         arx
       when :unicode
         mtype_blk = ->(_x) { :unicode }
         urx
       when :both
         mtype_blk = ->(x) { x[2].nil? ? :ascii : :unicode }

         Regexp.union(arx, urx)
       else
         raise 'Encoding must be :unicode, :ascii, or :both'
       end

  ret = []

  # wow ruby 1.9 string encoding is a total cluster
  force_to_binary.scan(rx) do
    mtch = $~

    stype = mtype_blk.call(mtch)

    startoff, endoff = mtch.offset(0)
    mret = [startoff, endoff, stype, mtch[0]]

    # yield to a block for additional criteria
    next if block_given? and !yield(*mret)

    ret << mret
  end

  ret
end

#substitution(keymap) ⇒ Object

(en|de)ciphers using a substition cipher en/decoder ring in the form of a hash with orig => substitute mappings



211
212
213
 
# File 'lib/rbkb/extends/string.rb', line 211

def substitution(keymap)
  split('').map { |c| (sub = keymap[c]) ? sub : c }.join
end

#substitution_xor(keymap) ⇒ Object

(en|de)crypts using a substition xor en/decoder ring in the form of a hash with orig => substitute mappings. Used in conjunction with char_frequency, this sometimes provides a shorter way to derive a single character xor key used in conjunction with char_frequency.



219
220
221
 
# File 'lib/rbkb/extends/string.rb', line 219

def substitution_xor(keymap)
  split('').map { |c| (sub = keymap[c]) ? sub.xor(c) : c }.join
end

#to_stringio ⇒ Object

Return a self encapsulated in a StringIO object. This is handy.



372
373
374
 
# File 'lib/rbkb/extends/string.rb', line 372

def to_stringio
  StringIO.new(self)
end

#unhexify(d = /\s*/) ⇒ Object

Convert ASCII hex string to raw.

@param [String] d optional 'delimiter' between hex bytes
              (zero+ spaces by default)


126
127
128
 
# File 'lib/rbkb/extends/string.rb', line 126

def unhexify(d = /\s*/)
  force_to_binary.strip.gsub(/([A-Fa-f0-9]{1,2})#{d}?/) { ::Regexp.last_match(1).hex.chr }
end

#urldec(opts = nil) ⇒ Object

Undo percent-hexified url encoded string

Parameters:

  • opts (Hash) (defaults to: nil) —

    Optional parameters

Options Hash (opts):

  • :noplus (optional, Boolean) —

    Treat + as a literal '+', instead of a space



74
75
76
77
78
79
80
81
82
 
# File 'lib/rbkb/extends/string.rb', line 74

def urldec(opts = nil)
  if opts.nil? or opts.empty?
    CGI.unescape(self)
  else
    s = self
    s.gsub!('+', ' ') unless opts[:noplus]
    s.gsub(/%([A-Fa-f0-9]{2})/) { ::Regexp.last_match(1).hex.chr }
  end
end

#urlenc(opts = {}) ⇒ Object

Encode to precent-hexify url encoded string

Parameters:

  • opts (Hash) (defaults to: {}) —

    Optional parameters

Options Hash (opts):

  • :plus (optional, Boolean) —

    Treat + as a literal '+', instead of a space



24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/rbkb/extends/string.rb', line 24

def urlenc(opts = {})
  s = self
  plus = opts[:plus]
  unless (opts[:rx] ||= /[^A-Za-z0-9_.~-]/).is_a? Regexp
    raise 'rx must be a regular expression for a character class'
  end

  hx = Rbkb::HEXCHARS

  s.force_to_binary.gsub(opts[:rx]) do |c|
    c = c.ord
    (plus and c == 32) ? '+' : '%' + (hx[(c >> 4)] + hx[(c & 0xf)])
  end
end

#xor(k) ⇒ Object

xor against a key. key will be repeated or truncated to self.size.



200
201
202
203
204
205
206
207
 
# File 'lib/rbkb/extends/string.rb', line 200

def xor(k)
  i = 0
  bytes.map do |b|
    x = k.getbyte(i) || k.getbyte(i = 0)
    i += 1
    (b ^ x).chr
  end.join
end