Module: Segment
- Extended by:
- Entity
- Included in:
- NLP::GdepChunk, NLP::GdepToken, NamedEntity, Token
- Defined in:
- lib/rbbt/segment.rb,
lib/rbbt/segment/encoding.rb,
lib/rbbt/segment/overlaps.rb
Defined Under Namespace
Modules: RangeIndex
Class Method Summary collapse
- .align(text, parts) ⇒ Object
- .ascii(text, replace = nil, &block) ⇒ Object
- .bad_chars(text) ⇒ Object
- .clean_sort(segments) ⇒ Object
- .collisions(main, secondary) ⇒ Object
- .index(*args) ⇒ Object
- .overlaps(sorted_segments) ⇒ Object
- .sort(segments, inline = true) ⇒ Object
- .split(text, segments, skip_segments = false) ⇒ Object
Instance Method Summary collapse
- #eend ⇒ Object (also: #end)
- #includes?(segment) ⇒ Boolean
- #make_relative(segments, &block) ⇒ Object
- #overlaps(segments) ⇒ Object
- #overlaps?(segment) ⇒ Boolean
- #pull(offset) ⇒ Object
- #push(offset) ⇒ Object
- #range ⇒ Object
- #range_in(container = nil) ⇒ Object
- #segment_length ⇒ Object
Class Method Details
.align(text, parts) ⇒ Object
159 160 161 162 163 164 165 166 167 168 169 |
# File 'lib/rbbt/segment.rb', line 159 def self.align(text, parts) pre_offset = 0 docid = text.respond_to?(:docid) ? text.docid : nil parts.each do |part| offset = text.index part next if offset.nil? Segment.setup(part, pre_offset + offset, docid) pre_offset += offset + part.segment_length - 1 text = text[(offset + part.segment_length - 1)..-1] end end |
.ascii(text, replace = nil, &block) ⇒ Object
13 14 15 16 17 |
# File 'lib/rbbt/segment/encoding.rb', line 13 def self.ascii(text, replace = nil, &block) bad = bad_chars(text) replace = "?" if replace.nil? Transformed.with_transform(text, bad, replace, &block) end |
.bad_chars(text) ⇒ Object
3 4 5 6 7 8 9 10 11 |
# File 'lib/rbbt/segment/encoding.rb', line 3 def self.bad_chars(text) segments = [] text.chars.each_with_index do |c,i| if ! c.ascii_only? segments << Segment.setup(c, :offset => i) end end segments end |
.clean_sort(segments) ⇒ Object
106 107 108 109 110 111 112 113 114 |
# File 'lib/rbbt/segment.rb', line 106 def self.clean_sort(segments) sorted = sort(segments).reject{|s| s.offset.nil?} overlaps = overlaps(sorted) overlaps.each do |s| sorted.delete s end sorted end |
.collisions(main, secondary) ⇒ Object
58 59 60 61 62 |
# File 'lib/rbbt/segment/overlaps.rb', line 58 def self.collisions(main, secondary) secondary.select do |ss| main.select{|ms| ms.overlaps? ss }.any? end end |
.index(*args) ⇒ Object
171 172 173 |
# File 'lib/rbbt/segment.rb', line 171 def self.index(*args) Segment::RangeIndex.index(*args) end |
.overlaps(sorted_segments) ⇒ Object
94 95 96 97 98 99 100 101 102 103 104 |
# File 'lib/rbbt/segment.rb', line 94 def self.overlaps(sorted_segments) last = nil overlaped = [] sorted_segments.reverse.each do |segment| overlaped << segment if (not last.nil?) and segment.range.end > last last = segment.range.begin end overlaped end |
.sort(segments, inline = true) ⇒ Object
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
# File 'lib/rbbt/segment.rb', line 73 def self.sort(segments, inline = true) if inline segments.sort do |a,b| case when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?)) 0 when (a.nil? or a.offset.nil?) -1 when (b.nil? or b.offset.nil?) +1 when (not a.range.include? b.offset.to_i and not b.range.include? a.offset.to_i) a.offset.to_i <=> b.offset.to_i else a.segment_length <=> b.segment_length end end else segments.sort_by do |segment| segment.offset.to_i || 0 end.reverse end end |
.split(text, segments, skip_segments = false) ⇒ Object
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
# File 'lib/rbbt/segment.rb', line 116 def self.split(text, segments, skip_segments = false) sorted_segments = clean_sort segments chunks = [] segment_end = 0 text_offset = 0 sorted_segments.each do |segment| return chunks if text.nil? or text.empty? next if segment.offset.nil? offset = segment.offset - text_offset # Consider segment offset. Save pre, or skip if overlap case when offset < 0 # Overlap, skip next when offset > 0 # Save pre chunk = text[0..offset - 1] Segment.setup(chunk, text_offset) chunks << chunk end segment_end = offset + segment.segment_length - 1 if not skip_segments chunk = text[offset..segment_end] Segment.setup(chunk, text_offset + offset) chunks << chunk end text_offset += segment_end + 1 text = text[segment_end + 1..-1] end if not text.nil? and not text.empty? chunk = text.dup Segment.setup(chunk, text_offset) chunks << chunk end chunks end |
Instance Method Details
#eend ⇒ Object Also known as: end
53 54 55 |
# File 'lib/rbbt/segment.rb', line 53 def eend offset.to_i + length - 1 end |
#includes?(segment) ⇒ Boolean
44 45 46 47 |
# File 'lib/rbbt/segment/overlaps.rb', line 44 def includes?(segment) (segment.offset.to_i >= self.offset.to_i) and (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i) end |
#make_relative(segments, &block) ⇒ Object
22 23 24 25 26 27 28 29 30 |
# File 'lib/rbbt/segment/overlaps.rb', line 22 def make_relative(segments, &block) if block_given? segments.each{|s| s.push offset} yield(segments) segments.each{|s| s.pull offset} else segments.each{|s| s.push offset} end end |
#overlaps(segments) ⇒ Object
54 55 56 |
# File 'lib/rbbt/segment/overlaps.rb', line 54 def overlaps(segments) segments.select{|s| self.overlaps?(s) } end |
#overlaps?(segment) ⇒ Boolean
49 50 51 52 |
# File 'lib/rbbt/segment/overlaps.rb', line 49 def overlaps?(segment) segment.offset.to_i >= self.offset.to_i && segment.offset.to_i <= self.eend || self.offset.to_i >= segment.offset.to_i && self.offset.to_i <= segment.eend end |
#pull(offset) ⇒ Object
2 3 4 5 6 7 8 9 10 |
# File 'lib/rbbt/segment/overlaps.rb', line 2 def pull(offset) if self.offset.nil? or offset.nil? self.offset = nil else self.offset += offset end self end |
#push(offset) ⇒ Object
12 13 14 15 16 17 18 19 20 |
# File 'lib/rbbt/segment/overlaps.rb', line 12 def push(offset) if self.offset.nil? or offset.nil? self.offset = nil else self.offset -= offset end self end |
#range ⇒ Object
59 60 61 |
# File 'lib/rbbt/segment.rb', line 59 def range (offset.to_i..eend) end |
#range_in(container = nil) ⇒ Object
32 33 34 35 36 37 38 39 40 41 42 |
# File 'lib/rbbt/segment/overlaps.rb', line 32 def range_in(container = nil) raise "No offset specified" if offset.nil? case when (Segment === container and not container.offset.nil?) ((offset - container.offset)..(self.eend - container.offset)) when Integer === container ((offset - container)..(self.eend - container)) else range end end |
#segment_length ⇒ Object
48 49 50 |
# File 'lib/rbbt/segment.rb', line 48 def segment_length length end |