Module: Segment
- Extended by:
- Annotation
- Included in:
- NLP::GdepChunk, NLP::GdepToken, NamedEntity, Relationship
- Defined in:
- lib/rbbt/ner/segment.rb
Defined Under Namespace
Classes: Index
Class Method Summary collapse
-
.align(text, parts) ⇒ Object
{{{ Align.
- .clean_sort(segments) ⇒ Object
- .index(segments, persist_file = :memory) ⇒ Object
- .load_tsv(tsv) ⇒ Object
- .load_tsv_values(text, values, fields) ⇒ Object
- .overlaps(sorted_segments) ⇒ Object
- .set_tsv_fields(fields, segments) ⇒ Object
-
.sort(segments, inline = true) ⇒ Object
{{{ Sorting.
-
.split(text, segments, skip_segments = false) ⇒ Object
{{{ Splitting.
- .tsv(segments, *fields) ⇒ Object
-
.tsv_values_for_segment(segment, fields) ⇒ Object
{{{ Save and load.
Instance Method Summary collapse
-
#end ⇒ Object
{{{ Ranges.
- #includes?(segment) ⇒ Boolean
- #make_relative(segments, &block) ⇒ Object
- #pull(offset) ⇒ Object
- #push(offset) ⇒ Object
- #range ⇒ Object
- #range_in(container = nil) ⇒ Object
- #segment_length ⇒ Object
Class Method Details
.align(text, parts) ⇒ Object
{{{ Align
168 169 170 171 172 173 174 175 176 177 |
# File 'lib/rbbt/ner/segment.rb', line 168 def self.align(text, parts) pre_offset = 0 parts.each do |part| offset = text.index part next if offset.nil? Segment.setup(part, pre_offset + offset) pre_offset += offset + part.segment_length - 1 text = text[(offset + part.segment_length - 1)..-1] end end |
.clean_sort(segments) ⇒ Object
110 111 112 113 114 115 116 117 118 |
# File 'lib/rbbt/ner/segment.rb', line 110 def self.clean_sort(segments) sorted = sort(segments).reject{|s| s.offset.nil?} overlaps = overlaps(sorted) overlaps.each do |s| sorted.delete s end sorted end |
.index(segments, persist_file = :memory) ⇒ Object
193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
# File 'lib/rbbt/ner/segment.rb', line 193 def self.index(segments, persist_file = :memory) segments = segments.values.flatten if Hash === segments annotation_index = Persist.persist("Segment_index", :fwt, :persist => (! (persist_file.nil? or persist_file == :memory)), :file => persist_file) do value_size = 0 index_data = segments.collect{|segment| next if segment.offset.nil? range = segment.range value_size = [segment.id.length, value_size].max [segment.id, [range.begin, range.end]] }.compact fwt = FixWidthTable.get :memory, value_size, true fwt.add_range index_data fwt end data = {} segments.each do |segment| data[segment.id] = segment end Index.new annotation_index, data end |
.load_tsv(tsv) ⇒ Object
318 319 320 321 322 323 324 325 |
# File 'lib/rbbt/ner/segment.rb', line 318 def self.load_tsv(tsv) fields = tsv.fields tsv.with_unnamed do tsv.collect do |id, values| Annotated.load_tsv_values(id, values, fields) end end end |
.load_tsv_values(text, values, fields) ⇒ Object
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 |
# File 'lib/rbbt/ner/segment.rb', line 242 def self.load_tsv_values(text, values, fields) info = {} literal_pos = fields.index "literal" object = if literal_pos.nil? "" else v = values[literal_pos] v = v.first if Array === v v end fields.each_with_index do |field, i| if field == "JSON" JSON.parse(values[i]).each do |key, value| info[key.to_sym] = value end else info[field.to_sym] = values[i] end end start = info.delete(:Start) if not (start.nil? or ((Array === start or String === start) and start.empty?)) if Array === start start = start.first end start = start.to_i info[:offset] = start eend = info.delete(:End) if Array === eend eend = eend.first end eend = eend.to_i if object.empty? object.replace text[start..eend] end end info[:annotation_types] = [Segment] unless info.include? :annotation_types Annotated.load(object, info) end |
.overlaps(sorted_segments) ⇒ Object
98 99 100 101 102 103 104 105 106 107 108 |
# File 'lib/rbbt/ner/segment.rb', line 98 def self.overlaps(sorted_segments) last = nil overlaped = [] sorted_segments.reverse.each do |segment| overlaped << segment if (not last.nil?) and segment.range.end > last last = segment.range.begin end overlaped end |
.set_tsv_fields(fields, segments) ⇒ Object
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 |
# File 'lib/rbbt/ner/segment.rb', line 288 def self.set_tsv_fields(fields, segments) tsv_fields = [] add_types = ! (fields.delete(:no_types) || fields.delete("no_types") || fields.include?(:JSON) || fields.include?("JSON")) literal = (fields.delete(:literal) || fields.delete("literal")) tsv_fields << "Start" << "End" tsv_fields << :annotation_types if add_types tsv_fields << :literal if literal if fields.any? and not (fields == [:all] or fields == ["all"]) tsv_fields.concat fields else tsv_fields.concat segments.first.annotations if segments.any? end tsv_fields tsv_fields.collect!{|f| f.to_s} tsv_fields.delete "offset" tsv_fields end |
.sort(segments, inline = true) ⇒ Object
{{{ Sorting
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 |
# File 'lib/rbbt/ner/segment.rb', line 77 def self.sort(segments, inline = true) if inline segments.sort do |a,b| case when ((a.nil? and b.nil?) or (a.offset.nil? and b.offset.nil?)) 0 when (a.nil? or a.offset.nil?) -1 when (b.nil? or b.offset.nil?) +1 when (not a.range.include? b.offset and not b.range.include? a.offset) a.offset <=> b.offset else a.segment_length <=> b.segment_length end end else segments.sort_by do |segment| segment.offset || 0 end.reverse end end |
.split(text, segments, skip_segments = false) ⇒ Object
{{{ Splitting
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 |
# File 'lib/rbbt/ner/segment.rb', line 122 def self.split(text, segments, skip_segments = false) sorted_segments = clean_sort segments chunks = [] segment_end = 0 text_offset = 0 sorted_segments.each do |segment| return chunks if text.nil? or text.empty? next if segment.offset.nil? offset = segment.offset - text_offset # Consider segment offset. Save pre, or skip if overlap case when offset < 0 # Overlap, skip next when offset > 0 # Save pre chunk = text[0..offset - 1] Segment.setup(chunk, text_offset) chunks << chunk end segment_end = offset + segment.segment_length - 1 if not skip_segments chunk = text[offset..segment_end] Segment.setup(chunk, text_offset + offset) chunks << chunk end text_offset += segment_end + 1 text = text[segment_end + 1..-1] end if not text.nil? and not text.empty? chunk = text.dup Segment.setup(chunk, text_offset) chunks << chunk end chunks end |
.tsv(segments, *fields) ⇒ Object
307 308 309 310 311 312 313 314 315 316 |
# File 'lib/rbbt/ner/segment.rb', line 307 def self.tsv(segments, *fields) fields = set_tsv_fields fields, segments tsv = TSV.setup({}, :key_field => "ID", :fields => fields, :type => :double) segments.each do |segment| tsv[segment.id] = self.tsv_values_for_segment(segment, fields) end tsv end |
.tsv_values_for_segment(segment, fields) ⇒ Object
{{{ Save and load
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 |
# File 'lib/rbbt/ner/segment.rb', line 220 def self.tsv_values_for_segment(segment, fields) info = segment.info values = [] fields.each do |field| values << case when field == "JSON" info.to_json when field == "literal" segment.gsub(/\n|\t/, ' ') when field == "Start" segment.offset when field == "End" segment.end else info.delete(field.to_sym) end end values end |
Instance Method Details
#end ⇒ Object
{{{ Ranges
18 19 20 21 |
# File 'lib/rbbt/ner/segment.rb', line 18 def end return nil if offset.nil? offset.to_i + segment_length - 1 end |
#includes?(segment) ⇒ Boolean
70 71 72 73 |
# File 'lib/rbbt/ner/segment.rb', line 70 def includes?(segment) (segment.offset.to_i >= self.offset.to_i) and (segment.offset.to_i + segment.segment_length.to_i <= self.offset.to_i + self.segment_length.to_i) end |
#make_relative(segments, &block) ⇒ Object
48 49 50 51 52 53 54 55 56 |
# File 'lib/rbbt/ner/segment.rb', line 48 def make_relative(segments, &block) if block_given? segments.each{|s| s.push offset} yield(segments) segments.each{|s| s.pull offset} else segments.each{|s| s.push offset} end end |
#pull(offset) ⇒ Object
28 29 30 31 32 33 34 35 36 |
# File 'lib/rbbt/ner/segment.rb', line 28 def pull(offset) if self.offset.nil? or offset.nil? self.offset = nil else self.offset += offset end self end |
#push(offset) ⇒ Object
38 39 40 41 42 43 44 45 46 |
# File 'lib/rbbt/ner/segment.rb', line 38 def push(offset) if self.offset.nil? or offset.nil? self.offset = nil else self.offset -= offset end self end |
#range ⇒ Object
23 24 25 26 |
# File 'lib/rbbt/ner/segment.rb', line 23 def range raise "No offset specified" if offset.nil? (offset.to_i..self.end) end |
#range_in(container = nil) ⇒ Object
58 59 60 61 62 63 64 65 66 67 68 |
# File 'lib/rbbt/ner/segment.rb', line 58 def range_in(container = nil) raise "No offset specified" if offset.nil? case when (Segment === container and not container.offset.nil?) ((offset - container.offset)..(self.end - container.offset)) when Integer === container ((offset - container)..(self.end - container)) else range end end |
#segment_length ⇒ Object
8 9 10 11 12 13 14 |
# File 'lib/rbbt/ner/segment.rb', line 8 def segment_length begin super() rescue self.length end end |