Module: TextDataTools::Column

Defined in:
lib/text-data-tools.rb

Overview

Tools for extracting data from text files where the data appears in columns with or without headers for each column.

Defined Under Namespace

Classes: DataFile, NotFoundError

Class Method Summary collapse

Class Method Details

.column_index_from_headers(line, column_header, header_match) ⇒ Object

Raises:

  • (ArgumentError) 


151
152
153
154
155
156
157
158
159
 
# File 'lib/text-data-tools.rb', line 151

def self.column_index_from_headers(line, column_header, header_match)
	headers = line.scan(header_match)
	#p headers
	index_array = headers.map{|head| head =~ (column_header.kind_of?(Regexp) ? column_header : Regexp.new(Regexp.escape(column_header)))}
	#p index_array
	raise ArgumentError.new("column_header: #{column_header.inspect} does not match any columns in #{headers.inspect}") if index_array.compact.size == 0
	raise ArgumentError.new("column_header: #{column_header.inspect} matches more than 1 column in #{headers.inspect}") if index_array.compact.size > 1
	column_header = index_array.index(index_array.compact[0])
end

.get_1d_array(filename, has_header_line, column_header, match = /\S+/, header_match = /\S+/, skip_blank = true) ⇒ Object

Return a one-dimensional array containing data from the file filename,

which may or may not have a line of column headers,
in the column column_header, where column_header maybe either a string
or a regex which matches the title of the column,  or an integer
giving the zero-based  column number.

Match is a regexp that matches data items, and header_match is a regexp that
matches items in the headers.

All data is returned as strings

Raises:

  • (ArgumentError) 


34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
 
# File 'lib/text-data-tools.rb', line 34

def self.get_1d_array(filename, has_header_line, column_header, match=/\S+/, header_match=/\S+/, skip_blank=true)
	raise ArgumentError.new("column_header header should be a string, regex or integer") unless [String, Regexp, Integer].find{|cls| column_header.kind_of? cls}
	array = []
	File.open(filename) do |file|
		headers = file.gets if has_header_line
		if [String, Regexp].find{|cls| column_header.kind_of? cls}
			raise ("Header search given but has_header_line = false") if not has_header_line
			column_header = column_index_from_headers(headers, column_header, header_match)
		end
		while line = file.gets
       next if line == "\n" and skip_blank
			values = line.scan(match)
		 	array.push values[column_header]	
			#puts line
		end
	end
	array
end

.get_1d_array_float(*args) ⇒ Object

Calls get_1d_array and converts all data elements to floats



55
56
57
 
# File 'lib/text-data-tools.rb', line 55

def self.get_1d_array_float(*args)
	get_1d_array(*args).map{|v| v.to_f}
end

.get_1d_array_integer(*args) ⇒ Object 



58
59
60
 
# File 'lib/text-data-tools.rb', line 58

def self.get_1d_array_integer(*args)
	get_1d_array(*args).map{|v| v.to_i}
end

.get_2d_array(filename, has_header_line, column_header, index_header = nil, match = /\S+/, header_match = /\S+/) ⇒ Object

Return a two-dimensional array containing data from the file filename,

which may or may not have a line of column headers,
in the column column_header, where column_header maybe either a string
or a regex which matches the title of the column,  or an integer
giving the zero-based  column number.

It is assumed that two-dimensional array is in one column. 
If index_header is nil, data is assumed to be separated by blank lines.
E.g.
		1.2
		4.2
		7.2

		8.2
		4.2
		2.2
If index_header is an integer or string or regexp, it selects a column
in the manner of column_header, and the data is divided by values of this
column.
E.g. 
		1  5.5
		1  3.2
		1  2.6
		2  3.2

2 2.2 2 6.3

Match is a regexp that matches data items, and header_match is a regexp that
matches items in the headers.

All data is returned as strings

Raises:

  • (ArgumentError) 


93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
 
# File 'lib/text-data-tools.rb', line 93

def self.get_2d_array(filename, has_header_line, column_header, index_header=nil, match=/\S+/, header_match=/\S+/)
	raise ArgumentError.new("column_header header should be a string, regex or integer") unless [String, Regexp, Integer].find{|cls| column_header.kind_of? cls}
	raise ArgumentError.new("index_header should be a string, regex, integer or nil") unless [String, Regexp, Integer, NilClass].find{|cls| column_header.kind_of? cls}
	array = []
	File.open(filename) do |file|
		headers = file.gets if has_header_line
		if [String, Regexp].find{|cls| column_header.kind_of? cls}
			raise ("Header search given but has_header_line = false") if not has_header_line
			column_header = column_index_from_headers(headers, column_header, header_match)
		end
		if [String, Regexp].find{|cls| index_header.kind_of? cls}
			raise ("Header search given but has_header_line = false") if not has_header_line
			index_header = column_index_from_headers(headers, index_header, header_match)
		end
		index_value = false
		index = 0
		while line = file.gets
			if index_header.nil?
				if line =~ /^\s*$/
					if array.size == 0 # ignore empty lines at top
						next
					else
						(array.push []; index+=1;next) 
					end
				end
				array.push [] if array.size = 0
			else
				next if line =~ /^\s*$/
			end
			values = line.scan(match)
			if not index_header.nil?
				if array.size ==0
					array.push []
					index_value = values[index_header]
				elsif index_value != values[index_header]
					array.push []
					index+=1
					index_value = values[index_header]
				end
			end
		 	array[index].push values[column_header]	
			#puts line
		end
	end
	array
end

.get_2d_array_float(*args) ⇒ Object

Calls get_2d_array and converts all data elements to floats



141
142
143
 
# File 'lib/text-data-tools.rb', line 141

def self.get_2d_array_float(*args)
	get_2d_array(*args).map{|a| a.map{|v| v.to_f}}
end

.get_2d_array_integer(*args) ⇒ Object 



144
145
146
 
# File 'lib/text-data-tools.rb', line 144

def self.get_2d_array_integer(*args)
	get_2d_array(*args).map{|a| a.map{|v| v.to_i}}
end