12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
# File 'lib/man_merger.rb', line 12
def merge_files
subject_list = load_subject_list
subject_list.each do |subject_code, file_list|
merged_file = CSV.open("/usr/local/htdocs/access/lib/data/etl/klerman_merge_man_files/merged_files/#{subject_code}_merged.csv", "wb")
merged_file << %w(SUBJECT_CODE LABTIME SLEEP_STAGE SLEEP_PERIOD SEM_FLAG)
MY_LOG.info "---- #{subject_code}"
previous_first_labtime = nil
previous_last_labtime = nil
subject_year = get_subject_year(file_list)
file_list.each do |file_hash|
matched_files = Dir.glob("#{T_DRIVE_DIRS[0]}#{subject_code}/PSG/SCORED/**/#{file_hash[:pattern]}.man", File::FNM_CASEFOLD)
matched_files = Dir.glob("#{T_DRIVE_DIRS[1]}#{subject_code}/Sleep/#{file_hash[:pattern]}.man", File::FNM_CASEFOLD) if matched_files.length != 1
if matched_files.length != 1
raise StandardError, "None or more than one matched file. #{file_hash[:pattern]} #{matched_files} #{matched_files.length} #{subject_code}"
else
man_file_path = matched_files[0]
end
man_file = File.open(man_file_path)
LOADER_LOGGER.info "--- Loading #{man_file_path}"
file_info = {}
matched_date = /_(\d\d)(\d\d)(\d\d)_/.match(man_file_path)
file_info[:fn_date] = (matched_date ? Time.zone.local((matched_date[3].to_i > 30 ? matched_date[3].to_i + 1900 : matched_date[3].to_i + 2000), matched_date[1].to_i, matched_date[2].to_i) : nil)
lines = man_file.readlines("\r")
lines.pop if lines.last.blank?
matched_time = /(\d\d):(\d\d):(\d\d):(\d\d\d)/.match(lines.first)
file_info[:first_time] = {hour: matched_time[1].to_i, min: matched_time[2].to_i, sec: matched_time[3].to_i}
matched_time = /(\d\d):(\d\d):(\d\d):(\d\d\d)/.match(lines.last)
file_info[:last_time] = {hour: matched_time[1].to_i, min: matched_time[2].to_i, sec: matched_time[3].to_i}
if file_hash[:start_time] != file_info[:first_time]
MY_LOG.error "---- FIRST TIME MISMATCH ---\n#{man_file_path}\n#{file_hash[:start_time]} #{file_info[:first_time]}\n\n"
end
if file_hash[:last_line_time] != file_info[:last_time]
MY_LOG.error "---- LAST TIME MISMATCH ----\n#{man_file_path}\n#{file_hash[:last_line_time]} #{file_info[:last_time]}\n\n"
end
if file_hash[:last_line_number] != lines.length
MY_LOG.error "---- LINE COUNT MISMATCH ----\n#{man_file_path}\n#{file_hash[:last_line_number]} #{lines.length}\n\n"
end
file_hash[:start_labtime] = Labtime.from_decimal(file_hash[:start_labtime], subject_year)
file_hash[:last_line_labtime] = Labtime.from_decimal(file_hash[:last_line_labtime], subject_year)
start_realtime = file_hash[:start_labtime].to_time
last_line_realtime = file_hash[:last_line_labtime].to_time
first_realtime = file_hash[:start_labtime].time_zone.local(start_realtime.year, start_realtime.month, start_realtime.day, file_info[:first_time][:hour], file_info[:first_time][:min], file_info[:first_time][:sec])
last_realtime = file_hash[:last_line_labtime].time_zone.local(last_line_realtime.year, last_line_realtime.month, last_line_realtime.day, file_info[:last_time][:hour], file_info[:last_time][:min], file_info[:last_time][:sec])
file_info[:first_labtime] = Labtime.parse(first_realtime)
file_info[:last_labtime] = Labtime.parse(last_realtime)
predicted_last_labtime = Labtime.parse(file_info[:first_labtime].to_time + ((lines.length - 1) * 30).seconds)
sep = false
if (file_hash[:start_labtime].time_in_seconds - file_info[:first_labtime].time_in_seconds).abs > 2
MY_LOG.error "---- FIRST LABTIME MISMATCH ----\n#{man_file_path}\n#{file_hash[:start_labtime].time_in_seconds - file_info[:first_labtime].time_in_seconds} | #{file_hash[:start_labtime].to_time}\n#{file_hash[:start_labtime]} | #{file_info[:first_labtime]}\n"
sep = true
end
if last_line_realtime.dst? == start_realtime.dst?
if (file_hash[:last_line_labtime].time_in_seconds - file_info[:last_labtime].time_in_seconds).abs > 2
MY_LOG.error "---- LAST LABTIME MISMATCH ----\n#{man_file_path}\n#{file_hash[:last_line_labtime].time_in_seconds - file_info[:last_labtime].time_in_seconds} | #{file_hash[:last_line_labtime].to_time}\n#{file_hash[:last_line_labtime]} | #{file_info[:last_labtime]}\n"
sep = true
end
if (file_info[:last_labtime].time_in_seconds - predicted_last_labtime.time_in_seconds).abs > 0
MY_LOG.error "---- PRED LABTIME MISMATCH ----\n#{man_file_path}\n#{(file_info[:last_labtime].time_in_seconds - predicted_last_labtime.time_in_seconds)} | #{predicted_last_labtime.to_time}\nl: #{file_info[:last_labtime]} | #{predicted_last_labtime}\n"
sep = true
end
end
if (file_hash[:last_line_labtime].time_in_seconds - predicted_last_labtime.time_in_seconds).abs > 2
MY_LOG.error "---- !PRED LABTIME MISMATCH ----\n#{man_file_path}\n#{(file_hash[:last_line_labtime].time_in_seconds - predicted_last_labtime.time_in_seconds)} | #{predicted_last_labtime.to_time}\nl: #{file_info[:last_line_labtime]} | #{predicted_last_labtime}\n"
sep = true
end
unless previous_first_labtime.nil? or previous_last_labtime.nil?
MY_LOG.error "Start time is before previous end labtime for #{man_file_path}" if file_info[:first_labtime] < previous_last_labtime
end
raise StandardError, "AHHHHH" if file_info[:first_labtime].sec != first_realtime.sec
raise StandardError, "AHHHHH" if file_info[:last_labtime].sec != last_realtime.sec
MY_LOG.info "-----------------------------------\n\n" if sep
last_labtime = nil
ibob_flag = 0
lines.each_with_index do |line, line_number|
=begin
sleep man file:
0 undef/unscored
1 stage 1
2 stage 2
3 stage 3
4 stage 4
5 wake
6 REM
7 MVT
8 LOff and LOn
wake man file:
0 undef/un
cored
1 stage 1
2 stage 2
3 stage 3
4 stage 4
5 wake
6 REM
7 MVT
8 SEM
=end
line_labtime = file_info[:first_labtime].add_seconds(EPOCH_LENGTH * line_number)
line_code = /(\d)\s\d\d:\d\d:\d\d:\d\d\d/.match(line)[1].to_i
if file_hash[:type] == :sleep and line_code == 8
if ibob_flag == 0
sleep_period = 1
ibob_flag = 1
else
sleep_period = 2
ibob_flag = 0
end
else
sleep_period = nil
end
if line_code >= 1 and line_code <= 4
line_event = line_code
elsif line_code == 0
line_event = 7
elsif line_code == 5 or line_code == 8
line_event = 9
elsif line_code == 6
line_event = 5
elsif line_code == 7
line_event = 6
else
raise StandardError, "Cannot map the following event: #{line_code}"
end
if file_hash[:type] == :wake and line_code == 8
sem_event = 1
else
sem_event = 0
end
last_labtime = line_labtime
output_line = [subject_code.upcase, line_labtime.to_decimal, line_event, sleep_period, sem_event]
merged_file << output_line
end
previous_first_labtime = file_info[:first_labtime]
previous_last_labtime = last_labtime
end
merged_file.close
MY_LOG.info "---- end #{subject_code}\n\n"
end
end
|