Class: Kishu::Report

Inherits:
Object
  • Object
show all
Includes:
Base, Utils
Defined in:
lib/kishu/report.rb

Constant Summary

Constants included from Base

Base::ES_HOST

Instance Attribute Summary collapse

Instance Method Summary collapse

Methods included from Utils

#checksum, #clean_tmp, #encoded, #encoded_file, #format_instance, #generate_header_footer, #get_authors, #get_metadata, #merged_file

Constructor Details

#initialize(options = {}) ⇒ Report

Returns a new instance of Report.



18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/kishu/report.rb', line 18

def initialize options={}
  set_period 
  @es_client = Client.new()
  @logger = Logger.new(STDOUT)
  @report_id = options[:report_id] ? options[:report_id] : ""
  @total = 0
  @aggs_size = options[:aggs_size] 
  @after = options[:after_key] ||=""
  @enrich = options[:enrich] 
  @schema = options[:schema] 
  @encoding = options[:encoding] 
  @created_by = options[:created_by] 
  @report_size = options[:report_size] 
  if @schema == "resolution" 
    @enrich = false
    @encoding = "gzip"
    # @report_size = 20000
  end
end

Instance Attribute Details

#period ⇒ Object (readonly)

Returns the value of attribute period.



16
17
18
 
# File 'lib/kishu/report.rb', line 16

def period
  @period
end

#total ⇒ Object (readonly)

Returns the value of attribute total.



16
17
18
 
# File 'lib/kishu/report.rb', line 16

def total
  @total
end

#uid ⇒ Object (readonly)

Returns the value of attribute uid.



16
17
18
 
# File 'lib/kishu/report.rb', line 16

def uid
  @uid
end

Instance Method Details

#compress(report) ⇒ Object 



89
90
91
92
93
94
95
 
# File 'lib/kishu/report.rb', line 89

def compress report
  gzip = Zlib::GzipWriter.new(StringIO.new)
  string = report.to_json
  gzip << string
  body = gzip.close.string
  body
end

#generate ⇒ Object 



77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/kishu/report.rb', line 77

def generate
  @datasets = []
  loop do
    response = get_events({after_key: @after ||=""})
    @datasets = @datasets.concat response[:data]
    @after = response[:after]
    @total += @datasets.size
    break if @total > 40000
    break if @after.nil?
  end
end

#generate_chunk_report ⇒ Object 



98
99
100
101
102
103
104
105
106
107
 
# File 'lib/kishu/report.rb', line 98

def generate_chunk_report
  # puts get_template
  # LagottoJob.perform_async(get_template(@datasets))
  file = merged_file #+ 'after_key_' + @after
  File.open(file,"w") do |f|
    f.write(JSON.pretty_generate get_template)
  end
  send_report get_template
  @datasets = []
end

#generate_dataset_array ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/kishu/report.rb', line 65

def generate_dataset_array
  @datasets = []
  loop do
    response = get_events({after_key: @after ||=""})
    @datasets = @datasets.concat response[:data]
    @after = response[:after]
    @total += @datasets.size
    generate_chunk_report if @datasets.size > @report_size
    break if @after.nil?
  end
end

#get_events(options = {}) ⇒ Object 



47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/kishu/report.rb', line 47

def get_events options={}
  logger = Logger.new(STDOUT)
  es_client = Client.new()
  response = es_client.get({aggs_size: @aggs_size || 400, after_key: options[:after_key] ||=""})
  aggs = response.dig("aggregations","doi","buckets")
  x = aggs.map do |agg|
    case @schema
      when "resolution" then ResolutionEvent.new(agg,{period: @period, report_id: @report_id}).wrap_event
      when "usage"      then UsageEvent.new(agg,{period: @period, report_id: @report_id}).wrap_event
    end
  end
  after = response.dig("aggregations","doi").fetch("after_key",{"doi"=>nil}).dig("doi")
  logger.info "After_key for pagination #{after}"
  y = {data: x, after: after}
  y
end

#get_header ⇒ Object 



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
 
# File 'lib/kishu/report.rb', line 165

def get_header 
  report_type = case @schema
    when "resolution" then {release:"drl", title:"resolution report"}
    when "usage"      then {release:"rd1", title:"usage report"}
  end 
  {
    "report-name": report_type.dig(:title),
    "report-id": "dsr",
    release: report_type.dig(:release),
    created: Date.today.strftime("%Y-%m-%d"),
    "created-by": @created_by,
    "reporting-period": @period,
    "report-filters": [],
    "report-attributes": [],
    exceptions: [{code: 69,severity: "warning", message: "Report is compressed using gzip","help-url": "https://github.com/datacite/sashimi",data: "usage data needs to be uncompressed"}]
  }
end

#get_template ⇒ Object 



158
159
160
161
162
163
 
# File 'lib/kishu/report.rb', line 158

def get_template 
  {
  "report-header": get_header,
  "report-datasets": @datasets
  }
end

#report_period(options = {}) ⇒ Object 



38
39
40
41
42
43
44
 
# File 'lib/kishu/report.rb', line 38

def report_period options={}
  es_client = Client.new()

  logdate = es_client.get_logdate({aggs_size: 1})
  puts logdate
  Date.parse(logdate)
end

#send_report(report, options = {}) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
 
# File 'lib/kishu/report.rb', line 118

def send_report report, options={}
  uri = HUB_URL+'/reports'   
  puts uri

  case @encoding 
    when "gzip" then
      headers = {
        content_type: "application/gzip",
        content_encoding: 'gzip',
        accept: 'gzip'
      }
      body = compress(report)
    when "json" then
      headers = {
        content_type: "application/json",
        accept: 'application/json'
      }
      body = report
  end

  n = 0
  loop do
    request = Maremma.post(uri, data: body,
      bearer: ENV['HUB_TOKEN'],
      headers: headers,
      timeout: 100)

    puts body
    puts headers

    @uid = request.body.dig("data","report","id")
    @logger.info "#{LOGS_TAG} Hub response #{request.status} for Report finishing in #{@after}"
    @logger.info "#{LOGS_TAG} Hub response #{@uid} for Report finishing in #{@after}"
    n += 1
    break if request.status == 201
    fail "#{LOGS_TAG} Too many attempts were tried to push this report" if n > 1
    sleep 1
  end
end

#set_period ⇒ Object 



110
111
112
113
114
115
116
 
# File 'lib/kishu/report.rb', line 110

def set_period 
  rp = report_period
  @period = { 
    "begin-date": Date.civil(rp.year, rp.mon, 1).strftime("%Y-%m-%d"),
    "end-date": Date.civil(rp.year, rp.mon, -1).strftime("%Y-%m-%d"),
  }
end