17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
# File 'lib/inat-channel/data.rb', line 17
def select_uuid fresh
fresh.reject! { |rec| sent?(rec[:uuid]) }
taxon_uniq = IC::CONFIG[:unique_taxon]
fresh.reject! { |rec| used?(rec.dig :taxon, :id) } if taxon_uniq == :strict
if taxon_uniq == :priority
uniq_fresh = fresh.reject { |rec| used?(rec.dig :taxon, :id) }
unless uniq_fresh.empty?
IC::logger.info "Take a fresh & unique (from #{uniq_fresh.size})"
sample = sample_with_weight uniq_fresh, field: :faves_count
fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
pool[rec[:uuid]] = {
'created_at' => Date.parse(rec[:created_at]),
'faves_count' => rec[:faves_count],
'taxon_id' => rec.dig(:taxon, :id)
}
end
sample[:taxon_id] = sample.dig :taxon, :id
@in_process = sample
return sample[:uuid]
end
end
unless fresh.empty?
IC::logger.info "Take a fresh (from #{fresh.size})"
sample = sample_with_weight fresh, field: :faves_count
fresh.reject { |rec| rec[:uuid] == sample[:uuid] }.each do |rec|
pool[rec[:uuid]] = {
"created_at" => Date.parse(rec[:created_at]),
"faves_count" => rec[:faves_count],
"taxon_id" => rec.dig(:taxon, :id),
}
end
sample[:taxon_id] = sample.dig :taxon, :id
@in_process = sample
return sample[:uuid]
end
pool_records = pool.map { |k, v| v.merge({ 'uuid' => k }) }
if taxon_uniq == :priority
uniq_pool = pool_records.reject { |rec| used?(rec['taxon_id']) }
unless uniq_pool.empty?
IC::logger.info "Take an unique pool record (from #{uniq_pool.size})"
sample = sample_with_weight uniq_pool, field: 'faves_count'
sample.transform_keys!(&:to_sym)
@in_process = sample
return sample[:uuid]
end
end
unless pool_records.empty?
IC::logger.info "Take a pool record (from #{pool.size})"
sample = sample_with_weight pool_records, field: 'faves_count'
sample.transform_keys!(&:to_sym)
@in_process = sample
return sample[:uuid]
end
return nil
end
|