Class: Embulk::Filter::Unique

Inherits:
FilterPlugin
  • Object
show all
Defined in:
lib/embulk/filter/unique.rb

Class Method Summary collapse

Instance Method Summary collapse

Class Method Details

.transaction(config, in_schema) {|task, in_schema| ... } ⇒ Object

Yields:

  • (task, in_schema) 


9
10
11
12
13
14
15
 
# File 'lib/embulk/filter/unique.rb', line 9

def self.transaction(config, in_schema, &control)
  task = {
    "columns" => config.param("columns", :array),
  }

  yield(task, in_schema)
end

Instance Method Details

#add(page) ⇒ Object 



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/embulk/filter/unique.rb', line 25

def add(page)
  page.each do |record|
    part = {}
    @cols.each do |c|
      cs = page.schema.select{|s| s.name == c}
      idx = cs[0].index
      part[c] = record[idx]
    end

    unless @exists.include?(part)
      page_builder.add(record)
      @exists.add(part)
    end
  end
end

#close ⇒ Object 



22
23
 
# File 'lib/embulk/filter/unique.rb', line 22

def close
end

#finish ⇒ Object 



41
42
43
 
# File 'lib/embulk/filter/unique.rb', line 41

def finish
  page_builder.finish
end

#init ⇒ Object 



17
18
19
20
 
# File 'lib/embulk/filter/unique.rb', line 17

def init
  @cols = task["columns"]
  @exists = Set.new
end