Class: Fluent::Plugin::S3Output::ArrowCompressor

Inherits:
Compressor
  • Object
show all
Defined in:
lib/fluent/plugin/s3_compressor_arrow.rb

Constant Summary collapse

INVALID＿COMBINATIONS = 
{
  :arrow => [:snappy],
  :feather => [:gzip, :snappy],
}

Instance Method Summary collapse

Instance Method Details

#compress(chunk, tmp) ⇒ Object 



53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/fluent/plugin/s3_compressor_arrow.rb', line 53

def compress(chunk, tmp)
  buffer = Arrow::Buffer.new(chunk.read)
  stream = Arrow::BufferInputStream.new(buffer)
  table = Arrow::JSONReader.new(stream, @options)

  table.read.save(tmp,
    format: @arrow.format,
    chunk_size: @arrow.chunk_size,
    compression: @arrow.compression,
  )
end

#configure(conf) ⇒ Object 



33
34
35
36
37
38
39
40
41
42
43
 
# File 'lib/fluent/plugin/s3_compressor_arrow.rb', line 33

def configure(conf)
  super

  if INVALID

#content_type ⇒ Object 



49
50
51
 
# File 'lib/fluent/plugin/s3_compressor_arrow.rb', line 49

def content_type
  'application/x-apache-arrow-file'.freeze
end

#ext ⇒ Object 



45
46
47
 
# File 'lib/fluent/plugin/s3_compressor_arrow.rb', line 45

def ext
  @arrow.format.freeze
end