Class: OdpsDatahub::Serializer

Inherits:
Object
  • Object
show all
Defined in:
lib/fluent/plugin/serialize/serializer.rb

Instance Method Summary collapse

Instance Method Details

#encodeBool(value) ⇒ Object 



32
33
34
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 32

def encodeBool(value)
  [value ? 1 : 0].pack('C')
end

#encodeDataTime(value) ⇒ Object 



59
60
61
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 59

def encodeDataTime(value)
  self.encodeSInt64(value)
end

#encodeDouble(value) ⇒ Object 



36
37
38
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 36

def encodeDouble(value)
  [value].pack('E')
end

#encodeFixed32(value) ⇒ Object 



76
77
78
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 76

def encodeFixed32(value)
  [value].pack('V')
end

#encodeFixed64(value) ⇒ Object 



71
72
73
74
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 71

def encodeFixed64(value)
  # we don't use 'Q' for pack/unpack. 'Q' is machine-dependent.
  [value & 0xffff_ffff, value >> 32].pack('VV')
end

#encodeFixedString(value) ⇒ Object 



80
81
82
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 80

def encodeFixedString(value)
  [value].pack('V')
end

#encodeSInt64(value) ⇒ Object 



40
41
42
43
44
45
46
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 40

def encodeSInt64(value)
  if value >= 0
    ::Protobuf::Field::VarintField.encode(value << 1)
  else
    ::Protobuf::Field::VarintField.encode(~(value << 1))
  end
end

#encodeString(value) ⇒ Object 



63
64
65
66
67
68
69
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 63

def encodeString(value)
  value_to_encode = value.dup
  value_to_encode.encode!(::Protobuf::Field::StringField::ENCODING, :invalid => :replace, :undef => :replace, :replace => "")
  value_to_encode.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
  string_bytes = ::Protobuf::Field::VarintField.encode(value_to_encode.size)
  string_bytes << value_to_encode
end

#encodeUInt32(value) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 48

def encodeUInt32(value)
  return [value].pack('C') if value < 128
  bytes = []
  until value == 0
    bytes << (0x80 | (value & 0x7f))
    value >>= 7
  end
  bytes[-1] &= 0x7f
  bytes.pack('C*')
end

#serialize(upStream, recordList) ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 89

def serialize(upStream, recordList)
  crc32cPack = StringIO.new
  if recordList.is_a?Array
    recordList.each { |record|
      crc32cRecord = StringIO.new
      schema = OdpsTableSchema.new
      schema = record.getTableSchema
      schema.mCols.each { | col |
        cellValue = record.getValue(col.mIdx)
        if cellValue == nil
          next
        end
        crc32cRecord.write(encodeFixed32(col.mIdx + 1))
        case col.mType
          when $ODPS_BIGINT
            crc32cRecord.write(encodeFixed64(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
            upStream.write(encodeSInt64(cellValue))
          when $ODPS_DOUBLE
            crc32cRecord.write(encodeDouble(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::FIXED64, upStream)
            upStream.write(encodeDouble(cellValue))
          when $ODPS_BOOLEAN
            crc32cRecord.write(encodeBool(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
            upStream.write(encodeBool(cellValue))
          when $ODPS_DATETIME
            crc32cRecord.write(encodeFixed64(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
            upStream.write(encodeDataTime(cellValue))
          when $ODPS_STRING
            crc32cRecord.write(cellValue)
            writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
            upStream.write(encodeString(cellValue))
          when $ODPS_DECIMAL
            crc32cRecord.write(cellValue)
            writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
            upStream.write(encodeString(cellValue))
          else
            raise OdpsDatahubException.new($INVALID_ARGUMENT, "invalid mType")
        end
      }
      recordCrc = CrcCalculator::calculate(crc32cRecord)
      writeTag($TUNNEL_END_RECORD, ::Protobuf::WireType::VARINT, upStream)
      upStream.write(encodeUInt32(recordCrc))
      crc32cPack.write(encodeFixed32(recordCrc))
    }
    writeTag($TUNNEL_META_COUNT, ::Protobuf::WireType::VARINT, upStream)
    upStream.write(encodeSInt64(recordList.size))
    writeTag($TUNNEL_META_CHECKSUM, ::Protobuf::WireType::VARINT, upStream)
    upStream.write(encodeUInt32(CrcCalculator::calculate(crc32cPack)))
  else
    raise OdpsDatahubException.new($INVALID_ARGUMENT, "param must be a array")
  end
end

#writeTag(idx, type, stream) ⇒ Object 



84
85
86
87
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 84

def writeTag(idx, type, stream)
  key = (idx << 3) | type
  stream << ::Protobuf::Field::VarintField.encode(key)
end