Class: OdpsDatahub::Serializer

Inherits:
Object
  • Object
show all
Defined in:
lib/fluent/plugin/serialize/serializer.rb

Instance Method Summary collapse

Instance Method Details

#encodeBool(value) ⇒ Object 



32
33
34
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 32

def encodeBool(value)
  [value ? 1 : 0].pack('C')
end

#encodeDataTime(value) ⇒ Object 



59
60
61
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 59

def encodeDataTime(value)
  self.encodeSInt64(value)
end

#encodeDouble(value) ⇒ Object 



36
37
38
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 36

def encodeDouble(value)
  [value].pack('E')
end

#encodeFixed32(value) ⇒ Object 



75
76
77
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 75

def encodeFixed32(value)
  [value].pack('V')
end

#encodeFixed64(value) ⇒ Object 



70
71
72
73
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 70

def encodeFixed64(value)
  # we don't use 'Q' for pack/unpack. 'Q' is machine-dependent.
  [value & 0xffff_ffff, value >> 32].pack('VV')
end

#encodeFixedString(value) ⇒ Object 



79
80
81
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 79

def encodeFixedString(value)
  [value].pack('V')
end

#encodeSInt64(value) ⇒ Object 



40
41
42
43
44
45
46
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 40

def encodeSInt64(value)
  if value >= 0
    ::Protobuf::Field::VarintField.encode(value << 1)
  else
    ::Protobuf::Field::VarintField.encode(~(value << 1))
  end
end

#encodeString(value) ⇒ Object 



63
64
65
66
67
68
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 63

def encodeString(value)
  value.encode!(::Protobuf::Field::StringField::ENCODING, :invalid => :replace, :undef => :replace, :replace => "")
  value.force_encoding(::Protobuf::Field::BytesField::BYTES_ENCODING)
  string_bytes = ::Protobuf::Field::VarintField.encode(value.size)
  string_bytes << value
end

#encodeUInt32(value) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 48

def encodeUInt32(value)
  return [value].pack('C') if value < 128
  bytes = []
  until value == 0
    bytes << (0x80 | (value & 0x7f))
    value >>= 7
  end
  bytes[-1] &= 0x7f
  bytes.pack('C*')
end

#serialize(upStream, recordList) ⇒ Object 



88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 88

def serialize(upStream, recordList)
  crc32cPack = StringIO.new
  if recordList.is_a?Array
    recordList.each { |record|
      crc32cRecord = StringIO.new
      schema = OdpsTableSchema.new
      schema = record.getTableSchema
      schema.mCols.each { | col |
        cellValue = record.getValue(col.mIdx)
        if cellValue == nil
          next
        end
        crc32cRecord.write(encodeFixed32(col.mIdx + 1))
        case col.mType
          when $ODPS_BIGINT
            crc32cRecord.write(encodeFixed64(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
            upStream.write(encodeSInt64(cellValue))
          when $ODPS_DOUBLE
            crc32cRecord.write(encodeDouble(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::FIXED64, upStream)
            upStream.write(encodeDouble(cellValue))
          when $ODPS_BOOLEAN
            crc32cRecord.write(encodeBool(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
            upStream.write(encodeBool(cellValue))
          when $ODPS_DATETIME
            crc32cRecord.write(encodeFixed64(cellValue))
            writeTag(col.mIdx + 1, ::Protobuf::WireType::VARINT, upStream)
            upStream.write(encodeDataTime(cellValue))
          when $ODPS_STRING
            cellValue = cellValue.encode('utf-8')
            encode_str = encodeString(cellValue)
            crc32cRecord.write(cellValue)
            writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
            upStream.write(encode_str)
          when $ODPS_DECIMAL
            encode_str = encodeString(cellValue)
            crc32cRecord.write(cellValue)
            writeTag(col.mIdx + 1, ::Protobuf::WireType::LENGTH_DELIMITED, upStream)
            upStream.write(encode_str)
          else
            raise OdpsDatahubException.new($INVALID_ARGUMENT, "invalid mType")
        end
      }
      recordCrc = CrcCalculator::calculate(crc32cRecord)
      writeTag($TUNNEL_END_RECORD, ::Protobuf::WireType::VARINT, upStream)
      upStream.write(encodeUInt32(recordCrc))
      crc32cPack.write(encodeFixed32(recordCrc))
    }
    writeTag($TUNNEL_META_COUNT, ::Protobuf::WireType::VARINT, upStream)
    upStream.write(encodeSInt64(recordList.size))
    writeTag($TUNNEL_META_CHECKSUM, ::Protobuf::WireType::VARINT, upStream)
    upStream.write(encodeUInt32(CrcCalculator::calculate(crc32cPack)))
  else
    raise OdpsDatahubException.new($INVALID_ARGUMENT, "param must be a array")
  end
end

#writeTag(idx, type, stream) ⇒ Object 



83
84
85
86
 
# File 'lib/fluent/plugin/serialize/serializer.rb', line 83

def writeTag(idx, type, stream)
  key = (idx << 3) | type
  stream << ::Protobuf::Field::VarintField.encode(key)
end