Class: TreasureData::API

Inherits:
Object
  • Object
show all
Includes:
AccessControl, Account, BulkImport, BulkLoad, Database, Export, Import, Job, PartialDelete, Result, Schedule, ServerStatus, Table, User
Defined in:
lib/td/client/api.rb,
lib/td/client/api/job.rb,
lib/td/client/api/user.rb,
lib/td/client/api/table.rb,
lib/td/client/api/export.rb,
lib/td/client/api/import.rb,
lib/td/client/api/result.rb,
lib/td/client/api/account.rb,
lib/td/client/api/database.rb,
lib/td/client/api/schedule.rb,
lib/td/client/api/bulk_load.rb,
lib/td/client/api/bulk_import.rb,
lib/td/client/api/server_status.rb,
lib/td/client/api/access_control.rb,
lib/td/client/api/partial_delete.rb

Defined Under Namespace

Modules: AccessControl, Account, BulkImport, BulkLoad, CountReadBodyTotalSize, Database, DeflateReadBodyMixin, DirectReadBodyMixin, Export, Import, Job, PartialDelete, Result, Schedule, ServerStatus, Table, User Classes: IncompleteError

Constant Summary collapse

DEFAULT_ENDPOINT = 
'api.treasure-data.com'
DEFAULT_IMPORT_ENDPOINT = 
'api-import.treasure-data.com'
NEW_DEFAULT_ENDPOINT = 
'api.treasuredata.com'
NEW_DEFAULT_IMPORT_ENDPOINT = 
'api-import.treasuredata.com'

Constants included from BulkLoad

BulkLoad::JOB, BulkLoad::LIST, BulkLoad::SESSION

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from User

#add_apikey, #add_user, #authenticate, #change_email, #change_my_password, #change_password, #list_apikeys, #list_users, #remove_apikey, #remove_user

Methods included from Table

#create_log_table, #delete_table, #list_tables, #swap_table, #tail, #update_expire, #update_schema

Methods included from ServerStatus

#server_status

Methods included from Schedule

#create_schedule, #delete_schedule, #history, #list_schedules, #run_schedule, #update_schedule

Methods included from Result

#create_result, #delete_result, #list_result

Methods included from PartialDelete

#partial_delete

Methods included from Job

#hive_query, #job_result, #job_result_each, #job_result_each_with_compr_size, #job_result_format, #job_result_raw, #job_status, #kill, #list_jobs, #pig_query, #query, #show_job

Methods included from Import

#import

Methods included from Export

#export

Methods included from Database

#create_database, #delete_database, #list_databases

Methods included from BulkLoad

#bulk_load_create, #bulk_load_delete, #bulk_load_guess, #bulk_load_history, #bulk_load_issue, #bulk_load_list, #bulk_load_preview, #bulk_load_run, #bulk_load_show, #bulk_load_update

Methods included from BulkImport

#bulk_import_delete_part, #bulk_import_error_records, #bulk_import_upload_part, #commit_bulk_import, #create_bulk_import, #delete_bulk_import, #freeze_bulk_import, #list_bulk_import_parts, #list_bulk_imports, #perform_bulk_import, #show_bulk_import, #unfreeze_bulk_import

Methods included from Account

#account_core_utilization, #show_account

Methods included from AccessControl

#grant_access_control, #list_access_controls, #revoke_access_control, #test_access_control

Constructor Details

#initialize(apikey, opts = {}) ⇒ API

Returns a new instance of API.

Parameters:

  • apikey (String)
  • opts (Hash) (defaults to: {}) 


49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/td/client/api.rb', line 49

def initialize(apikey, opts={})
  require 'json'
  require 'time'
  require 'uri'
  require 'net/http'
  require 'net/https'
  require 'time'
  #require 'faraday' # faraday doesn't support streaming upload with httpclient yet so now disabled
  require 'httpclient'
  require 'zlib'
  require 'stringio'
  require 'cgi'
  require 'msgpack'

  @apikey = apikey
  @user_agent = "TD-Client-Ruby: #{TreasureData::Client::VERSION}"
  @user_agent = "#{opts[:user_agent]}; " + @user_agent if opts.has_key?(:user_agent)

  endpoint = opts[:endpoint] || ENV['TD_API_SERVER'] || DEFAULT_ENDPOINT
  uri = URI.parse(endpoint)

  @connect_timeout = opts[:connect_timeout] || 60
  @read_timeout = opts[:read_timeout] || 600
  @send_timeout = opts[:send_timeout] || 600
  @retry_post_requests = opts[:retry_post_requests] || false
  @retry_delay = opts[:retry_delay] || 5
  @max_cumul_retry_delay = opts[:max_cumul_retry_delay] || 600

  case uri.scheme
  when 'http', 'https'
    @host = uri.host
    @port = uri.port
    # the opts[:ssl] option is ignored here, it's value
    #   overridden by the scheme of the endpoint URI
    @ssl = (uri.scheme == 'https')
    @base_path = uri.path.to_s

  else
    if uri.port
      # invalid URI
      raise "Invalid endpoint: #{endpoint}"
    end

    # generic URI
    @host, @port = endpoint.split(':', 2)
    @port = @port.to_i
    if opts[:ssl]
      @port = 443 if @port == 0
      @ssl = true
    else
      @port = 80 if @port == 0
      @ssl = false
    end
    @base_path = ''
  end

  @http_proxy = opts[:http_proxy] || ENV['HTTP_PROXY']
  if @http_proxy
    http_proxy = if @http_proxy =~ /\Ahttp:\/\/(.*)\z/
                   $~[1]
                 else
                   @http_proxy
                 end
    proxy_host, proxy_port = http_proxy.split(':', 2)
    proxy_port = (proxy_port ? proxy_port.to_i : 80)
    @http_class = Net::HTTP::Proxy(proxy_host, proxy_port)
  else
    @http_class = Net::HTTP
  end

  @headers = opts[:headers] || {}
  @api = api_client("#{@ssl ? 'https' : 'http'}://#{@host}:#{@port}")
end

Instance Attribute Details

#apikey ⇒ Object (readonly)

Returns the value of attribute apikey.



126
127
128
 
# File 'lib/td/client/api.rb', line 126

def apikey
  @apikey
end

Class Method Details

.create_empty_gz_data ⇒ String

for fluent-plugin-td / td command to check table existence with import onlt user

Returns:

  • (String) 


231
232
233
234
235
 
# File 'lib/td/client/api.rb', line 231

def self.create_empty_gz_data
  io = StringIO.new
  Zlib::GzipWriter.new(io).close
  io.string
end

.normalize_database_name(name) ⇒ Object

Parameters:

  • name (String) 


189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/td/client/api.rb', line 189

def self.normalize_database_name(name)
  name = name.to_s
  if name.empty?
    raise "Empty name is not allowed"
  end
  if name.length < 3
    name += "_" * (3 - name.length)
  end
  if 255 < name.length
    name = name[0, 253] + "__"
  end
  name = name.downcase
  name = name.gsub(/[^a-z0-9_]/, '_')
  name
end

.normalize_table_name(name) ⇒ Object

Parameters:

  • name (String) 


206
207
208
 
# File 'lib/td/client/api.rb', line 206

def self.normalize_table_name(name)
  normalize_database_name(name)
end

.normalize_type_name(name) ⇒ Object

TODO support array types

Parameters:

  • name (String) 


212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/td/client/api.rb', line 212

def self.normalize_type_name(name)
  case name
  when /int/i, /integer/i
    "int"
  when /long/i, /bigint/i
    "long"
  when /string/i
    "string"
  when /float/i
    "float"
  when /double/i
    "double"
  else
    raise "Type name must eather of int, long, string float or double"
  end
end

.normalized_msgpack(record, out = nil) ⇒ Object

Parameters:

  • record (Hash)
  • out (IO) (defaults to: nil) 


130
131
132
133
134
135
136
137
138
 
# File 'lib/td/client/api.rb', line 130

def self.normalized_msgpack(record, out = nil)
  record.keys.each { |k|
    v = record[k]
    if v.kind_of?(Bignum)
      record[k] = v.to_s
    end
  }
  record.to_msgpack(out)
end

.validate_column_name(name) ⇒ Object

Parameters:

  • name (String) 


184
185
186
 
# File 'lib/td/client/api.rb', line 184

def self.validate_column_name(name)
  validate_name("column", 1, 255, name)
end

.validate_database_name(name) ⇒ Object

Parameters:

  • name (String) 


169
170
171
 
# File 'lib/td/client/api.rb', line 169

def self.validate_database_name(name)
  validate_name("database", 3, 255, name)
end

.validate_name(target, min_len, max_len, name) ⇒ Object

Parameters:

  • target (String)
  • min_len (Fixnum)
  • max_len (Fixnum)
  • name (String) 


144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/td/client/api.rb', line 144

def self.validate_name(target, min_len, max_len, name)
  if !target.instance_of?(String) || target.empty?
    raise ParameterValidationError,
          "A valid target name is required"
  end

  name = name.to_s
  if name.empty?
    raise ParameterValidationError,
          "Empty #{target} name is not allowed"
  end
  if name.length < min_len || name.length > max_len
    raise ParameterValidationError,
          "#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
          (name.length == 1 ? "character" : "characters") + "."
  end
  unless name =~ /^([a-z0-9_]+)$/
    raise ParameterValidationError,
          "#{target.capitalize} name must only consist of lower-case alpha-numeric characters and '_'."
  end

  name
end

.validate_result_set_name(name) ⇒ Object

Parameters:

  • name (String) 


179
180
181
 
# File 'lib/td/client/api.rb', line 179

def self.validate_result_set_name(name)
  validate_name("result set", 3, 255, name)
end

.validate_table_name(name) ⇒ Object

Parameters:

  • name (String) 


174
175
176
 
# File 'lib/td/client/api.rb', line 174

def self.validate_table_name(name)
  validate_name("table", 3, 255, name)
end

Instance Method Details

#ssl_ca_file=(ssl_ca_file) ⇒ Object

Parameters:

  • ssl_ca_file (String) 


238
239
240
 
# File 'lib/td/client/api.rb', line 238

def ssl_ca_file=(ssl_ca_file)
  @ssl_ca_file = ssl_ca_file
end