Class: TreasureData::API

Inherits:
Object
  • Object
show all
Includes:
AccessControl, Account, BulkImport, BulkLoad, Database, Export, Import, Job, PartialDelete, Result, Schedule, ServerStatus, Table, User
Defined in:
lib/td/client/api.rb,
lib/td/client/api/job.rb,
lib/td/client/api/user.rb,
lib/td/client/api/table.rb,
lib/td/client/api/export.rb,
lib/td/client/api/import.rb,
lib/td/client/api/result.rb,
lib/td/client/api/account.rb,
lib/td/client/api/database.rb,
lib/td/client/api/schedule.rb,
lib/td/client/api/bulk_load.rb,
lib/td/client/api/bulk_import.rb,
lib/td/client/api/server_status.rb,
lib/td/client/api/access_control.rb,
lib/td/client/api/partial_delete.rb

Defined Under Namespace

Modules: AccessControl, Account, BulkImport, BulkLoad, CountReadBodyTotalSize, Database, DeflateReadBodyMixin, DirectReadBodyMixin, Export, Import, Job, PartialDelete, Result, Schedule, ServerStatus, Table, User Classes: IncompleteError

Constant Summary collapse

DEFAULT_ENDPOINT = 
'api.treasure-data.com'
DEFAULT_IMPORT_ENDPOINT = 
'api-import.treasure-data.com'
NEW_DEFAULT_ENDPOINT = 
'api.treasuredata.com'
NEW_DEFAULT_IMPORT_ENDPOINT = 
'api-import.treasuredata.com'

Constants included from BulkLoad

BulkLoad::JOB, BulkLoad::LIST, BulkLoad::SESSION

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods included from User

#add_apikey, #add_user, #authenticate, #change_email, #change_my_password, #change_password, #list_apikeys, #list_users, #remove_apikey, #remove_user

Methods included from Table

#create_log_table, #delete_table, #list_tables, #swap_table, #tail, #update_expire, #update_schema

Methods included from ServerStatus

#server_status

Methods included from Schedule

#create_schedule, #delete_schedule, #history, #list_schedules, #run_schedule, #update_schedule

Methods included from Result

#create_result, #delete_result, #list_result

Methods included from PartialDelete

#partial_delete

Methods included from Job

#hive_query, #job_result, #job_result_each, #job_result_each_with_compr_size, #job_result_format, #job_result_raw, #job_status, #kill, #list_jobs, #pig_query, #query, #show_job

Methods included from Import

#import

Methods included from Export

#export

Methods included from Database

#create_database, #delete_database, #list_databases

Methods included from BulkLoad

#bulk_load_create, #bulk_load_delete, #bulk_load_guess, #bulk_load_history, #bulk_load_issue, #bulk_load_list, #bulk_load_preview, #bulk_load_run, #bulk_load_show, #bulk_load_update

Methods included from BulkImport

#bulk_import_delete_part, #bulk_import_error_records, #bulk_import_upload_part, #commit_bulk_import, #create_bulk_import, #delete_bulk_import, #freeze_bulk_import, #list_bulk_import_parts, #list_bulk_imports, #perform_bulk_import, #show_bulk_import, #unfreeze_bulk_import

Methods included from Account

#account_core_utilization, #show_account

Methods included from AccessControl

#grant_access_control, #list_access_controls, #revoke_access_control, #test_access_control

Constructor Details

#initialize(apikey, opts = {}) ⇒ API 



49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
 
# File 'lib/td/client/api.rb', line 49

def initialize(apikey, opts={})
  require 'json'
  require 'time'
  require 'uri'
  require 'net/http'
  require 'net/https'
  require 'time'
  #require 'faraday' # faraday doesn't support streaming upload with httpclient yet so now disabled
  require 'httpclient'
  require 'zlib'
  require 'stringio'
  require 'cgi'
  require 'msgpack'

  @apikey = apikey
  @user_agent = "TD-Client-Ruby: #{TreasureData::Client::VERSION}"
  @user_agent = "#{opts[:user_agent]}; " + @user_agent if opts.has_key?(:user_agent)

  endpoint = opts[:endpoint] || ENV['TD_API_SERVER'] || DEFAULT_ENDPOINT
  uri = URI.parse(endpoint)

  @connect_timeout = opts[:connect_timeout] || 60
  @read_timeout = opts[:read_timeout] || 600
  @send_timeout = opts[:send_timeout] || 600
  @retry_post_requests = opts[:retry_post_requests] || false
  @retry_delay = opts[:retry_delay] || 5
  @max_cumul_retry_delay = opts[:max_cumul_retry_delay] || 600

  case uri.scheme
  when 'http', 'https'
    @host = uri.host
    @port = uri.port
    # the opts[:ssl] option is ignored here, it's value
    #   overridden by the scheme of the endpoint URI
    @ssl = (uri.scheme == 'https')
    @base_path = uri.path.to_s

  else
    if uri.port
      # invalid URI
      raise "Invalid endpoint: #{endpoint}"
    end

    # generic URI
    @host, @port = endpoint.split(':', 2)
    @port = @port.to_i
    if opts[:ssl]
      @port = 443 if @port == 0
      @ssl = true
    else
      @port = 80 if @port == 0
      @ssl = false
    end
    @base_path = ''
  end

  @http_proxy = opts[:http_proxy] || ENV['HTTP_PROXY']
  if @http_proxy
    http_proxy = if @http_proxy =~ /\Ahttp:\/\/(.*)\z/
                   $~[1]
                 else
                   @http_proxy
                 end
    proxy_host, proxy_port = http_proxy.split(':', 2)
    proxy_port = (proxy_port ? proxy_port.to_i : 80)
    @http_class = Net::HTTP::Proxy(proxy_host, proxy_port)
  else
    @http_class = Net::HTTP
  end

  @headers = opts[:headers] || {}
  @api = api_client("#{@ssl ? 'https' : 'http'}://#{@host}:#{@port}")
end

Instance Attribute Details

#apikey ⇒ Object (readonly)

Returns the value of attribute apikey.



126
127
128
 
# File 'lib/td/client/api.rb', line 126

def apikey
  @apikey
end

Class Method Details

.create_empty_gz_data ⇒ String

for fluent-plugin-td / td command to check table existence with import onlt user



231
232
233
234
235
 
# File 'lib/td/client/api.rb', line 231

def self.create_empty_gz_data
  io = StringIO.new
  Zlib::GzipWriter.new(io).close
  io.string
end

.normalize_database_name(name) ⇒ Object 



189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
 
# File 'lib/td/client/api.rb', line 189

def self.normalize_database_name(name)
  name = name.to_s
  if name.empty?
    raise "Empty name is not allowed"
  end
  if name.length < 3
    name += "_" * (3 - name.length)
  end
  if 255 < name.length
    name = name[0, 253] + "__"
  end
  name = name.downcase
  name = name.gsub(/[^a-z0-9_]/, '_')
  name
end

.normalize_table_name(name) ⇒ Object 



206
207
208
 
# File 'lib/td/client/api.rb', line 206

def self.normalize_table_name(name)
  normalize_database_name(name)
end

.normalize_type_name(name) ⇒ Object

TODO support array types



212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/td/client/api.rb', line 212

def self.normalize_type_name(name)
  case name
  when /int/i, /integer/i
    "int"
  when /long/i, /bigint/i
    "long"
  when /string/i
    "string"
  when /float/i
    "float"
  when /double/i
    "double"
  else
    raise "Type name must eather of int, long, string float or double"
  end
end

.normalized_msgpack(record, out = nil) ⇒ Object 



130
131
132
133
134
135
136
137
138
 
# File 'lib/td/client/api.rb', line 130

def self.normalized_msgpack(record, out = nil)
  record.keys.each { |k|
    v = record[k]
    if v.kind_of?(Bignum)
      record[k] = v.to_s
    end
  }
  record.to_msgpack(out)
end

.validate_column_name(name) ⇒ Object 



184
185
186
 
# File 'lib/td/client/api.rb', line 184

def self.validate_column_name(name)
  validate_name("column", 1, 255, name)
end

.validate_database_name(name) ⇒ Object 



169
170
171
 
# File 'lib/td/client/api.rb', line 169

def self.validate_database_name(name)
  validate_name("database", 3, 255, name)
end

.validate_name(target, min_len, max_len, name) ⇒ Object 



144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/td/client/api.rb', line 144

def self.validate_name(target, min_len, max_len, name)
  if !target.instance_of?(String) || target.empty?
    raise ParameterValidationError,
          "A valid target name is required"
  end

  name = name.to_s
  if name.empty?
    raise ParameterValidationError,
          "Empty #{target} name is not allowed"
  end
  if name.length < min_len || name.length > max_len
    raise ParameterValidationError,
          "#{target.capitalize} name must be between #{min_len} and #{max_len} characters long. Got #{name.length} " +
          (name.length == 1 ? "character" : "characters") + "."
  end
  unless name =~ /^([a-z0-9_]+)$/
    raise ParameterValidationError,
          "#{target.capitalize} name must only consist of lower-case alpha-numeric characters and '_'."
  end

  name
end

.validate_result_set_name(name) ⇒ Object 



179
180
181
 
# File 'lib/td/client/api.rb', line 179

def self.validate_result_set_name(name)
  validate_name("result set", 3, 255, name)
end

.validate_table_name(name) ⇒ Object 



174
175
176
 
# File 'lib/td/client/api.rb', line 174

def self.validate_table_name(name)
  validate_name("table", 3, 255, name)
end

Instance Method Details

#ssl_ca_file=(ssl_ca_file) ⇒ Object 



238
239
240
 
# File 'lib/td/client/api.rb', line 238

def ssl_ca_file=(ssl_ca_file)
  @ssl_ca_file = ssl_ca_file
end