Module: TreasureData::API::BulkLoad

Included in:
TreasureData::API
Defined in:
lib/td/client/api/bulk_load.rb

Constant Summary collapse

LIST =

The ‘BulkLoadSession’ resource in td-api is as follows; {

"config": {
  "type": "s3",
  "access_key_id": s3 access key id,
  "secret_access_key": s3 secret key,
  "endpoint": s3 endpoint name,
  "bucket": s3 bucket name,
  "path_prefix": "a/prefix/of/files",
  "decoders": []
},
"name": account_wide_unique_name,
"cron": cron_string,
"timezone": timezone_string,
"delay": delay_seconds,
"database": database_name,
"table": table_name

}

'/v3/bulk_loads'
SESSION = 
LIST +     '/%s'
JOB = 
SESSION +         '/jobs'

Instance Method Summary collapse

Instance Method Details

#bulk_load_create(name, database, table, job, opts = {}) ⇒ Object

name: String, database: String, table: String, job: Hash -> Hash



94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/td/client/api/bulk_load.rb', line 94

def bulk_load_create(name, database, table, job, opts = {})
  job = job.dup
  job['name'] = name
  [:cron, :timezone, :delay, :time_column].each do |prop|
    job[prop.to_s] = opts[prop] if opts.key?(prop)
  end
  job['database'] = database
  job['table'] = table
  res = api { post(LIST, job.to_json) }
  unless res.ok?
    raise_error("BulkLoadSession: #{name} create failed", res)
  end
  JSON.load(res.body)
end

#bulk_load_delete(name) ⇒ Object

name: String -> Hash



130
131
132
133
134
135
136
137
 
# File 'lib/td/client/api/bulk_load.rb', line 130

def bulk_load_delete(name)
  path = session_path(name)
  res = api { delete(path) }
  unless res.ok?
    raise_error("BulkLoadSession: #{name} delete failed", res)
  end
  JSON.load(res.body)
end

#bulk_load_guess(job) ⇒ Object

job: Hash -> Hash



48
49
50
51
52
53
54
55
56
 
# File 'lib/td/client/api/bulk_load.rb', line 48

def bulk_load_guess(job)
  # retry_request = true
  path = LIST + '/guess'
  res = api { post(path, job.to_json) }
  unless res.ok?
    raise_error('BulkLoad configuration guess failed', res)
  end
  JSON.load(res.body)
end

#bulk_load_history(name) ⇒ Object

name: String -> [Hash]



140
141
142
143
144
145
146
147
 
# File 'lib/td/client/api/bulk_load.rb', line 140

def bulk_load_history(name)
  path = job_path(name)
  res = api { get(path) }
  unless res.ok?
    raise_error("history of BulkLoadSession: #{name} retrieve failed", res)
  end
  JSON.load(res.body)
end

#bulk_load_issue(database, table, job) ⇒ Object

job: Hash -> String (job_id)



70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/td/client/api/bulk_load.rb', line 70

def bulk_load_issue(database, table, job)
  type = 'bulkload'
  job = job.dup
  job['database'] = database
  job['table'] = table
  path = "/v3/job/issue/#{e type}/#{e database}"
  res = api { post(path, job.to_json) }
  unless res.ok?
    raise_error('BulkLoad job issuing failed', res)
  end
  js = checked_json(res.body)
  js['job_id'].to_s
end

#bulk_load_list ⇒ Object

nil -> [Hash]



85
86
87
88
89
90
91
 
# File 'lib/td/client/api/bulk_load.rb', line 85

def bulk_load_list
  res = api { get(LIST) }
  unless res.ok?
    raise_error("BulkLoadSession list retrieve failed", res)
  end
  JSON.load(res.body)
end

#bulk_load_preview(job) ⇒ Object

job: Hash -> Hash



59
60
61
62
63
64
65
66
67
 
# File 'lib/td/client/api/bulk_load.rb', line 59

def bulk_load_preview(job)
  # retry_request = true
  path = LIST + '/preview'
  res = api { post(path, job.to_json) }
  unless res.ok?
    raise_error('BulkLoad job preview failed', res)
  end
  JSON.load(res.body)
end

#bulk_load_run(name, scheduled_time = nil) ⇒ Object 



149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/td/client/api/bulk_load.rb', line 149

def bulk_load_run(name, scheduled_time = nil)
  path = job_path(name)
  opts = {}
  opts[:scheduled_time] = scheduled_time.to_s unless scheduled_time.nil?
  res = api { post(path, opts.to_json) }
  unless res.ok?
    raise_error("BulkLoadSession: #{name} job create failed", res)
  end
  js = checked_json(res.body)
  js['job_id'].to_s
end

#bulk_load_show(name) ⇒ Object

name: String -> Hash



110
111
112
113
114
115
116
117
 
# File 'lib/td/client/api/bulk_load.rb', line 110

def bulk_load_show(name)
  path = session_path(name)
  res = api { get(path) }
  unless res.ok?
    raise_error("BulkLoadSession: #{name} retrieve failed", res)
  end
  JSON.load(res.body)
end

#bulk_load_update(name, settings) ⇒ Object

name: String, settings: Hash -> Hash



120
121
122
123
124
125
126
127
 
# File 'lib/td/client/api/bulk_load.rb', line 120

def bulk_load_update(name, settings)
  path = session_path(name)
  res = api { put(path, settings.to_json) }
  unless res.ok?
    raise_error("BulkLoadSession: #{name} update failed", res)
  end
  JSON.load(res.body)
end