Method: GoodData::Model.upload_multiple_data

Defined in:

lib/gooddata/models/model.rb

.upload_multiple_data(data, project_blueprint, options = { :client => GoodData.connection, :project => GoodData.project }) ⇒ Hash

Uploads multiple data sets using batch upload interface

Parameters:

• Input data

• Project blueprint

• (defaults to: { :client => GoodData.connection, :project => GoodData.project })

  Additional options

Returns:

• Batch upload result

# File 'lib/gooddata/models/model.rb', line 154

def upload_multiple_data(data, project_blueprint, options = { :client => GoodData.connection, :project => GoodData.project })
  client, project = GoodData.get_client_and_project(options)
  project ||= GoodData.project

  manifest = {
    'dataSetSLIManifestList' => data.map do |d|
      mode = d[:options] && d[:options][:mode] ? d[:options][:mode] : options[:mode] || 'FULL'
      GoodData::Model::ToManifest.dataset_to_manifest(project_blueprint, d[:dataset], mode)
    end
  }

  csv_headers = []

  dir = Dir.mktmpdir
  begin
    Zip::File.open("#{dir}/upload.zip", create: true) do |zip|
      # TODO: make sure schema columns match CSV column names
      zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }

      data.zip(manifest['dataSetSLIManifestList']).each do |item|
        column_mapping = item[0][:options] ? item[0][:options][:column_mapping] : nil
        path = item[0][:data]
        path = item[0][:data].path if item[0][:data].respond_to? :path
        inline_data = !path.is_a?(String)

        data_to_upload = inline_data ? path : File.open(path)

        filename = item[1]['dataSetSLIManifest']['file']

        zip.get_output_stream(filename) do |file|
          data_to_upload.each_with_index do |row, index|
            row = CSV.parse(row).first unless inline_data

            if index.zero?
              row.map! { |h| column_mapping.key(h) || h } if column_mapping
              csv_headers << row
            end

            file.puts row.to_csv
          end
        end
      end
    end

    # upload it
    client.upload_to_user_webdav("#{dir}/upload.zip", :directory => File.basename(dir), :client => options[:client], :project => options[:project])
  ensure
    FileUtils.rm_rf dir
  end
  csv_headers.flatten!

  # kick the load
  pull = { 'pullIntegration' => File.basename(dir) }
  link = project.md.links('etl')['pull2']

  # TODO: List uploaded datasets
  task = client.post(link, pull, :info_message => 'Starting the data load from user storage to dataset.')

  res = client.poll_on_response(task['pull2Task']['links']['poll'], :info_message => 'Getting status of the dataload task.') do |body|
    body['wTaskStatus']['status'] == 'RUNNING' || body['wTaskStatus']['status'] == 'PREPARED'
  end

  if res['wTaskStatus']['status'] == 'ERROR'
    s = StringIO.new

    messages = res['wTaskStatus']['messages'] || []
    messages.each do |msg|
      GoodData.logger.error(JSON.pretty_generate(msg))
    end

    begin
      client.download_from_user_webdav(File.basename(dir) + '/upload_status.json', s, :client => client, :project => project)
    rescue => e
      raise "Unable to download upload_status.json from remote server, reason: #{e.message}"
    end

    js = MultiJson.load(s.string)
    manifests = manifest['dataSetSLIManifestList'].map do |m|
      m['dataSetSLIManifest']
    end

    parts = manifests.map do |m|
      m['parts']
    end

    manifest_cols = parts.flatten.map { |c| c['columnName'] }

    # extract some human readable error message from the webdav file
    csv_headers.map!(&:to_sym)
    manifest_cols.map!(&:to_sym)
    manifest_extra = manifest_cols - csv_headers
    csv_extra = csv_headers - manifest_cols

    error_message = begin
      js['error']['message'] % js['error']['parameters']
    rescue NoMethodError, ArgumentError
      ''
    end
    m = "Load failed with error '#{error_message}'.\n"
    m += "Columns that should be there (manifest) but aren't in uploaded csv: #{manifest_extra}\n" unless manifest_extra.empty?
    m += "Columns that are in csv but shouldn't be there (manifest): #{csv_extra}\n" unless csv_extra.empty?
    m += "Columns in the uploaded csv: #{csv_headers}\n"
    m += "Columns in the manifest: #{manifest_cols}\n"
    m += "Original message:\n#{JSON.pretty_generate(js)}\n"
    m += "Manifest used for uploading:\n#{JSON.pretty_generate(manifest)}"
    fail m
  end

  res
end