Class: Contentful::Importer::ParallelImporter

Inherits:
Object
  • Object
show all
Defined in:
lib/contentful/importer/parallel_importer.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(settings) ⇒ ParallelImporter

Returns a new instance of ParallelImporter.



17
18
19
20
21
22
 
# File 'lib/contentful/importer/parallel_importer.rb', line 17

def initialize(settings)
  @config = settings
  @logger = Logger.new(STDOUT)
  @data_organizer = DataOrganizer.new(@config)
  Contentful::Management::Client.new(config.config['access_token'], default_locale: config.config['default_locale'] || 'en-US')
end

Instance Attribute Details

#config ⇒ Object (readonly)

Returns the value of attribute config.



14
15
16
 
# File 'lib/contentful/importer/parallel_importer.rb', line 14

def config
  @config
end

#content_type=(value) ⇒ Object

Sets the attribute content_type

Parameters:

  • value —

    the value to set the attribute content_type to.



15
16
17
 
# File 'lib/contentful/importer/parallel_importer.rb', line 15

def content_type=(value)
  @content_type = value
end

#data_organizer ⇒ Object (readonly)

Returns the value of attribute data_organizer.



14
15
16
 
# File 'lib/contentful/importer/parallel_importer.rb', line 14

def data_organizer
  @data_organizer
end

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



14
15
16
 
# File 'lib/contentful/importer/parallel_importer.rb', line 14

def logger
  @logger
end

#space ⇒ Object (readonly)

Returns the value of attribute space.



14
15
16
 
# File 'lib/contentful/importer/parallel_importer.rb', line 14

def space
  @space
end

Instance Method Details

#asset_not_imported_yet?(asset_attributes, assets_ids) ⇒ Boolean

Returns:

  • (Boolean) 


100
101
102
 
# File 'lib/contentful/importer/parallel_importer.rb', line 100

def asset_not_imported_yet?(asset_attributes, assets_ids)
  !assets_ids.to_a.flatten.include?(asset_attributes['id'])
end

#asset_status(asset, asset_attributes) ⇒ Object 



112
113
114
115
116
117
118
119
120
121
 
# File 'lib/contentful/importer/parallel_importer.rb', line 112

def asset_status(asset, asset_attributes)
  if asset.is_a?(Contentful::Management::Asset)
    logger.info "Process asset - #{asset.id} "
    asset.process_file
    CSV.open("#{config.log_files_dir}/success_assets.csv", 'a') { |csv| csv << [asset.id] }
  else
    logger.info "Error - #{asset.message} - #{asset.response.raw} "
    CSV.open("#{config.log_files_dir}/failure_assets.csv", 'a') { |csv| csv << [asset_attributes['id'], asset.message, asset.response.raw] }
  end
end

#asset_url_param_start_with_http?(asset_attributes) ⇒ Boolean

Returns:

  • (Boolean) 


96
97
98
 
# File 'lib/contentful/importer/parallel_importer.rb', line 96

def asset_url_param_start_with_http?(asset_attributes)
  asset_attributes['url'] && asset_attributes['url'].start_with?('http')
end

#create_asset_file(asset_title, params) ⇒ Object 



104
105
106
107
108
109
110
 
# File 'lib/contentful/importer/parallel_importer.rb', line 104

def create_asset_file(asset_title, params)
  Contentful::Management::File.new.tap do |file|
    file.properties[:contentType] = file_content_type(params)
    file.properties[:fileName] = asset_title
    file.properties[:upload] = params['url']
  end
end

#create_contentful_model(space) ⇒ Object 



24
25
26
27
 
# File 'lib/contentful/importer/parallel_importer.rb', line 24

def create_contentful_model(space)
  initialize_space(space)
  import_content_types
end

#import_asset(asset_attributes) ⇒ Object 



86
87
88
89
90
91
92
93
94
 
# File 'lib/contentful/importer/parallel_importer.rb', line 86

def import_asset(asset_attributes)
  logger.info "Import asset - #{asset_attributes['id']} "
  asset_title = asset_attributes['name'].present? ? asset_attributes['name'] : asset_attributes['id']
  asset_description = asset_attributes['description'].present? ? asset_attributes['description'] : ''
  asset_file = create_asset_file(asset_title, asset_attributes)
  space = Contentful::Management::Space.find(config.config['space_id'])
  asset = space.assets.create(id: "#{asset_attributes['id']}", title: "#{asset_title}", description: asset_description, file: asset_file)
  asset_status(asset, asset_attributes)
end

#import_data(threads) ⇒ Object 



29
30
31
32
33
 
# File 'lib/contentful/importer/parallel_importer.rb', line 29

def import_data(threads)
  clean_threads_dir_before_import(threads)
  data_organizer.execute(threads)
  import_in_threads
end

#import_entries(path, space_id) ⇒ Object 



64
65
66
67
68
69
70
71
72
73
 
# File 'lib/contentful/importer/parallel_importer.rb', line 64

def import_entries(path, space_id)
  log_file_name = "success_thread_#{File.basename(path)}"
  create_log_file(log_file_name)
  load_log_files
  Dir.glob("#{path}/*.json") do |entry_path|
    content_type_id = File.basename(entry_path).match(/(.+)_\d+/)[1]
    entry_file_name = File.basename(entry_path)
    import_entry(entry_path, space_id, content_type_id, log_file_name) unless config.imported_entries.flatten.include?(entry_file_name)
  end
end

#import_in_threads ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
 
# File 'lib/contentful/importer/parallel_importer.rb', line 52

def import_in_threads
  threads = []
  number_of_threads.times do |thread_id|
    threads << Thread.new do
      self.class.new(config).import_entries("#{config.threads_dir}/#{thread_id}", config.space_id)
    end
  end
  threads.each do |thread|
    thread.join
  end
end

#import_only_assets ⇒ Object 



75
76
77
78
79
80
81
82
83
84
 
# File 'lib/contentful/importer/parallel_importer.rb', line 75

def import_only_assets
  create_log_file('success_assets')
  assets_ids = Set.new(CSV.read("#{config.data_dir}/logs/success_assets.csv", 'r'))
  Dir.glob("#{config.assets_dir}/**/*json") do |file_path|
    asset_attributes = JSON.parse(File.read(file_path))
    if asset_url_param_start_with_http?(asset_attributes) && asset_not_imported_yet?(asset_attributes, assets_ids)
      import_asset(asset_attributes)
    end
  end
end

#number_of_threads ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/contentful/importer/parallel_importer.rb', line 44

def number_of_threads
  number_of_threads = 0
  Dir.glob("#{config.threads_dir}/*") do |thread|
    number_of_threads += 1 if File.basename(thread).size == 1
  end
  number_of_threads
end

#publish_all_entries(thread_dir) ⇒ Object 



164
165
166
167
168
169
170
171
 
# File 'lib/contentful/importer/parallel_importer.rb', line 164

def publish_all_entries(thread_dir)
  create_log_file('success_published_entries')
  config.published_entries << CSV.read("#{config.log_files_dir}/success_published_entries.csv", 'r').flatten
  Dir.glob("#{thread_dir}/*json") do |entry_file|
    entry_id = JSON.parse(File.read(entry_file))['id']
    publish_entry(entry_id) unless config.published_entries.flatten.include?(entry_id)
  end
end

#publish_asset(asset_id) ⇒ Object 



158
159
160
161
162
 
# File 'lib/contentful/importer/parallel_importer.rb', line 158

def publish_asset(asset_id)
  logger.info "Publish an Asset - ID: #{asset_id}"
  asset = Contentful::Management::Asset.find(config.config['space_id'], asset_id).publish
  publish_status(asset, asset_id, 'published_assets')
end

#publish_assets(thread_dir) ⇒ Object 



149
150
151
152
153
154
155
156
 
# File 'lib/contentful/importer/parallel_importer.rb', line 149

def publish_assets(thread_dir)
  create_log_file('success_published_assets')
  config.published_assets << CSV.read("#{config.log_files_dir}/success_published_assets.csv", 'r').flatten
  Dir.glob("#{thread_dir}/*json") do |asset_file|
    asset_id = JSON.parse(File.read(asset_file))['id']
    publish_asset(asset_id) unless config.published_assets.flatten.include?(asset_id)
  end
end

#publish_assets_in_threads(number_of_threads) ⇒ Object 



135
136
137
138
139
140
141
142
143
144
145
146
147
 
# File 'lib/contentful/importer/parallel_importer.rb', line 135

def publish_assets_in_threads(number_of_threads)
  clean_assets_threads_dir_before_publish(number_of_threads)
  data_organizer.split_assets_to_threads(number_of_threads)
  threads =[]
  number_of_threads.times do |thread_id|
    threads << Thread.new do
      self.class.new(config).publish_assets("#{config.threads_dir}/assets/#{thread_id}")
    end
  end
  threads.each do |thread|
    thread.join
  end
end

#publish_entries_in_threads ⇒ Object 



123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/contentful/importer/parallel_importer.rb', line 123

def publish_entries_in_threads
  threads =[]
  number_of_threads.times do |thread_id|
    threads << Thread.new do
      self.class.new(config).publish_all_entries("#{config.threads_dir}/#{thread_id}")
    end
  end
  threads.each do |thread|
    thread.join
  end
end

#publish_entry(entry_id) ⇒ Object 



173
174
175
176
177
 
# File 'lib/contentful/importer/parallel_importer.rb', line 173

def publish_entry(entry_id)
  logger.info "Publish entries for #{entry_id}."
  entry = Contentful::Management::Entry.find(config.config['space_id'], entry_id).publish
  publish_status(entry, entry_id, 'published_entries')
end

#test_credentials ⇒ Object 



35
36
37
38
39
40
41
42
 
# File 'lib/contentful/importer/parallel_importer.rb', line 35

def test_credentials
  spaces = Contentful::Management::Space.all
  if spaces.is_a? Contentful::Management::Array
    logger.info 'Contentful Management API credentials: OK'
  end
rescue NoMethodError => _error
  logger.info 'Contentful Management API credentials: INVALID (check README)'
end