Class: Pipely::Deploy::Client

Inherits:
Object
  • Object
show all
Defined in:
lib/pipely/deploy/client.rb

Overview

Client for managing deployment of rendered definitions.

Defined Under Namespace

Classes: PipelineDeployerError

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(log = nil) ⇒ Client

Returns a new instance of Client.



19
20
21
22
23
24
25
26
 
# File 'lib/pipely/deploy/client.rb', line 19

def initialize(log=nil)
  @log = log || Logger.new(STDOUT)
  @aws = Aws::DataPipeline::Client.new
  @base_tags = {
    "environment" => ENV['env'],
    "creator" => ENV['USER']
  }
end

Instance Attribute Details

#base_tags ⇒ Object (readonly)

Returns the value of attribute base_tags.



14
15
16
 
# File 'lib/pipely/deploy/client.rb', line 14

def base_tags
  @base_tags
end

Instance Method Details

#activate_pipeline(response, pipeline) ⇒ Object 



99
100
101
102
103
104
105
106
107
108
 
# File 'lib/pipely/deploy/client.rb', line 99

def activate_pipeline(response, pipeline)
  if response[:errored]
    @log.error("Failed to put pipeline definition.")
    @log.error(response[:validation_errors].inspect)
    false
  else
    @aws.activate_pipeline(pipeline_id: pipeline.pipeline_id)
    pipeline.pipeline_id
  end
end

#create_pipeline(pipeline_name, definition, tags = {}) ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/pipely/deploy/client.rb', line 79

def create_pipeline(pipeline_name, definition, tags={})
  created_pipeline = @aws.create_pipeline(
    name: pipeline_name,
    unique_id: tags['deploy_id'] || SecureRandom.uuid,
    description: "Pipely Deployed Data Pipeline",
    tags: base_tags.merge(tags).map do |k,v|
      { key: k, value: v } unless v.nil?
    end.compact,
  )

  definition ||= yield(created_pipeline.pipeline_id) if block_given?

  response = @aws.put_pipeline_definition(
    pipeline_id: created_pipeline.pipeline_id,
    pipeline_objects: JSONDefinition.parse(definition)
  )

  activate_pipeline(response, created_pipeline)
end

#delete_pipeline(pipeline_id) ⇒ Object 



110
111
112
 
# File 'lib/pipely/deploy/client.rb', line 110

def delete_pipeline(pipeline_id)
  @aws.delete_pipeline(pipeline_id: pipeline_id)
end

#deploy_pipeline(pipeline_basename, definition = nil, &block) ⇒ Object 



28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/pipely/deploy/client.rb', line 28

def deploy_pipeline(pipeline_basename, definition = nil, &block)
  pipeline_name = pipeline_name(pipeline_basename)

  tags = base_tags.merge(
    "basename" => pipeline_basename,
    "deploy_id" => SecureRandom.uuid )

  # Get a list of all existing pipelines
  pipeline_ids = existing_pipelines(pipeline_name)
  @log.info("#{pipeline_ids.count} existing pipelines: #{pipeline_ids}")

  # Create new pipeline
  created_pipeline_id = create_pipeline(
    pipeline_name, definition, tags, &block
  )

  if created_pipeline_id
    @log.info("Created pipeline id '#{created_pipeline_id}'")

    # Delete old pipelines
    pipeline_ids.each do |pipeline_id|
      begin
        delete_pipeline(pipeline_id)
        @log.info("Deleted pipeline '#{pipeline_id}'")

      rescue PipelineDeployerError => error
        @log.warn(error)
      end
    end
  end

  created_pipeline_id
end

#existing_pipelines(pipeline_name) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/pipely/deploy/client.rb', line 62

def existing_pipelines(pipeline_name)
  ids = []
  marker = nil

  begin
    options = marker ? { marker: marker } : {}
    result = @aws.list_pipelines(options)

    ids += result[:pipeline_id_list].
             select { |p| p[:name] == pipeline_name }.
             map { |p| p[:id] }

  end while (result[:has_more_results] && marker = result[:marker])

  ids
end