Module: AwDatapipe

Defined in:: lib/aw_datapipe/session.rb,
lib/aw_datapipe/version.rb,
lib/aw_datapipe/pipeline.rb,
lib/aw_datapipe/object_hash.rb,
lib/aw_datapipe/source_writer.rb,
lib/aw_datapipe/pipeline_object.rb,
lib/aw_datapipe/parameter_metadata.rb,
lib/aw_datapipe/pipeline_serializer.rb

Defined Under Namespace

Classes: ObjectHash, ParameterMetadata, Pipeline, PipelineObject, PipelineSerializer, Session, SourceWriter

Constant Summary collapse

VERSION =

'0.3.1'

Configuration =

PipelineObject.new(:failure_and_rerun_mode, :pipeline_log_uri, :resource_role, :role, :schedule, :schedule_type)

Schedule =

PipelineObject.new(:period, :start_date_time)

Ec2Resource =

PipelineObject.new(:action_on_task_failure, :instance_type, :security_group_ids, :subnet_id, :terminate_after) do
  def copy_activity(params)
    pipeline.append_object CopyActivity.build(params.merge(runs_on: self))
  end

  def shell_command_activity(params)
    pipeline.append_object ShellCommandActivity.build(params.merge(runs_on: self))
  end
end

S3DataNode =

PipelineObject.new(:directory_path, :data_format, :file_path)

CsvDataFormat =

PipelineObject.new(:column) do
  def type
    'CSV'
  end
end

ShellCommandActivity =

PipelineObject.new(:input, :output, :runs_on, :command, :script_argument, :script_uri, :stage)

JdbcDatabase =

PipelineObject.new(:_password, :connection_string, :jdbc_driver_class, :username) do
  def sql_data_node(params)
    pipeline.append_object SqlDataNode.build(params.merge(database: self))
  end
end

SqlDataNode =

PipelineObject.new(:database, :select_query, :table)

CopyActivity =

PipelineObject.new(:input, :output, :runs_on)

RedshiftDatabase =

PipelineObject.new(:_password, :connection_string, :database_name, :username)

RedshiftDataNode =

PipelineObject.new(:create_table_sql, :database, :primary_keys, :schema_name, :table_name)

RedshiftCopyActivity =

PipelineObject.new(:input, :insert_mode, :output, :runs_on)