Class: Aws::Firehose::Types::CreateDeliveryStreamInput

Inherits:
Struct
  • Object
show all
Includes:
Structure
Defined in:
lib/aws-sdk-firehose/types.rb

Overview

Note:

When making an API call, you may pass CreateDeliveryStreamInput data as a hash:

{
  delivery_stream_name: "DeliveryStreamName", # required
  delivery_stream_type: "DirectPut", # accepts DirectPut, KinesisStreamAsSource
  kinesis_stream_source_configuration: {
    kinesis_stream_arn: "KinesisStreamARN", # required
    role_arn: "RoleARN", # required
  },
  s3_destination_configuration: {
    role_arn: "RoleARN", # required
    bucket_arn: "BucketARN", # required
    prefix: "Prefix",
    error_output_prefix: "ErrorOutputPrefix",
    buffering_hints: {
      size_in_m_bs: 1,
      interval_in_seconds: 1,
    },
    compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
    encryption_configuration: {
      no_encryption_config: "NoEncryption", # accepts NoEncryption
      kms_encryption_config: {
        awskms_key_arn: "AWSKMSKeyARN", # required
      },
    },
    cloud_watch_logging_options: {
      enabled: false,
      log_group_name: "LogGroupName",
      log_stream_name: "LogStreamName",
    },
  },
  extended_s3_destination_configuration: {
    role_arn: "RoleARN", # required
    bucket_arn: "BucketARN", # required
    prefix: "Prefix",
    error_output_prefix: "ErrorOutputPrefix",
    buffering_hints: {
      size_in_m_bs: 1,
      interval_in_seconds: 1,
    },
    compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
    encryption_configuration: {
      no_encryption_config: "NoEncryption", # accepts NoEncryption
      kms_encryption_config: {
        awskms_key_arn: "AWSKMSKeyARN", # required
      },
    },
    cloud_watch_logging_options: {
      enabled: false,
      log_group_name: "LogGroupName",
      log_stream_name: "LogStreamName",
    },
    processing_configuration: {
      enabled: false,
      processors: [
        {
          type: "Lambda", # required, accepts Lambda
          parameters: [
            {
              parameter_name: "LambdaArn", # required, accepts LambdaArn, NumberOfRetries, RoleArn, BufferSizeInMBs, BufferIntervalInSeconds
              parameter_value: "ProcessorParameterValue", # required
            },
          ],
        },
      ],
    },
    s3_backup_mode: "Disabled", # accepts Disabled, Enabled
    s3_backup_configuration: {
      role_arn: "RoleARN", # required
      bucket_arn: "BucketARN", # required
      prefix: "Prefix",
      error_output_prefix: "ErrorOutputPrefix",
      buffering_hints: {
        size_in_m_bs: 1,
        interval_in_seconds: 1,
      },
      compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
      encryption_configuration: {
        no_encryption_config: "NoEncryption", # accepts NoEncryption
        kms_encryption_config: {
          awskms_key_arn: "AWSKMSKeyARN", # required
        },
      },
      cloud_watch_logging_options: {
        enabled: false,
        log_group_name: "LogGroupName",
        log_stream_name: "LogStreamName",
      },
    },
    data_format_conversion_configuration: {
      schema_configuration: {
        role_arn: "NonEmptyStringWithoutWhitespace",
        catalog_id: "NonEmptyStringWithoutWhitespace",
        database_name: "NonEmptyStringWithoutWhitespace",
        table_name: "NonEmptyStringWithoutWhitespace",
        region: "NonEmptyStringWithoutWhitespace",
        version_id: "NonEmptyStringWithoutWhitespace",
      },
      input_format_configuration: {
        deserializer: {
          open_x_json_ser_de: {
            convert_dots_in_json_keys_to_underscores: false,
            case_insensitive: false,
            column_to_json_key_mappings: {
              "NonEmptyStringWithoutWhitespace" => "NonEmptyString",
            },
          },
          hive_json_ser_de: {
            timestamp_formats: ["NonEmptyString"],
          },
        },
      },
      output_format_configuration: {
        serializer: {
          parquet_ser_de: {
            block_size_bytes: 1,
            page_size_bytes: 1,
            compression: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, SNAPPY
            enable_dictionary_compression: false,
            max_padding_bytes: 1,
            writer_version: "V1", # accepts V1, V2
          },
          orc_ser_de: {
            stripe_size_bytes: 1,
            block_size_bytes: 1,
            row_index_stride: 1,
            enable_padding: false,
            padding_tolerance: 1.0,
            compression: "NONE", # accepts NONE, ZLIB, SNAPPY
            bloom_filter_columns: ["NonEmptyStringWithoutWhitespace"],
            bloom_filter_false_positive_probability: 1.0,
            dictionary_key_threshold: 1.0,
            format_version: "V0_11", # accepts V0_11, V0_12
          },
        },
      },
      enabled: false,
    },
  },
  redshift_destination_configuration: {
    role_arn: "RoleARN", # required
    cluster_jdbc_url: "ClusterJDBCURL", # required
    copy_command: { # required
      data_table_name: "DataTableName", # required
      data_table_columns: "DataTableColumns",
      copy_options: "CopyOptions",
    },
    username: "Username", # required
    password: "Password", # required
    retry_options: {
      duration_in_seconds: 1,
    },
    s3_configuration: { # required
      role_arn: "RoleARN", # required
      bucket_arn: "BucketARN", # required
      prefix: "Prefix",
      error_output_prefix: "ErrorOutputPrefix",
      buffering_hints: {
        size_in_m_bs: 1,
        interval_in_seconds: 1,
      },
      compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
      encryption_configuration: {
        no_encryption_config: "NoEncryption", # accepts NoEncryption
        kms_encryption_config: {
          awskms_key_arn: "AWSKMSKeyARN", # required
        },
      },
      cloud_watch_logging_options: {
        enabled: false,
        log_group_name: "LogGroupName",
        log_stream_name: "LogStreamName",
      },
    },
    processing_configuration: {
      enabled: false,
      processors: [
        {
          type: "Lambda", # required, accepts Lambda
          parameters: [
            {
              parameter_name: "LambdaArn", # required, accepts LambdaArn, NumberOfRetries, RoleArn, BufferSizeInMBs, BufferIntervalInSeconds
              parameter_value: "ProcessorParameterValue", # required
            },
          ],
        },
      ],
    },
    s3_backup_mode: "Disabled", # accepts Disabled, Enabled
    s3_backup_configuration: {
      role_arn: "RoleARN", # required
      bucket_arn: "BucketARN", # required
      prefix: "Prefix",
      error_output_prefix: "ErrorOutputPrefix",
      buffering_hints: {
        size_in_m_bs: 1,
        interval_in_seconds: 1,
      },
      compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
      encryption_configuration: {
        no_encryption_config: "NoEncryption", # accepts NoEncryption
        kms_encryption_config: {
          awskms_key_arn: "AWSKMSKeyARN", # required
        },
      },
      cloud_watch_logging_options: {
        enabled: false,
        log_group_name: "LogGroupName",
        log_stream_name: "LogStreamName",
      },
    },
    cloud_watch_logging_options: {
      enabled: false,
      log_group_name: "LogGroupName",
      log_stream_name: "LogStreamName",
    },
  },
  elasticsearch_destination_configuration: {
    role_arn: "RoleARN", # required
    domain_arn: "ElasticsearchDomainARN", # required
    index_name: "ElasticsearchIndexName", # required
    type_name: "ElasticsearchTypeName", # required
    index_rotation_period: "NoRotation", # accepts NoRotation, OneHour, OneDay, OneWeek, OneMonth
    buffering_hints: {
      interval_in_seconds: 1,
      size_in_m_bs: 1,
    },
    retry_options: {
      duration_in_seconds: 1,
    },
    s3_backup_mode: "FailedDocumentsOnly", # accepts FailedDocumentsOnly, AllDocuments
    s3_configuration: { # required
      role_arn: "RoleARN", # required
      bucket_arn: "BucketARN", # required
      prefix: "Prefix",
      error_output_prefix: "ErrorOutputPrefix",
      buffering_hints: {
        size_in_m_bs: 1,
        interval_in_seconds: 1,
      },
      compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
      encryption_configuration: {
        no_encryption_config: "NoEncryption", # accepts NoEncryption
        kms_encryption_config: {
          awskms_key_arn: "AWSKMSKeyARN", # required
        },
      },
      cloud_watch_logging_options: {
        enabled: false,
        log_group_name: "LogGroupName",
        log_stream_name: "LogStreamName",
      },
    },
    processing_configuration: {
      enabled: false,
      processors: [
        {
          type: "Lambda", # required, accepts Lambda
          parameters: [
            {
              parameter_name: "LambdaArn", # required, accepts LambdaArn, NumberOfRetries, RoleArn, BufferSizeInMBs, BufferIntervalInSeconds
              parameter_value: "ProcessorParameterValue", # required
            },
          ],
        },
      ],
    },
    cloud_watch_logging_options: {
      enabled: false,
      log_group_name: "LogGroupName",
      log_stream_name: "LogStreamName",
    },
  },
  splunk_destination_configuration: {
    hec_endpoint: "HECEndpoint", # required
    hec_endpoint_type: "Raw", # required, accepts Raw, Event
    hec_token: "HECToken", # required
    hec_acknowledgment_timeout_in_seconds: 1,
    retry_options: {
      duration_in_seconds: 1,
    },
    s3_backup_mode: "FailedEventsOnly", # accepts FailedEventsOnly, AllEvents
    s3_configuration: { # required
      role_arn: "RoleARN", # required
      bucket_arn: "BucketARN", # required
      prefix: "Prefix",
      error_output_prefix: "ErrorOutputPrefix",
      buffering_hints: {
        size_in_m_bs: 1,
        interval_in_seconds: 1,
      },
      compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
      encryption_configuration: {
        no_encryption_config: "NoEncryption", # accepts NoEncryption
        kms_encryption_config: {
          awskms_key_arn: "AWSKMSKeyARN", # required
        },
      },
      cloud_watch_logging_options: {
        enabled: false,
        log_group_name: "LogGroupName",
        log_stream_name: "LogStreamName",
      },
    },
    processing_configuration: {
      enabled: false,
      processors: [
        {
          type: "Lambda", # required, accepts Lambda
          parameters: [
            {
              parameter_name: "LambdaArn", # required, accepts LambdaArn, NumberOfRetries, RoleArn, BufferSizeInMBs, BufferIntervalInSeconds
              parameter_value: "ProcessorParameterValue", # required
            },
          ],
        },
      ],
    },
    cloud_watch_logging_options: {
      enabled: false,
      log_group_name: "LogGroupName",
      log_stream_name: "LogStreamName",
    },
  },
  tags: [
    {
      key: "TagKey", # required
      value: "TagValue",
    },
  ],
}

See Also:

Instance Attribute Summary collapse

Instance Attribute Details

#delivery_stream_name ⇒ String

The name of the delivery stream. This name must be unique per AWS account in the same AWS Region. If the delivery streams are in different accounts or different Regions, you can have multiple delivery streams with the same name.

Returns:

  • (String) 


539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#delivery_stream_type ⇒ String

The delivery stream type. This parameter can be one of the following values:

  • ‘DirectPut`: Provider applications access the delivery stream directly.

  • ‘KinesisStreamAsSource`: The delivery stream uses a Kinesis data stream as a source.

Returns:

  • (String) 


539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#elasticsearch_destination_configuration ⇒ Types::ElasticsearchDestinationConfiguration

The destination in Amazon ES. You can specify only one destination.

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#extended_s3_destination_configuration ⇒ Types::ExtendedS3DestinationConfiguration

The destination in Amazon S3. You can specify only one destination.

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#kinesis_stream_source_configuration ⇒ Types::KinesisStreamSourceConfiguration

When a Kinesis data stream is used as the source for the delivery stream, a KinesisStreamSourceConfiguration containing the Kinesis data stream Amazon Resource Name (ARN) and the role ARN for the source stream.

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#redshift_destination_configuration ⇒ Types::RedshiftDestinationConfiguration

The destination in Amazon Redshift. You can specify only one destination.

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#s3_destination_configuration ⇒ Types::S3DestinationConfiguration

[Deprecated] The destination in Amazon S3. You can specify only one destination.

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#splunk_destination_configuration ⇒ Types::SplunkDestinationConfiguration

The destination in Splunk. You can specify only one destination.

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end

#tags ⇒ Array<Types::Tag>

A set of tags to assign to the delivery stream. A tag is a key-value pair that you can define and assign to AWS resources. Tags are metadata. For example, you can add friendly names and descriptions or other types of information that can help you distinguish the delivery stream. For more information about tags, see [Using Cost Allocation Tags] in the AWS Billing and Cost Management User Guide.

You can specify up to 50 tags when creating a delivery stream.

[1]: docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html

Returns:



539
540
541
542
543
544
545
546
547
548
549
550
 
# File 'lib/aws-sdk-firehose/types.rb', line 539

class CreateDeliveryStreamInput < Struct.new(
  :delivery_stream_name,
  :delivery_stream_type,
  :kinesis_stream_source_configuration,
  :s3_destination_configuration,
  :extended_s3_destination_configuration,
  :redshift_destination_configuration,
  :elasticsearch_destination_configuration,
  :splunk_destination_configuration,
  :tags)
  include Aws::Structure
end