Class: Aws::Firehose::Types::ExtendedS3DestinationConfiguration

Inherits:
Struct
  • Object
show all
Includes:
Structure
Defined in:
lib/aws-sdk-firehose/types.rb

Overview

Note:

When making an API call, you may pass ExtendedS3DestinationConfiguration data as a hash:

{
  role_arn: "RoleARN", # required
  bucket_arn: "BucketARN", # required
  prefix: "Prefix",
  error_output_prefix: "ErrorOutputPrefix",
  buffering_hints: {
    size_in_m_bs: 1,
    interval_in_seconds: 1,
  },
  compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
  encryption_configuration: {
    no_encryption_config: "NoEncryption", # accepts NoEncryption
    kms_encryption_config: {
      awskms_key_arn: "AWSKMSKeyARN", # required
    },
  },
  cloud_watch_logging_options: {
    enabled: false,
    log_group_name: "LogGroupName",
    log_stream_name: "LogStreamName",
  },
  processing_configuration: {
    enabled: false,
    processors: [
      {
        type: "Lambda", # required, accepts Lambda
        parameters: [
          {
            parameter_name: "LambdaArn", # required, accepts LambdaArn, NumberOfRetries, RoleArn, BufferSizeInMBs, BufferIntervalInSeconds
            parameter_value: "ProcessorParameterValue", # required
          },
        ],
      },
    ],
  },
  s3_backup_mode: "Disabled", # accepts Disabled, Enabled
  s3_backup_configuration: {
    role_arn: "RoleARN", # required
    bucket_arn: "BucketARN", # required
    prefix: "Prefix",
    error_output_prefix: "ErrorOutputPrefix",
    buffering_hints: {
      size_in_m_bs: 1,
      interval_in_seconds: 1,
    },
    compression_format: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, ZIP, Snappy
    encryption_configuration: {
      no_encryption_config: "NoEncryption", # accepts NoEncryption
      kms_encryption_config: {
        awskms_key_arn: "AWSKMSKeyARN", # required
      },
    },
    cloud_watch_logging_options: {
      enabled: false,
      log_group_name: "LogGroupName",
      log_stream_name: "LogStreamName",
    },
  },
  data_format_conversion_configuration: {
    schema_configuration: {
      role_arn: "NonEmptyStringWithoutWhitespace",
      catalog_id: "NonEmptyStringWithoutWhitespace",
      database_name: "NonEmptyStringWithoutWhitespace",
      table_name: "NonEmptyStringWithoutWhitespace",
      region: "NonEmptyStringWithoutWhitespace",
      version_id: "NonEmptyStringWithoutWhitespace",
    },
    input_format_configuration: {
      deserializer: {
        open_x_json_ser_de: {
          convert_dots_in_json_keys_to_underscores: false,
          case_insensitive: false,
          column_to_json_key_mappings: {
            "NonEmptyStringWithoutWhitespace" => "NonEmptyString",
          },
        },
        hive_json_ser_de: {
          timestamp_formats: ["NonEmptyString"],
        },
      },
    },
    output_format_configuration: {
      serializer: {
        parquet_ser_de: {
          block_size_bytes: 1,
          page_size_bytes: 1,
          compression: "UNCOMPRESSED", # accepts UNCOMPRESSED, GZIP, SNAPPY
          enable_dictionary_compression: false,
          max_padding_bytes: 1,
          writer_version: "V1", # accepts V1, V2
        },
        orc_ser_de: {
          stripe_size_bytes: 1,
          block_size_bytes: 1,
          row_index_stride: 1,
          enable_padding: false,
          padding_tolerance: 1.0,
          compression: "NONE", # accepts NONE, ZLIB, SNAPPY
          bloom_filter_columns: ["NonEmptyStringWithoutWhitespace"],
          bloom_filter_false_positive_probability: 1.0,
          dictionary_key_threshold: 1.0,
          format_version: "V0_11", # accepts V0_11, V0_12
        },
      },
    },
    enabled: false,
  },
}

Describes the configuration of a destination in Amazon S3.

See Also:

Instance Attribute Summary collapse

Instance Attribute Details

#bucket_arn ⇒ String

The ARN of the S3 bucket. For more information, see [Amazon Resource Names (ARNs) and AWS Service Namespaces].

[1]: docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html

Returns:

  • (String) 


1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#buffering_hints ⇒ Types::BufferingHints

The buffering option.

Returns:



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#cloud_watch_logging_options ⇒ Types::CloudWatchLoggingOptions

The Amazon CloudWatch logging options for your delivery stream.

Returns:



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#compression_format ⇒ String

The compression format. If no value is specified, the default is UNCOMPRESSED.

Returns:

  • (String) 


1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#data_format_conversion_configuration ⇒ Types::DataFormatConversionConfiguration

The serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3.

Returns:



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#encryption_configuration ⇒ Types::EncryptionConfiguration

The encryption configuration. If no value is specified, the default is no encryption.

Returns:



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#error_output_prefix ⇒ String

A prefix that Kinesis Data Firehose evaluates and adds to failed records before writing them to S3. This prefix appears immediately following the bucket name.

Returns:

  • (String) 


1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#prefix ⇒ String

The “YYYY/MM/DD/HH” time format prefix is automatically used for delivered Amazon S3 files. You can specify an extra prefix to be added in front of the time format prefix. If the prefix ends with a slash, it appears as a folder in the S3 bucket. For more information, see [Amazon S3 Object Name Format] in the *Amazon Kinesis Data Firehose Developer Guide*.

[1]: docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#s3-object-name

Returns:

  • (String) 


1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#processing_configuration ⇒ Types::ProcessingConfiguration

The data processing configuration.

Returns:



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#role_arn ⇒ String

The Amazon Resource Name (ARN) of the AWS credentials. For more information, see [Amazon Resource Names (ARNs) and AWS Service Namespaces].

[1]: docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html

Returns:

  • (String) 


1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#s3_backup_configuration ⇒ Types::S3DestinationConfiguration

The configuration for backup in Amazon S3.

Returns:



1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end

#s3_backup_mode ⇒ String

The Amazon S3 backup mode.

Returns:

  • (String) 


1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
 
# File 'lib/aws-sdk-firehose/types.rb', line 1575

class ExtendedS3DestinationConfiguration < Struct.new(
  :role_arn,
  :bucket_arn,
  :prefix,
  :error_output_prefix,
  :buffering_hints,
  :compression_format,
  :encryption_configuration,
  :cloud_watch_logging_options,
  :processing_configuration,
  :s3_backup_mode,
  :s3_backup_configuration,
  :data_format_conversion_configuration)
  include Aws::Structure
end