Class: Aws::Glue::Types::KinesisStreamingSourceOptions

Inherits:
Struct
  • Object
show all
Includes:
Structure
Defined in:
lib/aws-sdk-glue/types.rb

Overview

Additional options for the Amazon Kinesis streaming data source.

Constant Summary collapse

SENSITIVE =
[]

Instance Attribute Summary collapse

Instance Attribute Details

#add_idle_time_between_readsBoolean

Adds a time delay between two consecutive getRecords operations. The default value is ‘“False”`. This option is only configurable for Glue version 2.0 and above.

Returns:

  • (Boolean)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#add_record_timestampString

When this option is set to ‘true’, the data output will contain an additional column named “_srctimestamp” that indicates the time when the corresponding record received by the stream. The default value is ‘false’. This option is supported in Glue version 4.0 or later.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#avoid_empty_batchesBoolean

Avoids creating an empty microbatch job by checking for unread data in the Kinesis data stream before the batch is started. The default value is ‘“False”`.

Returns:

  • (Boolean)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#classificationString

An optional classification.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#delimiterString

Specifies the delimiter character.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#describe_shard_intervalInteger

The minimum time interval between two ListShards API calls for your script to consider resharding. The default value is ‘1s`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#emit_consumer_lag_metricsString

When this option is set to ‘true’, for each batch, it will emit the metrics for the duration between the oldest record received by the stream and the time it arrives in Glue to CloudWatch. The metric’s name is “glue.driver.streaming.maxConsumerLagInMs”. The default value is ‘false’. This option is supported in Glue version 4.0 or later.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#endpoint_urlString

The URL of the Kinesis endpoint.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#idle_time_between_reads_in_msInteger

The minimum time delay between two consecutive getRecords operations, specified in ms. The default value is ‘1000`. This option is only configurable for Glue version 2.0 and above.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#max_fetch_records_per_shardInteger

The maximum number of records to fetch per shard in the Kinesis data stream per microbatch. Note: The client can exceed this limit if the streaming job has already read extra records from Kinesis (in the same get-records call). If ‘MaxFetchRecordsPerShard` needs to be strict then it needs to be a multiple of `MaxRecordPerRead`. The default value is `100000`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#max_fetch_time_in_msInteger

The maximum time spent for the job executor to read records for the current batch from the Kinesis data stream, specified in milliseconds (ms). Multiple ‘GetRecords` API calls may be made within this time. The default value is `1000`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#max_record_per_readInteger

The maximum number of records to fetch from the Kinesis data stream in each getRecords operation. The default value is ‘10000`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#max_retry_interval_msInteger

The maximum cool-off time period (specified in ms) between two retries of a Kinesis Data Streams API call. The default value is ‘10000`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#num_retriesInteger

The maximum number of retries for Kinesis Data Streams API requests. The default value is ‘3`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#retry_interval_msInteger

The cool-off time period (specified in ms) before retrying the Kinesis Data Streams API call. The default value is ‘1000`.

Returns:

  • (Integer)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#role_arnString

The Amazon Resource Name (ARN) of the role to assume using AWS Security Token Service (AWS STS). This role must have permissions for describe or read record operations for the Kinesis data stream. You must use this parameter when accessing a data stream in a different account. Used in conjunction with ‘“awsSTSSessionName”`.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#role_session_nameString

An identifier for the session assuming the role using AWS STS. You must use this parameter when accessing a data stream in a different account. Used in conjunction with ‘“awsSTSRoleARN”`.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#starting_positionString

The starting position in the Kinesis data stream to read data from. The possible values are ‘“latest”`, `“trim_horizon”`, `“earliest”`, or a timestamp string in UTC format in the pattern `yyyy-mm-ddTHH:MM:SSZ` (where `Z` represents a UTC timezone offset with a +/-. For example: “2023-04-04T08:00:00-04:00”). The default value is `“latest”`.

Note: Using a value that is a timestamp string in UTC format for “startingPosition” is supported only for Glue version 4.0 or later.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#starting_timestampTime

The timestamp of the record in the Kinesis data stream to start reading data from. The possible values are a timestamp string in UTC format of the pattern ‘yyyy-mm-ddTHH:MM:SSZ` (where Z represents a UTC timezone offset with a /-. For example: “2023-04-04T08:00:0008:00”).

Returns:

  • (Time)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#stream_arnString

The Amazon Resource Name (ARN) of the Kinesis data stream.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#stream_nameString

The name of the Kinesis data stream.

Returns:

  • (String)


14944
14945
14946
14947
14948
14949
14950
14951
14952
14953
14954
14955
14956
14957
14958
14959
14960
14961
14962
14963
14964
14965
14966
14967
14968
# File 'lib/aws-sdk-glue/types.rb', line 14944

class KinesisStreamingSourceOptions < Struct.new(
  :endpoint_url,
  :stream_name,
  :classification,
  :delimiter,
  :starting_position,
  :max_fetch_time_in_ms,
  :max_fetch_records_per_shard,
  :max_record_per_read,
  :add_idle_time_between_reads,
  :idle_time_between_reads_in_ms,
  :describe_shard_interval,
  :num_retries,
  :retry_interval_ms,
  :max_retry_interval_ms,
  :avoid_empty_batches,
  :stream_arn,
  :role_arn,
  :role_session_name,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end