Class: Aws::Glue::Types::KafkaStreamingSourceOptions

Inherits:

Struct

Object
Struct
Aws::Glue::Types::KafkaStreamingSourceOptions

show all

Includes:: Structure

Defined in:: lib/aws-sdk-glue/types.rb

Overview

Additional options for streaming.

Constant Summary collapse

SENSITIVE =

[]

Instance Attribute Summary collapse

#add_record_timestamp ⇒ String

When this option is set to ‘true’, the data output will contain an additional column named “_srctimestamp” that indicates the time when the corresponding record received by the topic.
#assign ⇒ String

The specific ‘TopicPartitions` to consume.
#bootstrap_servers ⇒ String

A list of bootstrap server URLs, for example, as ‘b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094`.
#classification ⇒ String

An optional classification.
#connection_name ⇒ String

The name of the connection.
#delimiter ⇒ String

Specifies the delimiter character.
#emit_consumer_lag_metrics ⇒ String

When this option is set to ‘true’, for each batch, it will emit the metrics for the duration between the oldest record received by the topic and the time it arrives in Glue to CloudWatch.
#ending_offsets ⇒ String

The end point when a batch query is ended.
#include_headers ⇒ Boolean

Whether to include the Kafka headers.
#max_offsets_per_trigger ⇒ Integer

The rate limit on the maximum number of offsets that are processed per trigger interval.
#min_partitions ⇒ Integer

The desired minimum number of partitions to read from Kafka.
#num_retries ⇒ Integer

The number of times to retry before failing to fetch Kafka offsets.
#poll_timeout_ms ⇒ Integer

The timeout in milliseconds to poll data from Kafka in Spark job executors.
#retry_interval_ms ⇒ Integer

The time in milliseconds to wait before retrying to fetch Kafka offsets.
#security_protocol ⇒ String

The protocol used to communicate with brokers.
#starting_offsets ⇒ String

The starting position in the Kafka topic to read data from.
#starting_timestamp ⇒ Time

The timestamp of the record in the Kafka topic to start reading data from.
#subscribe_pattern ⇒ String

A Java regex string that identifies the topic list to subscribe to.
#topic_name ⇒ String

The topic name as specified in Apache Kafka.

Instance Attribute Details

#add_record_timestamp ⇒ `String`

When this option is set to ‘true’, the data output will contain an additional column named “_srctimestamp” that indicates the time when the corresponding record received by the topic. The default value is ‘false’. This option is supported in Glue version 4.0 or later.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#assign ⇒ `String`

The specific ‘TopicPartitions` to consume. You must specify at least one of `“topicName”`, `“assign”` or `“subscribePattern”`.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#bootstrap_servers ⇒ `String`

A list of bootstrap server URLs, for example, as ‘b-1.vpc-test-2.o4q88o.c6.kafka.us-east-1.amazonaws.com:9094`. This option must be specified in the API call or defined in the table metadata in the Data Catalog.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#classification ⇒ `String`

An optional classification.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#connection_name ⇒ `String`

The name of the connection.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#delimiter ⇒ `String`

Specifies the delimiter character.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#emit_consumer_lag_metrics ⇒ `String`

When this option is set to ‘true’, for each batch, it will emit the metrics for the duration between the oldest record received by the topic and the time it arrives in Glue to CloudWatch. The metric’s name is “glue.driver.streaming.maxConsumerLagInMs”. The default value is ‘false’. This option is supported in Glue version 4.0 or later.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#ending_offsets ⇒ `String`

The end point when a batch query is ended. Possible values are either ‘“latest”` or a JSON string that specifies an ending offset for each `TopicPartition`.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#include_headers ⇒ `Boolean`

Whether to include the Kafka headers. When the option is set to “true”, the data output will contain an additional column named “glue_streaming_kafka_headers” with type ‘Array[Struct(key: String, value: String)]`. The default value is “false”. This option is available in Glue version 3.0 or later only.

Returns:

(Boolean)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#max_offsets_per_trigger ⇒ `Integer`

The rate limit on the maximum number of offsets that are processed per trigger interval. The specified total number of offsets is proportionally split across ‘topicPartitions` of different volumes. The default value is null, which means that the consumer reads all offsets until the known latest offset.

Returns:

(Integer)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#min_partitions ⇒ `Integer`

The desired minimum number of partitions to read from Kafka. The default value is null, which means that the number of spark partitions is equal to the number of Kafka partitions.

Returns:

(Integer)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#num_retries ⇒ `Integer`

The number of times to retry before failing to fetch Kafka offsets. The default value is ‘3`.

Returns:

(Integer)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#poll_timeout_ms ⇒ `Integer`

The timeout in milliseconds to poll data from Kafka in Spark job executors. The default value is ‘512`.

Returns:

(Integer)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#retry_interval_ms ⇒ `Integer`

The time in milliseconds to wait before retrying to fetch Kafka offsets. The default value is ‘10`.

Returns:

(Integer)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#security_protocol ⇒ `String`

The protocol used to communicate with brokers. The possible values are ‘“SSL”` or `“PLAINTEXT”`.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#starting_offsets ⇒ `String`

The starting position in the Kafka topic to read data from. The possible values are ‘“earliest”` or `“latest”`. The default value is `“latest”`.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#starting_timestamp ⇒ `Time`

The timestamp of the record in the Kafka topic to start reading data from. The possible values are a timestamp string in UTC format of the pattern ‘yyyy-mm-ddTHH:MM:SSZ` (where Z represents a UTC timezone offset with a /-. For example: “2023-04-04T08:00:0008:00”).

Only one of ‘StartingTimestamp` or `StartingOffsets` must be set.

Returns:

(Time)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#subscribe_pattern ⇒ `String`

A Java regex string that identifies the topic list to subscribe to. You must specify at least one of ‘“topicName”`, `“assign”` or `“subscribePattern”`.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

#topic_name ⇒ `String`

The topic name as specified in Apache Kafka. You must specify at least one of ‘“topicName”`, `“assign”` or `“subscribePattern”`.

Returns:

(String)

# File 'lib/aws-sdk-glue/types.rb', line 14346

class KafkaStreamingSourceOptions < Struct.new(
  :bootstrap_servers,
  :security_protocol,
  :connection_name,
  :topic_name,
  :assign,
  :subscribe_pattern,
  :classification,
  :delimiter,
  :starting_offsets,
  :ending_offsets,
  :poll_timeout_ms,
  :num_retries,
  :retry_interval_ms,
  :max_offsets_per_trigger,
  :min_partitions,
  :include_headers,
  :add_record_timestamp,
  :emit_consumer_lag_metrics,
  :starting_timestamp)
  SENSITIVE = []
  include Aws::Structure
end

Class: Aws::Glue::Types::KafkaStreamingSourceOptions

Overview

Constant Summary collapse

Instance Attribute Summary collapse

Instance Attribute Details

#add_record_timestamp ⇒ String

#assign ⇒ String

#bootstrap_servers ⇒ String

#classification ⇒ String

#connection_name ⇒ String

#delimiter ⇒ String

#emit_consumer_lag_metrics ⇒ String

#ending_offsets ⇒ String

#include_headers ⇒ Boolean

#max_offsets_per_trigger ⇒ Integer

#min_partitions ⇒ Integer

#num_retries ⇒ Integer

#poll_timeout_ms ⇒ Integer

#retry_interval_ms ⇒ Integer

#security_protocol ⇒ String

#starting_offsets ⇒ String

#starting_timestamp ⇒ Time

#subscribe_pattern ⇒ String

#topic_name ⇒ String

#add_record_timestamp ⇒ `String`

#assign ⇒ `String`

#bootstrap_servers ⇒ `String`

#classification ⇒ `String`

#connection_name ⇒ `String`

#delimiter ⇒ `String`

#emit_consumer_lag_metrics ⇒ `String`

#ending_offsets ⇒ `String`

#include_headers ⇒ `Boolean`

#max_offsets_per_trigger ⇒ `Integer`

#min_partitions ⇒ `Integer`

#num_retries ⇒ `Integer`

#poll_timeout_ms ⇒ `Integer`

#retry_interval_ms ⇒ `Integer`

#security_protocol ⇒ `String`

#starting_offsets ⇒ `String`

#starting_timestamp ⇒ `Time`

#subscribe_pattern ⇒ `String`

#topic_name ⇒ `String`