Class: Google::Cloud::Dataflow::V1beta3::WorkerPool

Inherits:
Object
  • Object
show all
Extended by:
Protobuf::MessageExts::ClassMethods
Includes:
Protobuf::MessageExts
Defined in:
proto_docs/google/dataflow/v1beta3/environment.rb

Overview

Describes one particular pool of Cloud Dataflow workers to be instantiated by the Cloud Dataflow service in order to perform the computations required by a job. Note that a workflow job may use multiple pools, in order to match the various computational requirements of the various stages of the job.

Defined Under Namespace

Classes: MetadataEntry

Instance Attribute Summary collapse

Instance Attribute Details

#autoscaling_settings ⇒ ::Google::Cloud::Dataflow::V1beta3::AutoscalingSettings

Returns Settings for autoscaling of this WorkerPool.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#data_disks ⇒ ::Array<::Google::Cloud::Dataflow::V1beta3::Disk>

Returns Data disks that are used by a VM in this workflow.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#default_package_set ⇒ ::Google::Cloud::Dataflow::V1beta3::DefaultPackageSet

Returns The default package set to install. This allows the service to select a default set of packages which are useful to worker harnesses written in a particular language.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#disk_size_gb ⇒ ::Integer

Returns Size of root disk for VMs, in GB. If zero or unspecified, the service will attempt to choose a reasonable default.

Returns:

  • (::Integer) —

    Size of root disk for VMs, in GB. If zero or unspecified, the service will attempt to choose a reasonable default.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#disk_source_image ⇒ ::String

Returns Fully qualified source image for disks.

Returns:

  • (::String) —

    Fully qualified source image for disks.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#disk_type ⇒ ::String

Returns Type of root disk for VMs. If empty or unspecified, the service will attempt to choose a reasonable default.

Returns:

  • (::String) —

    Type of root disk for VMs. If empty or unspecified, the service will attempt to choose a reasonable default.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#ip_configuration ⇒ ::Google::Cloud::Dataflow::V1beta3::WorkerIPAddressConfiguration

Returns Configuration for VM IPs.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#kind ⇒ ::String

Returns The kind of the worker pool; currently only harness and shuffle are supported.

Returns:

  • (::String) —

    The kind of the worker pool; currently only harness and shuffle are supported.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#machine_type ⇒ ::String

Returns Machine type (e.g. "n1-standard-1"). If empty or unspecified, the service will attempt to choose a reasonable default.

Returns:

  • (::String) —

    Machine type (e.g. "n1-standard-1"). If empty or unspecified, the service will attempt to choose a reasonable default.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#metadata ⇒ ::Google::Protobuf::Map{::String => ::String}

Returns Metadata to set on the Google Compute Engine VMs.

Returns:

  • (::Google::Protobuf::Map{::String => ::String}) —

    Metadata to set on the Google Compute Engine VMs.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#network ⇒ ::String

Returns Network to which VMs will be assigned. If empty or unspecified, the service will use the network "default".

Returns:

  • (::String) —

    Network to which VMs will be assigned. If empty or unspecified, the service will use the network "default".



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#num_threads_per_worker ⇒ ::Integer

Returns The number of threads per worker harness. If empty or unspecified, the service will choose a number of threads (according to the number of cores on the selected machine type for batch, or 1 by convention for streaming).

Returns:

  • (::Integer) —

    The number of threads per worker harness. If empty or unspecified, the service will choose a number of threads (according to the number of cores on the selected machine type for batch, or 1 by convention for streaming).



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#num_workers ⇒ ::Integer

Returns Number of Google Compute Engine workers in this pool needed to execute the job. If zero or unspecified, the service will attempt to choose a reasonable default.

Returns:

  • (::Integer) —

    Number of Google Compute Engine workers in this pool needed to execute the job. If zero or unspecified, the service will attempt to choose a reasonable default.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#on_host_maintenance ⇒ ::String

Returns The action to take on host maintenance, as defined by the Google Compute Engine API.

Returns:

  • (::String) —

    The action to take on host maintenance, as defined by the Google Compute Engine API.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#packages ⇒ ::Array<::Google::Cloud::Dataflow::V1beta3::Package>

Returns Packages to be installed on workers.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#pool_args ⇒ ::Google::Protobuf::Any

Returns Extra arguments for this worker pool.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#sdk_harness_container_images ⇒ ::Array<::Google::Cloud::Dataflow::V1beta3::SdkHarnessContainerImage>

Returns Set of SDK harness containers needed to execute this pipeline. This will only be set in the Fn API path. For non-cross-language pipelines this should have only one entry. Cross-language pipelines will have two or more entries.

Returns:

  • (::Array<::Google::Cloud::Dataflow::V1beta3::SdkHarnessContainerImage>) —

    Set of SDK harness containers needed to execute this pipeline. This will only be set in the Fn API path. For non-cross-language pipelines this should have only one entry. Cross-language pipelines will have two or more entries.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#subnetwork ⇒ ::String

Returns Subnetwork to which VMs will be assigned, if desired. Expected to be of the form "regions/REGION/subnetworks/SUBNETWORK".

Returns:

  • (::String) —

    Subnetwork to which VMs will be assigned, if desired. Expected to be of the form "regions/REGION/subnetworks/SUBNETWORK".



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#taskrunner_settings ⇒ ::Google::Cloud::Dataflow::V1beta3::TaskRunnerSettings

Returns Settings passed through to Google Compute Engine workers when using the standard Dataflow task runner. Users should ignore this field.

Returns:



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#teardown_policy ⇒ ::Google::Cloud::Dataflow::V1beta3::TeardownPolicy

Returns Sets the policy for determining when to turndown worker pool. Allowed values are: TEARDOWN_ALWAYS, TEARDOWN_ON_SUCCESS, and TEARDOWN_NEVER. TEARDOWN_ALWAYS means workers are always torn down regardless of whether the job succeeds. TEARDOWN_ON_SUCCESS means workers are torn down if the job succeeds. TEARDOWN_NEVER means the workers are never torn down.

If the workers are not torn down by the service, they will continue to run and use Google Compute Engine VM resources in the user's project until they are explicitly terminated by the user. Because of this, Google recommends using the TEARDOWN_ALWAYS policy except for small, manually supervised test jobs.

If unknown or unspecified, the service will attempt to choose a reasonable default.

Returns:

  • (::Google::Cloud::Dataflow::V1beta3::TeardownPolicy) —

    Sets the policy for determining when to turndown worker pool. Allowed values are: TEARDOWN_ALWAYS, TEARDOWN_ON_SUCCESS, and TEARDOWN_NEVER. TEARDOWN_ALWAYS means workers are always torn down regardless of whether the job succeeds. TEARDOWN_ON_SUCCESS means workers are torn down if the job succeeds. TEARDOWN_NEVER means the workers are never torn down.

    If the workers are not torn down by the service, they will continue to run and use Google Compute Engine VM resources in the user's project until they are explicitly terminated by the user. Because of this, Google recommends using the TEARDOWN_ALWAYS policy except for small, manually supervised test jobs.

    If unknown or unspecified, the service will attempt to choose a reasonable default.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#worker_harness_container_image ⇒ ::String

Returns Required. Docker container image that executes the Cloud Dataflow worker harness, residing in Google Container Registry.

Deprecated for the Fn API path. Use sdk_harness_container_images instead.

Returns:

  • (::String) —

    Required. Docker container image that executes the Cloud Dataflow worker harness, residing in Google Container Registry.

    Deprecated for the Fn API path. Use sdk_harness_container_images instead.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end

#zone ⇒ ::String

Returns Zone to run the worker pools in. If empty or unspecified, the service will attempt to choose a reasonable default.

Returns:

  • (::String) —

    Zone to run the worker pools in. If empty or unspecified, the service will attempt to choose a reasonable default.



456
457
458
459
460
461
462
463
464
465
466
467
468
 
# File 'proto_docs/google/dataflow/v1beta3/environment.rb', line 456

class WorkerPool
  include ::Google::Protobuf::MessageExts
  extend ::Google::Protobuf::MessageExts::ClassMethods

  # @!attribute [rw] key
  #   @return [::String]
  # @!attribute [rw] value
  #   @return [::String]
  class MetadataEntry
    include ::Google::Protobuf::MessageExts
    extend ::Google::Protobuf::MessageExts::ClassMethods
  end
end