lib/aws-sdk-glue/client.rb in aws-sdk-glue-1.138.0 vs lib/aws-sdk-glue/client.rb in aws-sdk-glue-1.139.0
- old
+ new
@@ -958,11 +958,11 @@
# resp.dev_endpoints[0].yarn_endpoint_address #=> String
# resp.dev_endpoints[0].private_address #=> String
# resp.dev_endpoints[0].zeppelin_remote_spark_interpreter_port #=> Integer
# resp.dev_endpoints[0].public_address #=> String
# resp.dev_endpoints[0].status #=> String
- # resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.dev_endpoints[0].glue_version #=> String
# resp.dev_endpoints[0].number_of_workers #=> Integer
# resp.dev_endpoints[0].number_of_nodes #=> Integer
# resp.dev_endpoints[0].availability_zone #=> String
# resp.dev_endpoints[0].vpc_id #=> String
@@ -1022,21 +1022,22 @@
# resp.jobs[0].last_modified_on #=> Time
# resp.jobs[0].execution_property.max_concurrent_runs #=> Integer
# resp.jobs[0].command.name #=> String
# resp.jobs[0].command.script_location #=> String
# resp.jobs[0].command.python_version #=> String
+ # resp.jobs[0].command.runtime #=> String
# resp.jobs[0].default_arguments #=> Hash
# resp.jobs[0].default_arguments["GenericString"] #=> String
# resp.jobs[0].non_overridable_arguments #=> Hash
# resp.jobs[0].non_overridable_arguments["GenericString"] #=> String
# resp.jobs[0].connections.connections #=> Array
# resp.jobs[0].connections.connections[0] #=> String
# resp.jobs[0].max_retries #=> Integer
# resp.jobs[0].allocated_capacity #=> Integer
# resp.jobs[0].timeout #=> Integer
# resp.jobs[0].max_capacity #=> Float
- # resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.jobs[0].number_of_workers #=> Integer
# resp.jobs[0].security_configuration #=> String
# resp.jobs[0].notification_property.notify_delay_after #=> Integer
# resp.jobs[0].glue_version #=> String
# resp.jobs[0].code_gen_configuration_nodes #=> Hash
@@ -2065,11 +2066,11 @@
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].allocated_capacity #=> Integer
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
- # resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.workflows[0].last_run.graph.nodes[0].job_details.job_runs[0].glue_version #=> String
@@ -2133,11 +2134,11 @@
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].allocated_capacity #=> Integer
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
- # resp.workflows[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.workflows[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.workflows[0].graph.nodes[0].job_details.job_runs[0].glue_version #=> String
@@ -3110,11 +3111,11 @@
# security_group_ids: ["GenericString"],
# subnet_id: "GenericString",
# public_key: "GenericString",
# public_keys: ["GenericString"],
# number_of_nodes: 1,
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X, Z.2X
# glue_version: "GlueVersionString",
# number_of_workers: 1,
# extra_python_libs_s3_path: "GenericString",
# extra_jars_s3_path: "GenericString",
# security_configuration: "NameString",
@@ -3135,11 +3136,11 @@
# resp.subnet_id #=> String
# resp.role_arn #=> String
# resp.yarn_endpoint_address #=> String
# resp.zeppelin_remote_spark_interpreter_port #=> Integer
# resp.number_of_nodes #=> Integer
- # resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.glue_version #=> String
# resp.number_of_workers #=> Integer
# resp.availability_zone #=> String
# resp.vpc_id #=> String
# resp.extra_python_libs_s3_path #=> String
@@ -3181,11 +3182,12 @@
#
# @option params [required, Types::JobCommand] :command
# The `JobCommand` that runs this job.
#
# @option params [Hash<String,String>] :default_arguments
- # The default arguments for this job.
+ # The default arguments for every run of this job, specified as
+ # name-value pairs.
#
# You can specify arguments here that your own job-execution script
# consumes, as well as arguments that Glue itself consumes.
#
# Job arguments may be logged. Do not pass plaintext secrets as
@@ -3195,21 +3197,27 @@
#
# For information about how to specify and consume your own Job
# arguments, see the [Calling Glue APIs in Python][1] topic in the
# developer guide.
#
- # For information about the key-value pairs that Glue consumes to set up
- # your job, see the [Special Parameters Used by Glue][2] topic in the
+ # For information about the arguments you can provide to this field when
+ # configuring Spark jobs, see the [Special Parameters Used by Glue][2]
+ # topic in the developer guide.
+ #
+ # For information about the arguments you can provide to this field when
+ # configuring Ray jobs, see [Using job parameters in Ray jobs][3] in the
# developer guide.
#
#
#
# [1]: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html
# [2]: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+ # [3]: https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html
#
# @option params [Hash<String,String>] :non_overridable_arguments
- # Non-overridable arguments for this job, specified as name-value pairs.
+ # Arguments for this job that are not overridden when providing job
+ # arguments in a job run, specified as name-value pairs.
#
# @option params [Types::ConnectionsList] :connections
# The connections used for this job.
#
# @option params [Integer] :max_retries
@@ -3236,33 +3244,34 @@
# @option params [Float] :max_capacity
# For Glue version 1.0 or earlier jobs, using the standard worker type,
# the number of Glue data processing units (DPUs) that can be allocated
# when this job runs. A DPU is a relative measure of processing power
# that consists of 4 vCPUs of compute capacity and 16 GB of memory. For
- # more information, see the [Glue pricing page][1].
+ # more information, see the [ Glue pricing page][1].
#
- # Do not set `Max Capacity` if using `WorkerType` and `NumberOfWorkers`.
+ # For Glue version 2.0+ jobs, you cannot specify a `Maximum capacity`.
+ # Instead, you should specify a `Worker type` and the `Number of
+ # workers`.
#
+ # Do not set `MaxCapacity` if using `WorkerType` and `NumberOfWorkers`.
+ #
# The value that can be allocated for `MaxCapacity` depends on whether
- # you are running a Python shell job or an Apache Spark ETL job:
+ # you are running a Python shell job, an Apache Spark ETL job, or an
+ # Apache Spark streaming ETL job:
#
# * When you specify a Python shell job
# (`JobCommand.Name`="pythonshell"), you can allocate either 0.0625
# or 1 DPU. The default is 0.0625 DPU.
#
# * When you specify an Apache Spark ETL job
# (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
- # (`JobCommand.Name`="gluestreaming"), you can allocate a minimum of
- # 2 DPUs. The default is 10 DPUs. This job type cannot have a
+ # (`JobCommand.Name`="gluestreaming"), you can allocate from 2 to
+ # 100 DPUs. The default is 10 DPUs. This job type cannot have a
# fractional DPU allocation.
#
- # For Glue version 2.0 jobs, you cannot instead specify a `Maximum
- # capacity`. Instead, you should specify a `Worker type` and the `Number
- # of workers`.
#
#
- #
# [1]: https://aws.amazon.com/glue/pricing/
#
# @option params [String] :security_configuration
# The name of the `SecurityConfiguration` structure to be used with this
# job.
@@ -3278,14 +3287,18 @@
#
# @option params [Types::NotificationProperty] :notification_property
# Specifies configuration properties of a job notification.
#
# @option params [String] :glue_version
- # Glue version determines the versions of Apache Spark and Python that
- # Glue supports. The Python version indicates the version supported for
- # jobs of type Spark.
+ # In Spark jobs, `GlueVersion` determines the versions of Apache Spark
+ # and Python that Glue available in a job. The Python version indicates
+ # the version supported for jobs of type Spark.
#
+ # Ray jobs should set `GlueVersion` to `4.0` or greater. However, the
+ # versions of Ray, Python and additional libraries available in your Ray
+ # job are determined by the `Runtime` parameter of the Job command.
+ #
# For more information about the available Glue versions and
# corresponding Spark and Python versions, see [Glue version][1] in the
# developer guide.
#
# Jobs that are created without specifying a Glue version default to
@@ -3299,11 +3312,12 @@
# The number of workers of a defined `workerType` that are allocated
# when a job runs.
#
# @option params [String] :worker_type
# The type of predefined worker that is allocated when a job runs.
- # Accepts a value of Standard, G.1X, G.2X, or G.025X.
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
+ # Accepts the value Z.2X for Ray jobs.
#
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
# of memory and a 50GB disk, and 2 executors per worker.
#
# * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16 GB
@@ -3317,10 +3331,14 @@
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPU,
# 4 GB of memory, 64 GB disk), and provides 1 executor per worker. We
# recommend this worker type for low volume streaming jobs. This
# worker type is only available for Glue version 3.0 streaming jobs.
#
+ # * For the `Z.2X` worker type, each worker maps to 2 M-DPU (8vCPU, 64
+ # GB of m emory, 128 GB disk), and provides up to 8 Ray workers based
+ # on the autoscaler.
+ #
# @option params [Hash<String,Types::CodeGenConfigurationNode>] :code_gen_configuration_nodes
# The representation of a directed acyclic graph on which both the Glue
# Studio visual component and Glue Studio code generation is based.
#
# @option params [String] :execution_class
@@ -3537,11 +3555,11 @@
# },
# },
# role: "RoleString", # required
# glue_version: "GlueVersionString",
# max_capacity: 1.0,
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X, Z.2X
# number_of_workers: 1,
# timeout: 1,
# max_retries: 1,
# tags: {
# "TagKey" => "TagValue",
@@ -4111,11 +4129,11 @@
# connections: {
# connections: ["GenericString"],
# },
# max_capacity: 1.0,
# number_of_workers: 1,
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X, Z.2X
# security_configuration: "NameString",
# glue_version: "GlueVersionString",
# tags: {
# "TagKey" => "TagValue",
# },
@@ -6636,11 +6654,11 @@
# resp.dev_endpoint.yarn_endpoint_address #=> String
# resp.dev_endpoint.private_address #=> String
# resp.dev_endpoint.zeppelin_remote_spark_interpreter_port #=> Integer
# resp.dev_endpoint.public_address #=> String
# resp.dev_endpoint.status #=> String
- # resp.dev_endpoint.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.dev_endpoint.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.dev_endpoint.glue_version #=> String
# resp.dev_endpoint.number_of_workers #=> Integer
# resp.dev_endpoint.number_of_nodes #=> Integer
# resp.dev_endpoint.availability_zone #=> String
# resp.dev_endpoint.vpc_id #=> String
@@ -6707,11 +6725,11 @@
# resp.dev_endpoints[0].yarn_endpoint_address #=> String
# resp.dev_endpoints[0].private_address #=> String
# resp.dev_endpoints[0].zeppelin_remote_spark_interpreter_port #=> Integer
# resp.dev_endpoints[0].public_address #=> String
# resp.dev_endpoints[0].status #=> String
- # resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.dev_endpoints[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.dev_endpoints[0].glue_version #=> String
# resp.dev_endpoints[0].number_of_workers #=> Integer
# resp.dev_endpoints[0].number_of_nodes #=> Integer
# resp.dev_endpoints[0].availability_zone #=> String
# resp.dev_endpoints[0].vpc_id #=> String
@@ -6763,21 +6781,22 @@
# resp.job.last_modified_on #=> Time
# resp.job.execution_property.max_concurrent_runs #=> Integer
# resp.job.command.name #=> String
# resp.job.command.script_location #=> String
# resp.job.command.python_version #=> String
+ # resp.job.command.runtime #=> String
# resp.job.default_arguments #=> Hash
# resp.job.default_arguments["GenericString"] #=> String
# resp.job.non_overridable_arguments #=> Hash
# resp.job.non_overridable_arguments["GenericString"] #=> String
# resp.job.connections.connections #=> Array
# resp.job.connections.connections[0] #=> String
# resp.job.max_retries #=> Integer
# resp.job.allocated_capacity #=> Integer
# resp.job.timeout #=> Integer
# resp.job.max_capacity #=> Float
- # resp.job.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.job.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.job.number_of_workers #=> Integer
# resp.job.security_configuration #=> String
# resp.job.notification_property.notify_delay_after #=> Integer
# resp.job.glue_version #=> String
# resp.job.code_gen_configuration_nodes #=> Hash
@@ -7644,11 +7663,11 @@
# resp.job_run.predecessor_runs[0].run_id #=> String
# resp.job_run.allocated_capacity #=> Integer
# resp.job_run.execution_time #=> Integer
# resp.job_run.timeout #=> Integer
# resp.job_run.max_capacity #=> Float
- # resp.job_run.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.job_run.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.job_run.number_of_workers #=> Integer
# resp.job_run.security_configuration #=> String
# resp.job_run.log_group_name #=> String
# resp.job_run.notification_property.notify_delay_after #=> Integer
# resp.job_run.glue_version #=> String
@@ -7710,11 +7729,11 @@
# resp.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.job_runs[0].allocated_capacity #=> Integer
# resp.job_runs[0].execution_time #=> Integer
# resp.job_runs[0].timeout #=> Integer
# resp.job_runs[0].max_capacity #=> Float
- # resp.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.job_runs[0].number_of_workers #=> Integer
# resp.job_runs[0].security_configuration #=> String
# resp.job_runs[0].log_group_name #=> String
# resp.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.job_runs[0].glue_version #=> String
@@ -7764,21 +7783,22 @@
# resp.jobs[0].last_modified_on #=> Time
# resp.jobs[0].execution_property.max_concurrent_runs #=> Integer
# resp.jobs[0].command.name #=> String
# resp.jobs[0].command.script_location #=> String
# resp.jobs[0].command.python_version #=> String
+ # resp.jobs[0].command.runtime #=> String
# resp.jobs[0].default_arguments #=> Hash
# resp.jobs[0].default_arguments["GenericString"] #=> String
# resp.jobs[0].non_overridable_arguments #=> Hash
# resp.jobs[0].non_overridable_arguments["GenericString"] #=> String
# resp.jobs[0].connections.connections #=> Array
# resp.jobs[0].connections.connections[0] #=> String
# resp.jobs[0].max_retries #=> Integer
# resp.jobs[0].allocated_capacity #=> Integer
# resp.jobs[0].timeout #=> Integer
# resp.jobs[0].max_capacity #=> Float
- # resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.jobs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.jobs[0].number_of_workers #=> Integer
# resp.jobs[0].security_configuration #=> String
# resp.jobs[0].notification_property.notify_delay_after #=> Integer
# resp.jobs[0].glue_version #=> String
# resp.jobs[0].code_gen_configuration_nodes #=> Hash
@@ -8771,11 +8791,11 @@
# resp.schema[0].name #=> String
# resp.schema[0].data_type #=> String
# resp.role #=> String
# resp.glue_version #=> String
# resp.max_capacity #=> Float
- # resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.number_of_workers #=> Integer
# resp.timeout #=> Integer
# resp.max_retries #=> Integer
# resp.transform_encryption.ml_user_data_encryption.ml_user_data_encryption_mode #=> String, one of "DISABLED", "SSE-KMS"
# resp.transform_encryption.ml_user_data_encryption.kms_key_id #=> String
@@ -8881,11 +8901,11 @@
# resp.transforms[0].schema[0].name #=> String
# resp.transforms[0].schema[0].data_type #=> String
# resp.transforms[0].role #=> String
# resp.transforms[0].glue_version #=> String
# resp.transforms[0].max_capacity #=> Float
- # resp.transforms[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.transforms[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.transforms[0].number_of_workers #=> Integer
# resp.transforms[0].timeout #=> Integer
# resp.transforms[0].max_retries #=> Integer
# resp.transforms[0].transform_encryption.ml_user_data_encryption.ml_user_data_encryption_mode #=> String, one of "DISABLED", "SSE-KMS"
# resp.transforms[0].transform_encryption.ml_user_data_encryption.kms_key_id #=> String
@@ -11247,11 +11267,11 @@
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].allocated_capacity #=> Integer
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
- # resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.workflow.last_run.graph.nodes[0].job_details.job_runs[0].glue_version #=> String
@@ -11315,11 +11335,11 @@
# resp.workflow.graph.nodes[0].job_details.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.workflow.graph.nodes[0].job_details.job_runs[0].allocated_capacity #=> Integer
# resp.workflow.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
# resp.workflow.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
# resp.workflow.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
- # resp.workflow.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.workflow.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.workflow.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
# resp.workflow.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
# resp.workflow.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
# resp.workflow.graph.nodes[0].job_details.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.workflow.graph.nodes[0].job_details.job_runs[0].glue_version #=> String
@@ -11436,11 +11456,11 @@
# resp.run.graph.nodes[0].job_details.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.run.graph.nodes[0].job_details.job_runs[0].allocated_capacity #=> Integer
# resp.run.graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
# resp.run.graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
# resp.run.graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
- # resp.run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.run.graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.run.graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
# resp.run.graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
# resp.run.graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
# resp.run.graph.nodes[0].job_details.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.run.graph.nodes[0].job_details.job_runs[0].glue_version #=> String
@@ -11597,11 +11617,11 @@
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].predecessor_runs[0].run_id #=> String
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].allocated_capacity #=> Integer
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].execution_time #=> Integer
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].timeout #=> Integer
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].max_capacity #=> Float
- # resp.runs[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X"
+ # resp.runs[0].graph.nodes[0].job_details.job_runs[0].worker_type #=> String, one of "Standard", "G.1X", "G.2X", "G.025X", "G.4X", "G.8X", "Z.2X"
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].number_of_workers #=> Integer
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].security_configuration #=> String
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].log_group_name #=> String
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].notification_property.notify_delay_after #=> Integer
# resp.runs[0].graph.nodes[0].job_details.job_runs[0].glue_version #=> String
@@ -13738,11 +13758,11 @@
#
# @option params [String] :job_run_id
# The ID of a previous `JobRun` to retry.
#
# @option params [Hash<String,String>] :arguments
- # The job arguments specifically for this run. For this job run, they
+ # The job arguments associated with this run. For this job run, they
# replace the default arguments set in the job definition itself.
#
# You can specify arguments here that your own job-execution script
# consumes, as well as arguments that Glue itself consumes.
#
@@ -13753,18 +13773,23 @@
#
# For information about how to specify and consume your own Job
# arguments, see the [Calling Glue APIs in Python][1] topic in the
# developer guide.
#
- # For information about the key-value pairs that Glue consumes to set up
- # your job, see the [Special Parameters Used by Glue][2] topic in the
+ # For information about the arguments you can provide to this field when
+ # configuring Spark jobs, see the [Special Parameters Used by Glue][2]
+ # topic in the developer guide.
+ #
+ # For information about the arguments you can provide to this field when
+ # configuring Ray jobs, see [Using job parameters in Ray jobs][3] in the
# developer guide.
#
#
#
# [1]: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-python-calling.html
# [2]: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+ # [3]: https://docs.aws.amazon.com/glue/latest/dg/author-job-ray-job-parameters.html
#
# @option params [Integer] :allocated_capacity
# This field is deprecated. Use `MaxCapacity` instead.
#
# The number of Glue data processing units (DPUs) to allocate to this
@@ -13784,28 +13809,35 @@
#
# Streaming jobs do not have a timeout. The default for non-streaming
# jobs is 2,880 minutes (48 hours).
#
# @option params [Float] :max_capacity
- # The number of Glue data processing units (DPUs) that can be allocated
+ # For Glue version 1.0 or earlier jobs, using the standard worker type,
+ # the number of Glue data processing units (DPUs) that can be allocated
# when this job runs. A DPU is a relative measure of processing power
# that consists of 4 vCPUs of compute capacity and 16 GB of memory. For
- # more information, see the [Glue pricing page][1].
+ # more information, see the [ Glue pricing page][1].
#
- # Do not set `Max Capacity` if using `WorkerType` and `NumberOfWorkers`.
+ # For Glue version 2.0+ jobs, you cannot specify a `Maximum capacity`.
+ # Instead, you should specify a `Worker type` and the `Number of
+ # workers`.
#
+ # Do not set `MaxCapacity` if using `WorkerType` and `NumberOfWorkers`.
+ #
# The value that can be allocated for `MaxCapacity` depends on whether
- # you are running a Python shell job, or an Apache Spark ETL job:
+ # you are running a Python shell job, an Apache Spark ETL job, or an
+ # Apache Spark streaming ETL job:
#
# * When you specify a Python shell job
# (`JobCommand.Name`="pythonshell"), you can allocate either 0.0625
# or 1 DPU. The default is 0.0625 DPU.
#
# * When you specify an Apache Spark ETL job
- # (`JobCommand.Name`="glueetl"), you can allocate a minimum of 2
- # DPUs. The default is 10 DPUs. This job type cannot have a fractional
- # DPU allocation.
+ # (`JobCommand.Name`="glueetl") or Apache Spark streaming ETL job
+ # (`JobCommand.Name`="gluestreaming"), you can allocate from 2 to
+ # 100 DPUs. The default is 10 DPUs. This job type cannot have a
+ # fractional DPU allocation.
#
#
#
# [1]: https://aws.amazon.com/glue/pricing/
#
@@ -13816,26 +13848,33 @@
# @option params [Types::NotificationProperty] :notification_property
# Specifies configuration properties of a job run notification.
#
# @option params [String] :worker_type
# The type of predefined worker that is allocated when a job runs.
- # Accepts a value of Standard, G.1X, G.2X, or G.025X.
+ # Accepts a value of Standard, G.1X, G.2X, or G.025X for Spark jobs.
+ # Accepts the value Z.2X for Ray jobs.
#
# * For the `Standard` worker type, each worker provides 4 vCPU, 16 GB
# of memory and a 50GB disk, and 2 executors per worker.
#
- # * For the `G.1X` worker type, each worker provides 4 vCPU, 16 GB of
- # memory and a 64GB disk, and 1 executor per worker.
+ # * For the `G.1X` worker type, each worker maps to 1 DPU (4 vCPU, 16 GB
+ # of memory, 64 GB disk), and provides 1 executor per worker. We
+ # recommend this worker type for memory-intensive jobs.
#
- # * For the `G.2X` worker type, each worker provides 8 vCPU, 32 GB of
- # memory and a 128GB disk, and 1 executor per worker.
+ # * For the `G.2X` worker type, each worker maps to 2 DPU (8 vCPU, 32 GB
+ # of memory, 128 GB disk), and provides 1 executor per worker. We
+ # recommend this worker type for memory-intensive jobs.
#
# * For the `G.025X` worker type, each worker maps to 0.25 DPU (2 vCPU,
# 4 GB of memory, 64 GB disk), and provides 1 executor per worker. We
# recommend this worker type for low volume streaming jobs. This
# worker type is only available for Glue version 3.0 streaming jobs.
#
+ # * For the `Z.2X` worker type, each worker maps to 2 DPU (8vCPU, 64 GB
+ # of m emory, 128 GB disk), and provides up to 8 Ray workers (one per
+ # vCPU) based on the autoscaler.
+ #
# @option params [Integer] :number_of_workers
# The number of workers of a defined `workerType` that are allocated
# when a job runs.
#
# @option params [String] :execution_class
@@ -13867,11 +13906,11 @@
# max_capacity: 1.0,
# security_configuration: "NameString",
# notification_property: {
# notify_delay_after: 1,
# },
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X, Z.2X
# number_of_workers: 1,
# execution_class: "FLEX", # accepts FLEX, STANDARD
# })
#
# @example Response structure
@@ -15193,11 +15232,11 @@
# },
# },
# role: "RoleString",
# glue_version: "GlueVersionString",
# max_capacity: 1.0,
- # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X
+ # worker_type: "Standard", # accepts Standard, G.1X, G.2X, G.025X, G.4X, G.8X, Z.2X
# number_of_workers: 1,
# timeout: 1,
# max_retries: 1,
# })
#
@@ -15822,10 +15861,10 @@
operation: config.api.operation(operation_name),
client: self,
params: params,
config: config)
context[:gem_name] = 'aws-sdk-glue'
- context[:gem_version] = '1.138.0'
+ context[:gem_version] = '1.139.0'
Seahorse::Client::Request.new(handlers, context)
end
# @api private
# @deprecated