public final class PySparkJob extends com.google.protobuf.GeneratedMessageV3 implements PySparkJobOrBuilder
A Cloud Dataproc job for running [Apache PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html) applications on YARN.Protobuf type
google.cloud.dataproc.v1.PySparkJob| Modifier and Type | Class and Description |
|---|---|
static class |
PySparkJob.Builder
A Cloud Dataproc job for running
[Apache
PySpark](https://spark.apache.org/docs/0.9.0/python-programming-guide.html)
applications on YARN.
|
com.google.protobuf.GeneratedMessageV3.BuilderParent, com.google.protobuf.GeneratedMessageV3.ExtendableBuilder<MessageType extends com.google.protobuf.GeneratedMessageV3.ExtendableMessage,BuilderType extends com.google.protobuf.GeneratedMessageV3.ExtendableBuilder<MessageType,BuilderType>>, com.google.protobuf.GeneratedMessageV3.ExtendableMessage<MessageType extends com.google.protobuf.GeneratedMessageV3.ExtendableMessage>, com.google.protobuf.GeneratedMessageV3.ExtendableMessageOrBuilder<MessageType extends com.google.protobuf.GeneratedMessageV3.ExtendableMessage>, com.google.protobuf.GeneratedMessageV3.FieldAccessorTable, com.google.protobuf.GeneratedMessageV3.UnusedPrivateParameter| Modifier and Type | Field and Description |
|---|---|
static int |
ARCHIVE_URIS_FIELD_NUMBER |
static int |
ARGS_FIELD_NUMBER |
static int |
FILE_URIS_FIELD_NUMBER |
static int |
JAR_FILE_URIS_FIELD_NUMBER |
static int |
LOGGING_CONFIG_FIELD_NUMBER |
static int |
MAIN_PYTHON_FILE_URI_FIELD_NUMBER |
static int |
PROPERTIES_FIELD_NUMBER |
static int |
PYTHON_FILE_URIS_FIELD_NUMBER |
| Modifier and Type | Method and Description |
|---|---|
boolean |
containsProperties(String key)
Optional.
|
boolean |
equals(Object obj) |
String |
getArchiveUris(int index)
Optional.
|
com.google.protobuf.ByteString |
getArchiveUrisBytes(int index)
Optional.
|
int |
getArchiveUrisCount()
Optional.
|
com.google.protobuf.ProtocolStringList |
getArchiveUrisList()
Optional.
|
String |
getArgs(int index)
Optional.
|
com.google.protobuf.ByteString |
getArgsBytes(int index)
Optional.
|
int |
getArgsCount()
Optional.
|
com.google.protobuf.ProtocolStringList |
getArgsList()
Optional.
|
static PySparkJob |
getDefaultInstance() |
PySparkJob |
getDefaultInstanceForType() |
static com.google.protobuf.Descriptors.Descriptor |
getDescriptor() |
String |
getFileUris(int index)
Optional.
|
com.google.protobuf.ByteString |
getFileUrisBytes(int index)
Optional.
|
int |
getFileUrisCount()
Optional.
|
com.google.protobuf.ProtocolStringList |
getFileUrisList()
Optional.
|
String |
getJarFileUris(int index)
Optional.
|
com.google.protobuf.ByteString |
getJarFileUrisBytes(int index)
Optional.
|
int |
getJarFileUrisCount()
Optional.
|
com.google.protobuf.ProtocolStringList |
getJarFileUrisList()
Optional.
|
LoggingConfig |
getLoggingConfig()
Optional.
|
LoggingConfigOrBuilder |
getLoggingConfigOrBuilder()
Optional.
|
String |
getMainPythonFileUri()
Required.
|
com.google.protobuf.ByteString |
getMainPythonFileUriBytes()
Required.
|
com.google.protobuf.Parser<PySparkJob> |
getParserForType() |
Map<String,String> |
getProperties()
Deprecated.
|
int |
getPropertiesCount()
Optional.
|
Map<String,String> |
getPropertiesMap()
Optional.
|
String |
getPropertiesOrDefault(String key,
String defaultValue)
Optional.
|
String |
getPropertiesOrThrow(String key)
Optional.
|
String |
getPythonFileUris(int index)
Optional.
|
com.google.protobuf.ByteString |
getPythonFileUrisBytes(int index)
Optional.
|
int |
getPythonFileUrisCount()
Optional.
|
com.google.protobuf.ProtocolStringList |
getPythonFileUrisList()
Optional.
|
int |
getSerializedSize() |
com.google.protobuf.UnknownFieldSet |
getUnknownFields() |
int |
hashCode() |
boolean |
hasLoggingConfig()
Optional.
|
protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable |
internalGetFieldAccessorTable() |
protected com.google.protobuf.MapField |
internalGetMapField(int number) |
boolean |
isInitialized() |
static PySparkJob.Builder |
newBuilder() |
static PySparkJob.Builder |
newBuilder(PySparkJob prototype) |
PySparkJob.Builder |
newBuilderForType() |
protected PySparkJob.Builder |
newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) |
static PySparkJob |
parseDelimitedFrom(InputStream input) |
static PySparkJob |
parseDelimitedFrom(InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry) |
static PySparkJob |
parseFrom(byte[] data) |
static PySparkJob |
parseFrom(byte[] data,
com.google.protobuf.ExtensionRegistryLite extensionRegistry) |
static PySparkJob |
parseFrom(ByteBuffer data) |
static PySparkJob |
parseFrom(ByteBuffer data,
com.google.protobuf.ExtensionRegistryLite extensionRegistry) |
static PySparkJob |
parseFrom(com.google.protobuf.ByteString data) |
static PySparkJob |
parseFrom(com.google.protobuf.ByteString data,
com.google.protobuf.ExtensionRegistryLite extensionRegistry) |
static PySparkJob |
parseFrom(com.google.protobuf.CodedInputStream input) |
static PySparkJob |
parseFrom(com.google.protobuf.CodedInputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry) |
static PySparkJob |
parseFrom(InputStream input) |
static PySparkJob |
parseFrom(InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry) |
static com.google.protobuf.Parser<PySparkJob> |
parser() |
PySparkJob.Builder |
toBuilder() |
void |
writeTo(com.google.protobuf.CodedOutputStream output) |
canUseUnsafe, computeStringSize, computeStringSizeNoTag, emptyBooleanList, emptyDoubleList, emptyFloatList, emptyIntList, emptyLongList, getAllFields, getDescriptorForType, getField, getOneofFieldDescriptor, getRepeatedField, getRepeatedFieldCount, hasField, hasOneof, makeExtensionsImmutable, mergeFromAndMakeImmutableInternal, mutableCopy, mutableCopy, mutableCopy, mutableCopy, mutableCopy, newBooleanList, newBuilderForType, newDoubleList, newFloatList, newInstance, newIntList, newLongList, parseDelimitedWithIOException, parseDelimitedWithIOException, parseUnknownField, parseUnknownFieldProto3, parseWithIOException, parseWithIOException, parseWithIOException, parseWithIOException, serializeBooleanMapTo, serializeIntegerMapTo, serializeLongMapTo, serializeStringMapTo, writeReplace, writeString, writeStringNoTagfindInitializationErrors, getInitializationErrorString, hashBoolean, hashEnum, hashEnumList, hashFields, hashLong, toStringaddAll, addAll, checkByteStringIsUtf8, toByteArray, toByteString, writeDelimitedTo, writeToclone, finalize, getClass, notify, notifyAll, wait, wait, waitpublic static final int MAIN_PYTHON_FILE_URI_FIELD_NUMBER
public static final int ARGS_FIELD_NUMBER
public static final int PYTHON_FILE_URIS_FIELD_NUMBER
public static final int JAR_FILE_URIS_FIELD_NUMBER
public static final int FILE_URIS_FIELD_NUMBER
public static final int ARCHIVE_URIS_FIELD_NUMBER
public static final int PROPERTIES_FIELD_NUMBER
public static final int LOGGING_CONFIG_FIELD_NUMBER
public final com.google.protobuf.UnknownFieldSet getUnknownFields()
getUnknownFields in interface com.google.protobuf.MessageOrBuildergetUnknownFields in class com.google.protobuf.GeneratedMessageV3public static final com.google.protobuf.Descriptors.Descriptor getDescriptor()
protected com.google.protobuf.MapField internalGetMapField(int number)
internalGetMapField in class com.google.protobuf.GeneratedMessageV3protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable()
internalGetFieldAccessorTable in class com.google.protobuf.GeneratedMessageV3public String getMainPythonFileUri()
Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.
string main_python_file_uri = 1;getMainPythonFileUri in interface PySparkJobOrBuilderpublic com.google.protobuf.ByteString getMainPythonFileUriBytes()
Required. The HCFS URI of the main Python file to use as the driver. Must be a .py file.
string main_python_file_uri = 1;getMainPythonFileUriBytes in interface PySparkJobOrBuilderpublic com.google.protobuf.ProtocolStringList getArgsList()
Optional. The arguments to pass to the driver. Do not include arguments, such as `--conf`, that can be set as job properties, since a collision may occur that causes an incorrect job submission.
repeated string args = 2;getArgsList in interface PySparkJobOrBuilderpublic int getArgsCount()
Optional. The arguments to pass to the driver. Do not include arguments, such as `--conf`, that can be set as job properties, since a collision may occur that causes an incorrect job submission.
repeated string args = 2;getArgsCount in interface PySparkJobOrBuilderpublic String getArgs(int index)
Optional. The arguments to pass to the driver. Do not include arguments, such as `--conf`, that can be set as job properties, since a collision may occur that causes an incorrect job submission.
repeated string args = 2;getArgs in interface PySparkJobOrBuilderpublic com.google.protobuf.ByteString getArgsBytes(int index)
Optional. The arguments to pass to the driver. Do not include arguments, such as `--conf`, that can be set as job properties, since a collision may occur that causes an incorrect job submission.
repeated string args = 2;getArgsBytes in interface PySparkJobOrBuilderpublic com.google.protobuf.ProtocolStringList getPythonFileUrisList()
Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.
repeated string python_file_uris = 3;getPythonFileUrisList in interface PySparkJobOrBuilderpublic int getPythonFileUrisCount()
Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.
repeated string python_file_uris = 3;getPythonFileUrisCount in interface PySparkJobOrBuilderpublic String getPythonFileUris(int index)
Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.
repeated string python_file_uris = 3;getPythonFileUris in interface PySparkJobOrBuilderpublic com.google.protobuf.ByteString getPythonFileUrisBytes(int index)
Optional. HCFS file URIs of Python files to pass to the PySpark framework. Supported file types: .py, .egg, and .zip.
repeated string python_file_uris = 3;getPythonFileUrisBytes in interface PySparkJobOrBuilderpublic com.google.protobuf.ProtocolStringList getJarFileUrisList()
Optional. HCFS URIs of jar files to add to the CLASSPATHs of the Python driver and tasks.
repeated string jar_file_uris = 4;getJarFileUrisList in interface PySparkJobOrBuilderpublic int getJarFileUrisCount()
Optional. HCFS URIs of jar files to add to the CLASSPATHs of the Python driver and tasks.
repeated string jar_file_uris = 4;getJarFileUrisCount in interface PySparkJobOrBuilderpublic String getJarFileUris(int index)
Optional. HCFS URIs of jar files to add to the CLASSPATHs of the Python driver and tasks.
repeated string jar_file_uris = 4;getJarFileUris in interface PySparkJobOrBuilderpublic com.google.protobuf.ByteString getJarFileUrisBytes(int index)
Optional. HCFS URIs of jar files to add to the CLASSPATHs of the Python driver and tasks.
repeated string jar_file_uris = 4;getJarFileUrisBytes in interface PySparkJobOrBuilderpublic com.google.protobuf.ProtocolStringList getFileUrisList()
Optional. HCFS URIs of files to be copied to the working directory of Python drivers and distributed tasks. Useful for naively parallel tasks.
repeated string file_uris = 5;getFileUrisList in interface PySparkJobOrBuilderpublic int getFileUrisCount()
Optional. HCFS URIs of files to be copied to the working directory of Python drivers and distributed tasks. Useful for naively parallel tasks.
repeated string file_uris = 5;getFileUrisCount in interface PySparkJobOrBuilderpublic String getFileUris(int index)
Optional. HCFS URIs of files to be copied to the working directory of Python drivers and distributed tasks. Useful for naively parallel tasks.
repeated string file_uris = 5;getFileUris in interface PySparkJobOrBuilderpublic com.google.protobuf.ByteString getFileUrisBytes(int index)
Optional. HCFS URIs of files to be copied to the working directory of Python drivers and distributed tasks. Useful for naively parallel tasks.
repeated string file_uris = 5;getFileUrisBytes in interface PySparkJobOrBuilderpublic com.google.protobuf.ProtocolStringList getArchiveUrisList()
Optional. HCFS URIs of archives to be extracted in the working directory of .jar, .tar, .tar.gz, .tgz, and .zip.
repeated string archive_uris = 6;getArchiveUrisList in interface PySparkJobOrBuilderpublic int getArchiveUrisCount()
Optional. HCFS URIs of archives to be extracted in the working directory of .jar, .tar, .tar.gz, .tgz, and .zip.
repeated string archive_uris = 6;getArchiveUrisCount in interface PySparkJobOrBuilderpublic String getArchiveUris(int index)
Optional. HCFS URIs of archives to be extracted in the working directory of .jar, .tar, .tar.gz, .tgz, and .zip.
repeated string archive_uris = 6;getArchiveUris in interface PySparkJobOrBuilderpublic com.google.protobuf.ByteString getArchiveUrisBytes(int index)
Optional. HCFS URIs of archives to be extracted in the working directory of .jar, .tar, .tar.gz, .tgz, and .zip.
repeated string archive_uris = 6;getArchiveUrisBytes in interface PySparkJobOrBuilderpublic int getPropertiesCount()
PySparkJobOrBuilderOptional. A mapping of property names to values, used to configure PySpark. Properties that conflict with values set by the Cloud Dataproc API may be overwritten. Can include properties set in /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;getPropertiesCount in interface PySparkJobOrBuilderpublic boolean containsProperties(String key)
Optional. A mapping of property names to values, used to configure PySpark. Properties that conflict with values set by the Cloud Dataproc API may be overwritten. Can include properties set in /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;containsProperties in interface PySparkJobOrBuilder@Deprecated public Map<String,String> getProperties()
getPropertiesMap() instead.getProperties in interface PySparkJobOrBuilderpublic Map<String,String> getPropertiesMap()
Optional. A mapping of property names to values, used to configure PySpark. Properties that conflict with values set by the Cloud Dataproc API may be overwritten. Can include properties set in /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;getPropertiesMap in interface PySparkJobOrBuilderpublic String getPropertiesOrDefault(String key, String defaultValue)
Optional. A mapping of property names to values, used to configure PySpark. Properties that conflict with values set by the Cloud Dataproc API may be overwritten. Can include properties set in /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;getPropertiesOrDefault in interface PySparkJobOrBuilderpublic String getPropertiesOrThrow(String key)
Optional. A mapping of property names to values, used to configure PySpark. Properties that conflict with values set by the Cloud Dataproc API may be overwritten. Can include properties set in /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7;getPropertiesOrThrow in interface PySparkJobOrBuilderpublic boolean hasLoggingConfig()
Optional. The runtime log config for job execution.
.google.cloud.dataproc.v1.LoggingConfig logging_config = 8;hasLoggingConfig in interface PySparkJobOrBuilderpublic LoggingConfig getLoggingConfig()
Optional. The runtime log config for job execution.
.google.cloud.dataproc.v1.LoggingConfig logging_config = 8;getLoggingConfig in interface PySparkJobOrBuilderpublic LoggingConfigOrBuilder getLoggingConfigOrBuilder()
Optional. The runtime log config for job execution.
.google.cloud.dataproc.v1.LoggingConfig logging_config = 8;getLoggingConfigOrBuilder in interface PySparkJobOrBuilderpublic final boolean isInitialized()
isInitialized in interface com.google.protobuf.MessageLiteOrBuilderisInitialized in class com.google.protobuf.GeneratedMessageV3public void writeTo(com.google.protobuf.CodedOutputStream output)
throws IOException
writeTo in interface com.google.protobuf.MessageLitewriteTo in class com.google.protobuf.GeneratedMessageV3IOExceptionpublic int getSerializedSize()
getSerializedSize in interface com.google.protobuf.MessageLitegetSerializedSize in class com.google.protobuf.GeneratedMessageV3public boolean equals(Object obj)
equals in interface com.google.protobuf.Messageequals in class com.google.protobuf.AbstractMessagepublic int hashCode()
hashCode in interface com.google.protobuf.MessagehashCode in class com.google.protobuf.AbstractMessagepublic static PySparkJob parseFrom(ByteBuffer data) throws com.google.protobuf.InvalidProtocolBufferException
com.google.protobuf.InvalidProtocolBufferExceptionpublic static PySparkJob parseFrom(ByteBuffer data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException
com.google.protobuf.InvalidProtocolBufferExceptionpublic static PySparkJob parseFrom(com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException
com.google.protobuf.InvalidProtocolBufferExceptionpublic static PySparkJob parseFrom(com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException
com.google.protobuf.InvalidProtocolBufferExceptionpublic static PySparkJob parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException
com.google.protobuf.InvalidProtocolBufferExceptionpublic static PySparkJob parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException
com.google.protobuf.InvalidProtocolBufferExceptionpublic static PySparkJob parseFrom(InputStream input) throws IOException
IOExceptionpublic static PySparkJob parseFrom(InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws IOException
IOExceptionpublic static PySparkJob parseDelimitedFrom(InputStream input) throws IOException
IOExceptionpublic static PySparkJob parseDelimitedFrom(InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws IOException
IOExceptionpublic static PySparkJob parseFrom(com.google.protobuf.CodedInputStream input) throws IOException
IOExceptionpublic static PySparkJob parseFrom(com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws IOException
IOExceptionpublic PySparkJob.Builder newBuilderForType()
newBuilderForType in interface com.google.protobuf.MessagenewBuilderForType in interface com.google.protobuf.MessageLitepublic static PySparkJob.Builder newBuilder()
public static PySparkJob.Builder newBuilder(PySparkJob prototype)
public PySparkJob.Builder toBuilder()
toBuilder in interface com.google.protobuf.MessagetoBuilder in interface com.google.protobuf.MessageLiteprotected PySparkJob.Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent)
newBuilderForType in class com.google.protobuf.GeneratedMessageV3public static PySparkJob getDefaultInstance()
public static com.google.protobuf.Parser<PySparkJob> parser()
public com.google.protobuf.Parser<PySparkJob> getParserForType()
getParserForType in interface com.google.protobuf.MessagegetParserForType in interface com.google.protobuf.MessageLitegetParserForType in class com.google.protobuf.GeneratedMessageV3public PySparkJob getDefaultInstanceForType()
getDefaultInstanceForType in interface com.google.protobuf.MessageLiteOrBuildergetDefaultInstanceForType in interface com.google.protobuf.MessageOrBuilderCopyright © 2019 Google LLC. All rights reserved.