Module pachyderm_sdk.api.pjs

Classes

class JobState (*args, **kwds)

The base class for protobuf enumerations, all generated enumerations will inherit from this. Bases :class:enum.IntEnum.

Expand source code

class JobState(betterproto.Enum):
    JobState_UNSPECIFIED = 0
    """UNSPECIFIED means the job state is unspecified."""

    QUEUED = 1
    """
    QUEUED means the job is currently in a queue. A QUEUED job will not have
    any descendants.
    """

    PROCESSING = 2
    """PROCESSING means the job is currently being processed by a worker."""

    DONE = 3
    """DONE means the job, and all of its descendants, are done."""

Ancestors

• betterproto.Enum
• enum.IntEnum
• builtins.int
• enum.ReprEnum
• enum.Enum

Class variables

var JobState_UNSPECIFIED

UNSPECIFIED means the job state is unspecified.

var QUEUED

QUEUED means the job is currently in a queue. A QUEUED job will not have any descendants.

var PROCESSING

PROCESSING means the job is currently being processed by a worker.

var DONE

DONE means the job, and all of its descendants, are done.

class JobErrorCode (*args, **kwds)

The base class for protobuf enumerations, all generated enumerations will inherit from this. Bases :class:enum.IntEnum.

Expand source code

class JobErrorCode(betterproto.Enum):
    JobErrorCode_UNSPECIFIED = 0
    """UNSPECIFIED means the job error code is unspecified."""

    FAILED = 1
    """
    FAILED means that the worker processing the job indicated that it failed.
    """

    DISCONNECTED = 2
    """DISCONNECTED means the worker processing the job disconnected."""

    CANCELED = 3
    """CANCELED means the job was canceled."""

Ancestors

• betterproto.Enum
• enum.IntEnum
• builtins.int
• enum.ReprEnum
• enum.Enum

Class variables

var JobErrorCode_UNSPECIFIED

UNSPECIFIED means the job error code is unspecified.

var FAILED

FAILED means that the worker processing the job indicated that it failed.

var DISCONNECTED

DISCONNECTED means the worker processing the job disconnected.

var CANCELED

CANCELED means the job was canceled.

class Job (id: int = None)

Job uniquely identifies a Job Job will be nil to indicate no Job, or an unset Job.

Expand source code

@dataclass(eq=False, repr=False)
class Job(betterproto.Message):
    """
    Job uniquely identifies a Job Job will be nil to indicate no Job, or an
    unset Job.
    """

    id: int = betterproto.int64_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var id : int

class JobInfo (job: Job = None, parent_job: Job = None, state: JobState = None, program: str = None, input: List[str] = None, success: JobInfoSuccess = None, error: JobErrorCode = None)

JobInfo describes a Job

Expand source code

@dataclass(eq=False, repr=False)
class JobInfo(betterproto.Message):
    """JobInfo describes a Job"""

    job: "Job" = betterproto.message_field(1)
    """Job is the Job's identity"""

    parent_job: "Job" = betterproto.message_field(2)
    """parent_job is the Job's parent if it exists."""

    state: "JobState" = betterproto.enum_field(3)
    """
    state is the Job's state. See JobState for a description of the possible
    states.
    """

    program: str = betterproto.string_field(4)
    """
    program is the fileset that contains the code specification for the Job.
    """

    input: List[str] = betterproto.string_field(5)
    """input is the input fileset handles for the Job."""

    success: "JobInfoSuccess" = betterproto.message_field(6, group="result")
    error: "JobErrorCode" = betterproto.enum_field(7, group="result")
    """error is set when the Job is unable to complete successfully"""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var job : Job

Job is the Job's identity

var parent_job : Job

parent_job is the Job's parent if it exists.

var state : JobState

state is the Job's state. See JobState for a description of the possible states.

var program : str

program is the fileset that contains the code specification for the Job.

var input : List[str]

input is the input fileset handles for the Job.

var success : JobInfoSuccess

var error : JobErrorCode

error is set when the Job is unable to complete successfully

class JobInfoSuccess (output: List[str] = None)

Success is produced by a successfully completing Job.

Expand source code

@dataclass(eq=False, repr=False)
class JobInfoSuccess(betterproto.Message):
    """Success is produced by a successfully completing Job."""

    output: List[str] = betterproto.string_field(1)
    """output is a list of fileset handles produced by a successful Job."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var output : List[str]

output is a list of fileset handles produced by a successful Job.

class JobInfoDetails (job_info: JobInfo = None)

JobInfoDetails is more detailed information about a Job. It contains a superset of the information in JobInfo

Expand source code

@dataclass(eq=False, repr=False)
class JobInfoDetails(betterproto.Message):
    """
    JobInfoDetails is more detailed information about a Job. It contains a
    superset of the information in JobInfo
    """

    job_info: "JobInfo" = betterproto.message_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var job_info : JobInfo

class Queue (id: bytes = None)

Queue uniquely identifies a Queue Queue will be nil to identify no Queue, or to indicate unset.

Expand source code

@dataclass(eq=False, repr=False)
class Queue(betterproto.Message):
    """
    Queue uniquely identifies a Queue Queue will be nil to identify no Queue,
    or to indicate unset.
    """

    id: bytes = betterproto.bytes_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var id : bytes

class QueueInfo (queue: Queue = None, spec: betterproto_lib_google_protobuf.Any = None)

QueueInfo describes a Queue

Expand source code

@dataclass(eq=False, repr=False)
class QueueInfo(betterproto.Message):
    """QueueInfo describes a Queue"""

    queue: "Queue" = betterproto.message_field(1)
    """queue is the Queue's identity"""

    spec: "betterproto_lib_google_protobuf.Any" = betterproto.message_field(2)
    """spec specifies the code to be run to process the Queue."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var queue : Queue

queue is the Queue's identity

var spec : betterproto.lib.google.protobuf.Any

spec specifies the code to be run to process the Queue.

class QueueInfoDetails (queue_info: QueueInfo = None, size: int = None)

QueueInfoDetails contains detailed information about a Queue, which may be more expensive to get. It contains a superset of the information in QueueInfo.

Expand source code

@dataclass(eq=False, repr=False)
class QueueInfoDetails(betterproto.Message):
    """
    QueueInfoDetails contains detailed information about a Queue, which may be
    more expensive to get. It contains a superset of the information in
    QueueInfo.
    """

    queue_info: "QueueInfo" = betterproto.message_field(1)
    size: int = betterproto.int64_field(2)
    """size is the number of elements queued."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var queue_info : QueueInfo

var size : int

size is the number of elements queued.

class CreateJobRequest (context: str = None, program: str = None, input: List[str] = None, cache_read: bool = None, cache_write: bool = None)

CreateJobRequest(context: str = None, program: str = None, input: List[str] = None, cache_read: bool = None, cache_write: bool = None)

Expand source code

@dataclass(eq=False, repr=False)
class CreateJobRequest(betterproto.Message):
    context: str = betterproto.string_field(1)
    """
    context is a bearer token used when calling from within a running Job.
    """

    program: str = betterproto.string_field(2)
    """program is a fileset handle."""

    input: List[str] = betterproto.string_field(3)
    """input is a list of fileset handles."""

    cache_read: bool = betterproto.bool_field(4)
    cache_write: bool = betterproto.bool_field(5)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used when calling from within a running Job.

var program : str

program is a fileset handle.

var input : List[str]

input is a list of fileset handles.

var cache_read : bool

var cache_write : bool

class CreateJobResponse (id: Job = None)

CreateJobResponse(id: 'Job' = None)

Expand source code

@dataclass(eq=False, repr=False)
class CreateJobResponse(betterproto.Message):
    id: "Job" = betterproto.message_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var id : Job

class CancelJobRequest (context: str = None, job: Job = None)

CancelJobRequest(context: str = None, job: 'Job' = None)

Expand source code

@dataclass(eq=False, repr=False)
class CancelJobRequest(betterproto.Message):
    context: str = betterproto.string_field(1)
    """
    context is a bearer token used when calling from within a running Job.
    """

    job: "Job" = betterproto.message_field(2)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used when calling from within a running Job.

var job : Job

class CancelJobResponse

CancelJobResponse()

Expand source code

@dataclass(eq=False, repr=False)
class CancelJobResponse(betterproto.Message):
    pass

Ancestors

• betterproto.Message
• abc.ABC

class DeleteJobRequest (context: str = None, job: Job = None)

DeleteJobRequest(context: str = None, job: 'Job' = None)

Expand source code

@dataclass(eq=False, repr=False)
class DeleteJobRequest(betterproto.Message):
    context: str = betterproto.string_field(1)
    """
    context is a bearer token used when calling from within a running Job.
    """

    job: "Job" = betterproto.message_field(2)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used when calling from within a running Job.

var job : Job

class DeleteJobResponse

DeleteJobResponse()

Expand source code

@dataclass(eq=False, repr=False)
class DeleteJobResponse(betterproto.Message):
    pass

Ancestors

• betterproto.Message
• abc.ABC

class ListJobRequest (context: str = None, job: Job = None)

TODO: - Filter - Paginate

Expand source code

@dataclass(eq=False, repr=False)
class ListJobRequest(betterproto.Message):
    """TODO: - Filter - Paginate"""

    context: str = betterproto.string_field(1)
    """
    context is a bearer token used when calling from within a running Job.
    """

    job: "Job" = betterproto.message_field(2)
    """
    job is the job to start listing at. If nil, then the listing starts at the
    first job in the natural ordering.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used when calling from within a running Job.

var job : Job

job is the job to start listing at. If nil, then the listing starts at the first job in the natural ordering.

class ListJobResponse (id: Job = None, info: JobInfo = None, details: JobInfoDetails = None)

ListJobResponse lists information about Jobs ID will always be set. Info and Details may not be set depending on how much information was requested.

Expand source code

@dataclass(eq=False, repr=False)
class ListJobResponse(betterproto.Message):
    """
    ListJobResponse lists information about Jobs ID will always be set. Info
    and Details may not be set depending on how much information was requested.
    """

    id: "Job" = betterproto.message_field(1)
    info: "JobInfo" = betterproto.message_field(2)
    details: "JobInfoDetails" = betterproto.message_field(3)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var id : Job

var info : JobInfo

var details : JobInfoDetails

class WalkJobRequest (context: str = None, job: Job = None)

WalkJobRequest(context: str = None, job: 'Job' = None)

Expand source code

@dataclass(eq=False, repr=False)
class WalkJobRequest(betterproto.Message):
    context: str = betterproto.string_field(1)
    """
    context is a bearer token used when calling from within a running Job.
    """

    job: "Job" = betterproto.message_field(2)
    """
    job is the job to start walking from.  If unset, the context Job is
    assumed.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used when calling from within a running Job.

var job : Job

job is the job to start walking from. If unset, the context Job is assumed.

class InspectJobRequest (context: str = None, job: Job = None)

InspectJobRequest(context: str = None, job: 'Job' = None)

Expand source code

@dataclass(eq=False, repr=False)
class InspectJobRequest(betterproto.Message):
    context: str = betterproto.string_field(1)
    """
    context is a bearer token used when calling from within a running Job.
    """

    job: "Job" = betterproto.message_field(2)
    """
    job is the job to start walking from.  If unset the context Job is assumed.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used when calling from within a running Job.

var job : Job

job is the job to start walking from. If unset the context Job is assumed.

class InspectJobResponse (details: JobInfoDetails = None)

InspectJobResponse(details: 'JobInfoDetails' = None)

Expand source code

@dataclass(eq=False, repr=False)
class InspectJobResponse(betterproto.Message):
    details: "JobInfoDetails" = betterproto.message_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var details : JobInfoDetails

class ProcessQueueRequest (queue: Queue = None, success: ProcessQueueRequestSuccess = None, failed: bool = None)

Queue Messages ProcessQueueRequest is the client -> server message for the bi-di ProcessQueue RPC.

Expand source code

@dataclass(eq=False, repr=False)
class ProcessQueueRequest(betterproto.Message):
    """
    Queue Messages ProcessQueueRequest is the client -> server message for the
    bi-di ProcessQueue RPC.
    """

    queue: "Queue" = betterproto.message_field(1)
    """queue is set to start processing from a Queue."""

    success: "ProcessQueueRequestSuccess" = betterproto.message_field(2, group="result")
    failed: bool = betterproto.bool_field(3, group="result")
    """
    failed is set by the client to fail the Job. The Job will transition to
    state DONE with code FAILED.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var queue : Queue

queue is set to start processing from a Queue.

var success : ProcessQueueRequestSuccess

var failed : bool

failed is set by the client to fail the Job. The Job will transition to state DONE with code FAILED.

class ProcessQueueRequestSuccess (output: List[str] = None)

Success is set by the client to complete the Job successfully.

Expand source code

@dataclass(eq=False, repr=False)
class ProcessQueueRequestSuccess(betterproto.Message):
    """Success is set by the client to complete the Job successfully."""

    output: List[str] = betterproto.string_field(1)
    """output is a list of fileset handles produced by a successful Job."""

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var output : List[str]

output is a list of fileset handles produced by a successful Job.

class ProcessQueueResponse (context: str = None, input: List[str] = None)

ProcessQueueResposne is the server -> client message for the bi-di ProcessQueue RPC.

Expand source code

@dataclass(eq=False, repr=False)
class ProcessQueueResponse(betterproto.Message):
    """
    ProcessQueueResposne is the server -> client message for the bi-di
    ProcessQueue RPC.
    """

    context: str = betterproto.string_field(1)
    """
    context is a bearer token used to act on behalf of the Job in other RPCs.
    The server issues this token to the client, and the client should use it
    when performing Job RPCs.
    """

    input: List[str] = betterproto.string_field(2)
    """
    input is the input data for a Job. The server sends this to ask the client
    to compute the output.
    """

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var context : str

context is a bearer token used to act on behalf of the Job in other RPCs. The server issues this token to the client, and the client should use it when performing Job RPCs.

var input : List[str]

input is the input data for a Job. The server sends this to ask the client to compute the output.

class ListQueueRequest

TODO: - Filter - Paginate

Expand source code

@dataclass(eq=False, repr=False)
class ListQueueRequest(betterproto.Message):
    """TODO: - Filter - Paginate"""

    pass

Ancestors

• betterproto.Message
• abc.ABC

class ListQueueResponse (id: Queue = None, info: QueueInfo = None, details: QueueInfoDetails = None)

ListQueueResponse(id: 'Queue' = None, info: 'QueueInfo' = None, details: 'QueueInfoDetails' = None)

Expand source code

@dataclass(eq=False, repr=False)
class ListQueueResponse(betterproto.Message):
    id: "Queue" = betterproto.message_field(1)
    info: "QueueInfo" = betterproto.message_field(2)
    details: "QueueInfoDetails" = betterproto.message_field(3)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var id : Queue

var info : QueueInfo

var details : QueueInfoDetails

class InspectQueueRequest (queue: Queue = None)

InspectQueueRequest(queue: 'Queue' = None)

Expand source code

@dataclass(eq=False, repr=False)
class InspectQueueRequest(betterproto.Message):
    queue: "Queue" = betterproto.message_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var queue : Queue

class InspectQueueResponse (details: QueueInfoDetails = None)

InspectQueueResponse(details: 'QueueInfoDetails' = None)

Expand source code

@dataclass(eq=False, repr=False)
class InspectQueueResponse(betterproto.Message):
    details: "QueueInfoDetails" = betterproto.message_field(1)

Ancestors

• betterproto.Message
• abc.ABC

Class variables

var details : QueueInfoDetails

class ApiStub (channel: grpc.Channel)

Expand source code

class ApiStub:

    def __init__(self, channel: "grpc.Channel"):
        self.__rpc_create_job = channel.unary_unary(
            "/pjs.API/CreateJob",
            request_serializer=CreateJobRequest.SerializeToString,
            response_deserializer=CreateJobResponse.FromString,
        )
        self.__rpc_cancel_job = channel.unary_unary(
            "/pjs.API/CancelJob",
            request_serializer=CancelJobRequest.SerializeToString,
            response_deserializer=CancelJobResponse.FromString,
        )
        self.__rpc_delete_job = channel.unary_unary(
            "/pjs.API/DeleteJob",
            request_serializer=DeleteJobRequest.SerializeToString,
            response_deserializer=DeleteJobResponse.FromString,
        )
        self.__rpc_list_job = channel.unary_stream(
            "/pjs.API/ListJob",
            request_serializer=ListJobRequest.SerializeToString,
            response_deserializer=ListJobResponse.FromString,
        )
        self.__rpc_walk_job = channel.unary_stream(
            "/pjs.API/WalkJob",
            request_serializer=WalkJobRequest.SerializeToString,
            response_deserializer=ListJobResponse.FromString,
        )
        self.__rpc_inspect_job = channel.unary_unary(
            "/pjs.API/InspectJob",
            request_serializer=InspectJobRequest.SerializeToString,
            response_deserializer=InspectJobResponse.FromString,
        )
        self.__rpc_process_queue = channel.stream_stream(
            "/pjs.API/ProcessQueue",
            request_serializer=ProcessQueueRequest.SerializeToString,
            response_deserializer=ProcessQueueResponse.FromString,
        )
        self.__rpc_list_queue = channel.unary_stream(
            "/pjs.API/ListQueue",
            request_serializer=ListQueueRequest.SerializeToString,
            response_deserializer=ListQueueResponse.FromString,
        )
        self.__rpc_inspect_queue = channel.unary_unary(
            "/pjs.API/InspectQueue",
            request_serializer=InspectQueueRequest.SerializeToString,
            response_deserializer=InspectQueueResponse.FromString,
        )

    def create_job(
        self,
        *,
        context: str = "",
        program: str = "",
        input: Optional[List[str]] = None,
        cache_read: bool = False,
        cache_write: bool = False
    ) -> "CreateJobResponse":
        input = input or []

        request = CreateJobRequest()
        request.context = context
        request.program = program
        request.input = input
        request.cache_read = cache_read
        request.cache_write = cache_write

        return self.__rpc_create_job(request)

    def cancel_job(
        self, *, context: str = "", job: "Job" = None
    ) -> "CancelJobResponse":

        request = CancelJobRequest()
        request.context = context
        if job is not None:
            request.job = job

        return self.__rpc_cancel_job(request)

    def delete_job(
        self, *, context: str = "", job: "Job" = None
    ) -> "DeleteJobResponse":

        request = DeleteJobRequest()
        request.context = context
        if job is not None:
            request.job = job

        return self.__rpc_delete_job(request)

    def list_job(
        self, *, context: str = "", job: "Job" = None
    ) -> Iterator["ListJobResponse"]:

        request = ListJobRequest()
        request.context = context
        if job is not None:
            request.job = job

        for response in self.__rpc_list_job(request):
            yield response

    def walk_job(
        self, *, context: str = "", job: "Job" = None
    ) -> Iterator["ListJobResponse"]:

        request = WalkJobRequest()
        request.context = context
        if job is not None:
            request.job = job

        for response in self.__rpc_walk_job(request):
            yield response

    def inspect_job(
        self, *, context: str = "", job: "Job" = None
    ) -> "InspectJobResponse":

        request = InspectJobRequest()
        request.context = context
        if job is not None:
            request.job = job

        return self.__rpc_inspect_job(request)

    def process_queue(
        self,
        request_iterator: Union[
            AsyncIterable["ProcessQueueRequest"], Iterable["ProcessQueueRequest"]
        ],
    ) -> Iterator["ProcessQueueResponse"]:

        for response in self.__rpc_process_queue(request_iterator):
            yield response

    def list_queue(self) -> Iterator["ListQueueResponse"]:

        request = ListQueueRequest()

        for response in self.__rpc_list_queue(request):
            yield response

    def inspect_queue(self, *, queue: "Queue" = None) -> "InspectQueueResponse":

        request = InspectQueueRequest()
        if queue is not None:
            request.queue = queue

        return self.__rpc_inspect_queue(request)

Methods

def create_job(self, *, context: str = '', program: str = '', input: Optional[List[str]] = None, cache_read: bool = False, cache_write: bool = False) ‑> CreateJobResponse

Expand source code

def create_job(
    self,
    *,
    context: str = "",
    program: str = "",
    input: Optional[List[str]] = None,
    cache_read: bool = False,
    cache_write: bool = False
) -> "CreateJobResponse":
    input = input or []

    request = CreateJobRequest()
    request.context = context
    request.program = program
    request.input = input
    request.cache_read = cache_read
    request.cache_write = cache_write

    return self.__rpc_create_job(request)

def cancel_job(self, *, context: str = '', job: Job = None) ‑> CancelJobResponse

Expand source code

def cancel_job(
    self, *, context: str = "", job: "Job" = None
) -> "CancelJobResponse":

    request = CancelJobRequest()
    request.context = context
    if job is not None:
        request.job = job

    return self.__rpc_cancel_job(request)

def delete_job(self, *, context: str = '', job: Job = None) ‑> DeleteJobResponse

Expand source code

def delete_job(
    self, *, context: str = "", job: "Job" = None
) -> "DeleteJobResponse":

    request = DeleteJobRequest()
    request.context = context
    if job is not None:
        request.job = job

    return self.__rpc_delete_job(request)

def list_job(self, *, context: str = '', job: Job = None) ‑> Iterator[ListJobResponse]

Expand source code

def list_job(
    self, *, context: str = "", job: "Job" = None
) -> Iterator["ListJobResponse"]:

    request = ListJobRequest()
    request.context = context
    if job is not None:
        request.job = job

    for response in self.__rpc_list_job(request):
        yield response

def walk_job(self, *, context: str = '', job: Job = None) ‑> Iterator[ListJobResponse]

Expand source code

def walk_job(
    self, *, context: str = "", job: "Job" = None
) -> Iterator["ListJobResponse"]:

    request = WalkJobRequest()
    request.context = context
    if job is not None:
        request.job = job

    for response in self.__rpc_walk_job(request):
        yield response

def inspect_job(self, *, context: str = '', job: Job = None) ‑> InspectJobResponse

Expand source code

def inspect_job(
    self, *, context: str = "", job: "Job" = None
) -> "InspectJobResponse":

    request = InspectJobRequest()
    request.context = context
    if job is not None:
        request.job = job

    return self.__rpc_inspect_job(request)

def process_queue(self, request_iterator: Union[AsyncIterable[ForwardRef('ProcessQueueRequest')], Iterable[ForwardRef('ProcessQueueRequest')]]) ‑> Iterator[ProcessQueueResponse]

Expand source code

def process_queue(
    self,
    request_iterator: Union[
        AsyncIterable["ProcessQueueRequest"], Iterable["ProcessQueueRequest"]
    ],
) -> Iterator["ProcessQueueResponse"]:

    for response in self.__rpc_process_queue(request_iterator):
        yield response

def list_queue(self) ‑> Iterator[ListQueueResponse]

Expand source code

def list_queue(self) -> Iterator["ListQueueResponse"]:

    request = ListQueueRequest()

    for response in self.__rpc_list_queue(request):
        yield response

def inspect_queue(self, *, queue: Queue = None) ‑> InspectQueueResponse

Expand source code

def inspect_queue(self, *, queue: "Queue" = None) -> "InspectQueueResponse":

    request = InspectQueueRequest()
    if queue is not None:
        request.queue = queue

    return self.__rpc_inspect_queue(request)