prefect_dbt.cloud.jobs

Module containing tasks and flows for interacting with dbt Cloud jobs

DbtCloudJob

Bases: JobBlock

Block that holds the information and methods to interact with a dbt Cloud job.

Attributes:

Name	Type	Description
dbt_cloud_credentials	DbtCloudCredentials	The credentials to use to authenticate with dbt Cloud.
job_id	int	The id of the dbt Cloud job.
timeout_seconds	int	The number of seconds to wait for the job to complete.
interval_seconds	int	The number of seconds to wait between polling for job completion.
trigger_job_run_options	TriggerJobRunOptions	The options to use when triggering a job run.

Examples:

Load a configured dbt Cloud job block.

from prefect_dbt.cloud import DbtCloudJob

dbt_cloud_job = DbtCloudJob.load("BLOCK_NAME")

Triggers a dbt Cloud job, waits for completion, and fetches the results.

from prefect import flow
from prefect_dbt.cloud import DbtCloudCredentials, DbtCloudJob

@flow
def dbt_cloud_job_flow():
    dbt_cloud_credentials = DbtCloudCredentials.load("dbt-token")
    dbt_cloud_job = DbtCloudJob.load(
        dbt_cloud_credentials=dbt_cloud_credentials,
        job_id=154217
    )
    dbt_cloud_job_run = dbt_cloud_job.trigger()
    dbt_cloud_job_run.wait_for_completion()
    dbt_cloud_job_run.fetch_result()
    return dbt_cloud_job_run

dbt_cloud_job_flow()

Source code in prefect_dbt/cloud/jobs.py

class DbtCloudJob(JobBlock):
    """
    Block that holds the information and methods to interact with a dbt Cloud job.

    Attributes:
        dbt_cloud_credentials: The credentials to use to authenticate with dbt Cloud.
        job_id: The id of the dbt Cloud job.
        timeout_seconds: The number of seconds to wait for the job to complete.
        interval_seconds:
            The number of seconds to wait between polling for job completion.
        trigger_job_run_options: The options to use when triggering a job run.

    Examples:
        Load a configured dbt Cloud job block.
        ```python
        from prefect_dbt.cloud import DbtCloudJob

        dbt_cloud_job = DbtCloudJob.load("BLOCK_NAME")
        ```

        Triggers a dbt Cloud job, waits for completion, and fetches the results.
        ```python
        from prefect import flow
        from prefect_dbt.cloud import DbtCloudCredentials, DbtCloudJob

        @flow
        def dbt_cloud_job_flow():
            dbt_cloud_credentials = DbtCloudCredentials.load("dbt-token")
            dbt_cloud_job = DbtCloudJob.load(
                dbt_cloud_credentials=dbt_cloud_credentials,
                job_id=154217
            )
            dbt_cloud_job_run = dbt_cloud_job.trigger()
            dbt_cloud_job_run.wait_for_completion()
            dbt_cloud_job_run.fetch_result()
            return dbt_cloud_job_run

        dbt_cloud_job_flow()
        ```
    """

    _block_type_name = "dbt Cloud Job"
    _logo_url = "https://images.ctfassets.net/gm98wzqotmnx/5zE9lxfzBHjw3tnEup4wWL/9a001902ed43a84c6c96d23b24622e19/dbt-bit_tm.png?h=250"  # noqa
    _documentation_url = "https://prefecthq.github.io/prefect-dbt/cloud/jobs/#prefect_dbt.cloud.jobs.DbtCloudJob"  # noqa

    dbt_cloud_credentials: DbtCloudCredentials = Field(
        default=...,
        description="The dbt Cloud credentials to use to authenticate with dbt Cloud.",
    )  # noqa: E501
    job_id: int = Field(
        default=..., description="The id of the dbt Cloud job.", title="Job ID"
    )
    timeout_seconds: int = Field(
        default=900,
        description="The number of seconds to wait for the job to complete.",
    )
    interval_seconds: int = Field(
        default=10,
        description="The number of seconds to wait between polling for job completion.",
    )
    trigger_job_run_options: TriggerJobRunOptions = Field(
        default_factory=TriggerJobRunOptions,
        description="The options to use when triggering a job run.",
    )

    @sync_compatible
    async def get_job(self, order_by: Optional[str] = None) -> Dict[str, Any]:
        """
        Retrieve information about a dbt Cloud job.

        Args:
            order_by: The field to order the results by.

        Returns:
            The job data.
        """
        try:
            async with self.dbt_cloud_credentials.get_administrative_client() as client:
                response = await client.get_job(
                    job_id=self.job_id,
                    order_by=order_by,
                )
        except HTTPStatusError as ex:
            raise DbtCloudGetJobFailed(extract_user_message(ex)) from ex
        return response.json()["data"]

    @sync_compatible
    async def trigger(
        self, trigger_job_run_options: Optional[TriggerJobRunOptions] = None
    ) -> DbtCloudJobRun:
        """
        Triggers a dbt Cloud job.

        Returns:
            A representation of the dbt Cloud job run.
        """
        try:
            trigger_job_run_options = (
                trigger_job_run_options or self.trigger_job_run_options
            )
            async with self.dbt_cloud_credentials.get_administrative_client() as client:
                response = await client.trigger_job_run(
                    job_id=self.job_id, options=trigger_job_run_options
                )
        except HTTPStatusError as ex:
            raise DbtCloudJobRunTriggerFailed(extract_user_message(ex)) from ex

        run_data = response.json()["data"]
        run_id = run_data.get("id")
        run = DbtCloudJobRun(
            dbt_cloud_job=self,
            run_id=run_id,
        )
        self.logger.info(
            f"dbt Cloud job {self.job_id} run {run_id} successfully triggered. "
            f"You can view the status of this run at "
            f"https://{self.dbt_cloud_credentials.domain}/#/accounts/"
            f"{self.dbt_cloud_credentials.account_id}/projects/"
            f"{run_data['project_id']}/runs/{run_id}/"
        )
        return run

get_job async

Retrieve information about a dbt Cloud job.

Parameters:

Name	Type	Description	Default
order_by	Optional[str]	The field to order the results by.	None

Returns:

Type	Description
Dict[str, Any]	The job data.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def get_job(self, order_by: Optional[str] = None) -> Dict[str, Any]:
    """
    Retrieve information about a dbt Cloud job.

    Args:
        order_by: The field to order the results by.

    Returns:
        The job data.
    """
    try:
        async with self.dbt_cloud_credentials.get_administrative_client() as client:
            response = await client.get_job(
                job_id=self.job_id,
                order_by=order_by,
            )
    except HTTPStatusError as ex:
        raise DbtCloudGetJobFailed(extract_user_message(ex)) from ex
    return response.json()["data"]

trigger async

Triggers a dbt Cloud job.

Returns:

Type	Description
DbtCloudJobRun	A representation of the dbt Cloud job run.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def trigger(
    self, trigger_job_run_options: Optional[TriggerJobRunOptions] = None
) -> DbtCloudJobRun:
    """
    Triggers a dbt Cloud job.

    Returns:
        A representation of the dbt Cloud job run.
    """
    try:
        trigger_job_run_options = (
            trigger_job_run_options or self.trigger_job_run_options
        )
        async with self.dbt_cloud_credentials.get_administrative_client() as client:
            response = await client.trigger_job_run(
                job_id=self.job_id, options=trigger_job_run_options
            )
    except HTTPStatusError as ex:
        raise DbtCloudJobRunTriggerFailed(extract_user_message(ex)) from ex

    run_data = response.json()["data"]
    run_id = run_data.get("id")
    run = DbtCloudJobRun(
        dbt_cloud_job=self,
        run_id=run_id,
    )
    self.logger.info(
        f"dbt Cloud job {self.job_id} run {run_id} successfully triggered. "
        f"You can view the status of this run at "
        f"https://{self.dbt_cloud_credentials.domain}/#/accounts/"
        f"{self.dbt_cloud_credentials.account_id}/projects/"
        f"{run_data['project_id']}/runs/{run_id}/"
    )
    return run

DbtCloudJobRun

Bases: JobRun

Class that holds the information and methods to interact with the resulting run of a dbt Cloud job.

Source code in prefect_dbt/cloud/jobs.py

class DbtCloudJobRun(JobRun):  # NOT A BLOCK
    """
    Class that holds the information and methods to interact
    with the resulting run of a dbt Cloud job.
    """

    def __init__(self, run_id: int, dbt_cloud_job: "DbtCloudJob"):
        self.run_id = run_id
        self._dbt_cloud_job = dbt_cloud_job
        self._dbt_cloud_credentials = dbt_cloud_job.dbt_cloud_credentials

    @property
    def _log_prefix(self):
        return f"dbt Cloud job {self._dbt_cloud_job.job_id} run {self.run_id}."

    async def _wait_until_state(
        self,
        in_final_state_fn: Awaitable[Callable],
        get_state_fn: Awaitable[Callable],
        log_state_fn: Callable = None,
        timeout_seconds: int = 60,
        interval_seconds: int = 1,
    ):
        """
        Wait until the job run reaches a specific state.

        Args:
            in_final_state_fn: An async function that accepts a run state
                and returns a boolean indicating whether the job run is
                in a final state.
            get_state_fn: An async function that returns
                the current state of the job run.
            log_state_fn: A callable that accepts a run
                state and makes it human readable.
            timeout_seconds: The maximum amount of time, in seconds, to wait
                for the job run to reach the final state.
            interval_seconds: The number of seconds to wait between checks of
                the job run's state.
        """
        start_time = time.time()
        last_state = run_state = None
        while not in_final_state_fn(run_state):
            run_state = await get_state_fn()
            if run_state != last_state:
                if self.logger is not None:
                    self.logger.info(
                        "%s has new state: %s",
                        self._log_prefix,
                        log_state_fn(run_state),
                    )
                last_state = run_state

            elapsed_time_seconds = time.time() - start_time
            if elapsed_time_seconds > timeout_seconds:
                raise DbtCloudJobRunTimedOut(
                    f"Max wait time of {timeout_seconds} "
                    "seconds exceeded while waiting"
                )
            await asyncio.sleep(interval_seconds)

    @sync_compatible
    async def get_run(self) -> Dict[str, Any]:
        """
        Makes a request to the dbt Cloud API to get the run data.

        Returns:
            The run data.
        """
        try:
            dbt_cloud_credentials = self._dbt_cloud_credentials
            async with dbt_cloud_credentials.get_administrative_client() as client:
                response = await client.get_run(self.run_id)
        except HTTPStatusError as ex:
            raise DbtCloudGetRunFailed(extract_user_message(ex)) from ex
        run_data = response.json()["data"]
        return run_data

    @sync_compatible
    async def get_status_code(self) -> int:
        """
        Makes a request to the dbt Cloud API to get the run status.

        Returns:
            The run status code.
        """
        run_data = await self.get_run()
        run_status_code = run_data.get("status")
        return run_status_code

    @sync_compatible
    async def wait_for_completion(self) -> None:
        """
        Waits for the job run to reach a terminal state.
        """
        await self._wait_until_state(
            in_final_state_fn=DbtCloudJobRunStatus.is_terminal_status_code,
            get_state_fn=self.get_status_code,
            log_state_fn=DbtCloudJobRunStatus,
            timeout_seconds=self._dbt_cloud_job.timeout_seconds,
            interval_seconds=self._dbt_cloud_job.interval_seconds,
        )

    @sync_compatible
    async def fetch_result(self, step: Optional[int] = None) -> Dict[str, Any]:
        """
        Gets the results from the job run. Since the results
        may not be ready, use wait_for_completion before calling this method.

        Args:
            step: The index of the step in the run to query for artifacts. The
                first step in the run has the index 1. If the step parameter is
                omitted, then this method will return the artifacts compiled
                for the last step in the run.
        """
        run_data = await self.get_run()
        run_status = DbtCloudJobRunStatus(run_data.get("status"))
        if run_status == DbtCloudJobRunStatus.SUCCESS:
            try:
                async with self._dbt_cloud_credentials.get_administrative_client() as client:  # noqa
                    response = await client.list_run_artifacts(
                        run_id=self.run_id, step=step
                    )
                run_data["artifact_paths"] = response.json()["data"]
                self.logger.info("%s completed successfully!", self._log_prefix)
            except HTTPStatusError as ex:
                raise DbtCloudListRunArtifactsFailed(extract_user_message(ex)) from ex
            return run_data
        elif run_status == DbtCloudJobRunStatus.CANCELLED:
            raise DbtCloudJobRunCancelled(f"{self._log_prefix} was cancelled.")
        elif run_status == DbtCloudJobRunStatus.FAILED:
            raise DbtCloudJobRunFailed(f"{self._log_prefix} has failed.")
        else:
            raise DbtCloudJobRunIncomplete(
                f"{self._log_prefix} is still running; "
                "use wait_for_completion() to wait until results are ready."
            )

    @sync_compatible
    async def get_run_artifacts(
        self,
        path: Literal["manifest.json", "catalog.json", "run_results.json"],
        step: Optional[int] = None,
    ) -> Union[Dict[str, Any], str]:
        """
        Get an artifact generated for a completed run.

        Args:
            path: The relative path to the run artifact.
            step: The index of the step in the run to query for artifacts. The
                first step in the run has the index 1. If the step parameter is
                omitted, then this method will return the artifacts compiled
                for the last step in the run.

        Returns:
            The contents of the requested manifest. Returns a `Dict` if the
                requested artifact is a JSON file and a `str` otherwise.
        """
        try:
            dbt_cloud_credentials = self._dbt_cloud_credentials
            async with dbt_cloud_credentials.get_administrative_client() as client:
                response = await client.get_run_artifact(
                    run_id=self.run_id, path=path, step=step
                )
        except HTTPStatusError as ex:
            raise DbtCloudGetRunArtifactFailed(extract_user_message(ex)) from ex

        if path.endswith(".json"):
            artifact_contents = response.json()
        else:
            artifact_contents = response.text
        return artifact_contents

    def _select_unsuccessful_commands(
        self,
        run_results: List[Dict[str, Any]],
        command_components: List[str],
        command: str,
        exe_command: str,
    ) -> List[str]:
        """
        Select nodes that were not successful and rebuild a command.
        """
        # note "fail" here instead of "cancelled" because
        # nodes do not have a cancelled state
        run_nodes = " ".join(
            run_result["unique_id"].split(".")[2]
            for run_result in run_results
            if run_result["status"] in ("error", "skipped", "fail")
        )

        select_arg = None
        if "-s" in command_components:
            select_arg = "-s"
        elif "--select" in command_components:
            select_arg = "--select"

        # prevent duplicate --select/-s statements
        if select_arg is not None:
            # dbt --fail-fast run, -s, bad_mod --vars '{"env": "prod"}' to:
            # dbt --fail-fast run -s other_mod bad_mod --vars '{"env": "prod"}'
            command_start, select_arg, command_end = command.partition(select_arg)
            modified_command = f"{command_start} {select_arg} {run_nodes} {command_end}"  # noqa
        else:
            # dbt --fail-fast, build, --vars '{"env": "prod"}' to:
            # dbt --fail-fast build --select bad_model --vars '{"env": "prod"}'
            dbt_global_args, exe_command, exe_args = command.partition(exe_command)
            modified_command = (
                f"{dbt_global_args} {exe_command} -s {run_nodes} {exe_args}"
            )
        return modified_command

    async def _build_trigger_job_run_options(
        self,
        job: Dict[str, Any],
        run: Dict[str, Any],
    ) -> TriggerJobRunOptions:
        """
        Compiles a list of steps (commands) to retry, then either build trigger job
        run options from scratch if it does not exist, else overrides the existing.
        """
        generate_docs = job.get("generate_docs", False)
        generate_sources = job.get("generate_sources", False)

        steps_override = []
        for run_step in run["run_steps"]:
            status = run_step["status_humanized"].lower()
            # Skipping cloning, profile setup, and dbt deps - always the first three
            # steps in any run, and note, index starts at 1 instead of 0
            if run_step["index"] <= 3 or status == "success":
                continue
            # get dbt build from "Invoke dbt with `dbt build`"
            command = run_step["name"].partition("`")[2].partition("`")[0]

            # These steps will be re-run regardless if
            # generate_docs or generate_sources are enabled for a given job
            # so if we don't skip, it'll run twice
            freshness_in_command = (
                "dbt source snapshot-freshness" in command
                or "dbt source freshness" in command
            )
            if "dbt docs generate" in command and generate_docs:
                continue
            elif freshness_in_command and generate_sources:
                continue

            # find an executable command like `build` or `run`
            # search in a list so that there aren't false positives, like
            # `"run" in "dbt run-operation"`, which is True; we actually want
            # `"run" in ["dbt", "run-operation"]` which is False
            command_components = shlex.split(command)
            for exe_command in EXE_COMMANDS:
                if exe_command in command_components:
                    break
            else:
                exe_command = ""

            is_exe_command = exe_command in EXE_COMMANDS
            is_not_success = status in ("error", "skipped", "cancelled")
            is_skipped = status == "skipped"
            if (not is_exe_command and is_not_success) or (
                is_exe_command and is_skipped
            ):
                # if no matches like `run-operation`, we will be rerunning entirely
                # or if it's one of the expected commands and is skipped
                steps_override.append(command)
            else:
                # errors and failures are when we need to inspect to figure
                # out the point of failure
                try:
                    run_artifact = await self.get_run_artifacts(
                        "run_results.json", run_step["index"]
                    )
                except JSONDecodeError:
                    # get the run results scoped to the step which had an error
                    # an error here indicates that either:
                    # 1) the fail-fast flag was set, in which case
                    #    the run_results.json file was never created; or
                    # 2) there was a problem on dbt Cloud's side saving
                    #    this artifact
                    steps_override.append(command)
                else:
                    # we only need to find the individual nodes
                    # for those run commands
                    run_results = run_artifact["results"]
                    modified_command = self._select_unsuccessful_commands(
                        run_results=run_results,
                        command_components=command_components,
                        command=command,
                        exe_command=exe_command,
                    )
                    steps_override.append(modified_command)

        if self._dbt_cloud_job.trigger_job_run_options is None:
            trigger_job_run_options_override = TriggerJobRunOptions(
                steps_override=steps_override
            )
        else:
            trigger_job_run_options_override = (
                self._dbt_cloud_job.trigger_job_run_options.copy()
            )
            trigger_job_run_options_override.steps_override = steps_override
        return trigger_job_run_options_override

    @sync_compatible
    async def retry_failed_steps(self) -> "DbtCloudJobRun":  # noqa: F821
        """
        Retries steps that did not complete successfully in a run.

        Returns:
            A representation of the dbt Cloud job run.
        """
        job = await self._dbt_cloud_job.get_job()
        run = await self.get_run()

        trigger_job_run_options_override = await self._build_trigger_job_run_options(
            job=job, run=run
        )

        num_steps = len(trigger_job_run_options_override.steps_override)
        if num_steps == 0:
            self.logger.info(f"{self._log_prefix} does not have any steps to retry.")
        else:
            self.logger.info(f"{self._log_prefix} has {num_steps} steps to retry.")
            run = await self._dbt_cloud_job.trigger(
                trigger_job_run_options=trigger_job_run_options_override,
            )
        return run

fetch_result async

Gets the results from the job run. Since the results may not be ready, use wait_for_completion before calling this method.

Parameters:

Name	Type	Description	Default
step	Optional[int]	The index of the step in the run to query for artifacts. The first step in the run has the index 1. If the step parameter is omitted, then this method will return the artifacts compiled for the last step in the run.	None

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def fetch_result(self, step: Optional[int] = None) -> Dict[str, Any]:
    """
    Gets the results from the job run. Since the results
    may not be ready, use wait_for_completion before calling this method.

    Args:
        step: The index of the step in the run to query for artifacts. The
            first step in the run has the index 1. If the step parameter is
            omitted, then this method will return the artifacts compiled
            for the last step in the run.
    """
    run_data = await self.get_run()
    run_status = DbtCloudJobRunStatus(run_data.get("status"))
    if run_status == DbtCloudJobRunStatus.SUCCESS:
        try:
            async with self._dbt_cloud_credentials.get_administrative_client() as client:  # noqa
                response = await client.list_run_artifacts(
                    run_id=self.run_id, step=step
                )
            run_data["artifact_paths"] = response.json()["data"]
            self.logger.info("%s completed successfully!", self._log_prefix)
        except HTTPStatusError as ex:
            raise DbtCloudListRunArtifactsFailed(extract_user_message(ex)) from ex
        return run_data
    elif run_status == DbtCloudJobRunStatus.CANCELLED:
        raise DbtCloudJobRunCancelled(f"{self._log_prefix} was cancelled.")
    elif run_status == DbtCloudJobRunStatus.FAILED:
        raise DbtCloudJobRunFailed(f"{self._log_prefix} has failed.")
    else:
        raise DbtCloudJobRunIncomplete(
            f"{self._log_prefix} is still running; "
            "use wait_for_completion() to wait until results are ready."
        )

get_run async

Makes a request to the dbt Cloud API to get the run data.

Returns:

Type	Description
Dict[str, Any]	The run data.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def get_run(self) -> Dict[str, Any]:
    """
    Makes a request to the dbt Cloud API to get the run data.

    Returns:
        The run data.
    """
    try:
        dbt_cloud_credentials = self._dbt_cloud_credentials
        async with dbt_cloud_credentials.get_administrative_client() as client:
            response = await client.get_run(self.run_id)
    except HTTPStatusError as ex:
        raise DbtCloudGetRunFailed(extract_user_message(ex)) from ex
    run_data = response.json()["data"]
    return run_data

get_run_artifacts async

Get an artifact generated for a completed run.

Parameters:

Name	Type	Description	Default
path	Literal['manifest.json', 'catalog.json', 'run_results.json']	The relative path to the run artifact.	required
step	Optional[int]	The index of the step in the run to query for artifacts. The first step in the run has the index 1. If the step parameter is omitted, then this method will return the artifacts compiled for the last step in the run.	None

Returns:

Type	Description
Union[Dict[str, Any], str]	The contents of the requested manifest. Returns a Dict if the requested artifact is a JSON file and a str otherwise.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def get_run_artifacts(
    self,
    path: Literal["manifest.json", "catalog.json", "run_results.json"],
    step: Optional[int] = None,
) -> Union[Dict[str, Any], str]:
    """
    Get an artifact generated for a completed run.

    Args:
        path: The relative path to the run artifact.
        step: The index of the step in the run to query for artifacts. The
            first step in the run has the index 1. If the step parameter is
            omitted, then this method will return the artifacts compiled
            for the last step in the run.

    Returns:
        The contents of the requested manifest. Returns a `Dict` if the
            requested artifact is a JSON file and a `str` otherwise.
    """
    try:
        dbt_cloud_credentials = self._dbt_cloud_credentials
        async with dbt_cloud_credentials.get_administrative_client() as client:
            response = await client.get_run_artifact(
                run_id=self.run_id, path=path, step=step
            )
    except HTTPStatusError as ex:
        raise DbtCloudGetRunArtifactFailed(extract_user_message(ex)) from ex

    if path.endswith(".json"):
        artifact_contents = response.json()
    else:
        artifact_contents = response.text
    return artifact_contents

get_status_code async

Makes a request to the dbt Cloud API to get the run status.

Returns:

Type	Description
int	The run status code.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def get_status_code(self) -> int:
    """
    Makes a request to the dbt Cloud API to get the run status.

    Returns:
        The run status code.
    """
    run_data = await self.get_run()
    run_status_code = run_data.get("status")
    return run_status_code

retry_failed_steps async

Retries steps that did not complete successfully in a run.

Returns:

Type	Description
DbtCloudJobRun	A representation of the dbt Cloud job run.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def retry_failed_steps(self) -> "DbtCloudJobRun":  # noqa: F821
    """
    Retries steps that did not complete successfully in a run.

    Returns:
        A representation of the dbt Cloud job run.
    """
    job = await self._dbt_cloud_job.get_job()
    run = await self.get_run()

    trigger_job_run_options_override = await self._build_trigger_job_run_options(
        job=job, run=run
    )

    num_steps = len(trigger_job_run_options_override.steps_override)
    if num_steps == 0:
        self.logger.info(f"{self._log_prefix} does not have any steps to retry.")
    else:
        self.logger.info(f"{self._log_prefix} has {num_steps} steps to retry.")
        run = await self._dbt_cloud_job.trigger(
            trigger_job_run_options=trigger_job_run_options_override,
        )
    return run

wait_for_completion async

Waits for the job run to reach a terminal state.

Source code in prefect_dbt/cloud/jobs.py

@sync_compatible
async def wait_for_completion(self) -> None:
    """
    Waits for the job run to reach a terminal state.
    """
    await self._wait_until_state(
        in_final_state_fn=DbtCloudJobRunStatus.is_terminal_status_code,
        get_state_fn=self.get_status_code,
        log_state_fn=DbtCloudJobRunStatus,
        timeout_seconds=self._dbt_cloud_job.timeout_seconds,
        interval_seconds=self._dbt_cloud_job.interval_seconds,
    )

get_dbt_cloud_job_info async

A task to retrieve information about a dbt Cloud job.

Parameters:

Name	Type	Description	Default
dbt_cloud_credentials	DbtCloudCredentials	Credentials for authenticating with dbt Cloud.	required
job_id	int	The ID of the job to get.	required

Returns:

Type	Description
Dict	The job data returned by the dbt Cloud administrative API.

Example

Get status of a dbt Cloud job:

from prefect import flow

from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import get_job

@flow
def get_job_flow():
    credentials = DbtCloudCredentials(api_key="my_api_key", account_id=123456789)

    return get_job(
        dbt_cloud_credentials=credentials,
        job_id=42
    )

get_job_flow()

Source code in prefect_dbt/cloud/jobs.py

@task(
    name="Get dbt Cloud job details",
    description="Retrieves details of a dbt Cloud job "
    "for the job with the given job_id.",
    retries=3,
    retry_delay_seconds=10,
)
async def get_dbt_cloud_job_info(
    dbt_cloud_credentials: DbtCloudCredentials,
    job_id: int,
    order_by: Optional[str] = None,
) -> Dict:
    """
    A task to retrieve information about a dbt Cloud job.

    Args:
        dbt_cloud_credentials: Credentials for authenticating with dbt Cloud.
        job_id: The ID of the job to get.

    Returns:
        The job data returned by the dbt Cloud administrative API.

    Example:
        Get status of a dbt Cloud job:
        ```python
        from prefect import flow

        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import get_job

        @flow
        def get_job_flow():
            credentials = DbtCloudCredentials(api_key="my_api_key", account_id=123456789)

            return get_job(
                dbt_cloud_credentials=credentials,
                job_id=42
            )

        get_job_flow()
        ```
    """  # noqa
    try:
        async with dbt_cloud_credentials.get_administrative_client() as client:
            response = await client.get_job(
                job_id=job_id,
                order_by=order_by,
            )
    except HTTPStatusError as ex:
        raise DbtCloudGetJobFailed(extract_user_message(ex)) from ex
    return response.json()["data"]

get_run_id

Task that extracts the run ID from a trigger job run API response,

This task is mainly used to maintain dependency tracking between the trigger_dbt_cloud_job_run task and downstream tasks/flows that use the run ID.

Parameters:

Name	Type	Description	Default
obj	Dict	The JSON body from the trigger job run response.	required

Example

from prefect import flow
from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run, get_run_id


@flow
def trigger_run_and_get_id():
    dbt_cloud_credentials=DbtCloudCredentials(
            api_key="my_api_key",
            account_id=123456789
        )

    triggered_run_data = trigger_dbt_cloud_job_run(
        dbt_cloud_credentials=dbt_cloud_credentials,
        job_id=job_id,
        options=trigger_job_run_options,
    )
    run_id = get_run_id.submit(triggered_run_data)
    return run_id

trigger_run_and_get_id()

Source code in prefect_dbt/cloud/jobs.py

@task(
    name="Get dbt Cloud job run ID",
    description="Extracts the run ID from a trigger job run API response",
)
def get_run_id(obj: Dict):
    """
    Task that extracts the run ID from a trigger job run API response,

    This task is mainly used to maintain dependency tracking between the
    `trigger_dbt_cloud_job_run` task and downstream tasks/flows that use the run ID.

    Args:
        obj: The JSON body from the trigger job run response.

    Example:
        ```python
        from prefect import flow
        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run, get_run_id


        @flow
        def trigger_run_and_get_id():
            dbt_cloud_credentials=DbtCloudCredentials(
                    api_key="my_api_key",
                    account_id=123456789
                )

            triggered_run_data = trigger_dbt_cloud_job_run(
                dbt_cloud_credentials=dbt_cloud_credentials,
                job_id=job_id,
                options=trigger_job_run_options,
            )
            run_id = get_run_id.submit(triggered_run_data)
            return run_id

        trigger_run_and_get_id()
        ```
    """
    id = obj.get("id")
    if id is None:
        raise RuntimeError("Unable to determine run ID for triggered job.")
    return id

retry_dbt_cloud_job_run_subset_and_wait_for_completion async

Flow that retrys a subset of dbt Cloud job run, filtered by select statuses, and waits for the triggered retry to complete.

Parameters:

Name	Type	Description	Default
dbt_cloud_credentials	DbtCloudCredentials	Credentials for authenticating with dbt Cloud.	required
trigger_job_run_options	Optional[TriggerJobRunOptions]	An optional TriggerJobRunOptions instance to specify overrides for the triggered job run.	None
max_wait_seconds	int	Maximum number of seconds to wait for job to complete	900
poll_frequency_seconds	int	Number of seconds to wait in between checks for run completion.	10
run_id	int	The ID of the job run to retry.	required

Raises:

Type	Description
ValueError	If trigger_job_run_options.steps_override is set by the user.

Returns:

Type	Description
Dict	The run data returned by the dbt Cloud administrative API.

Examples:

Retry a subset of models in a dbt Cloud job run and wait for completion:

from prefect import flow

from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import retry_dbt_cloud_job_run_subset_and_wait_for_completion

@flow
def retry_dbt_cloud_job_run_subset_and_wait_for_completion_flow():
    credentials = DbtCloudCredentials.load("MY_BLOCK_NAME")
    retry_dbt_cloud_job_run_subset_and_wait_for_completion(
        dbt_cloud_credentials=credentials,
        run_id=88640123,
    )

retry_dbt_cloud_job_run_subset_and_wait_for_completion_flow()

Source code in prefect_dbt/cloud/jobs.py

@flow(
    name="Retry subset of dbt Cloud job run and wait for completion",
    description=(
        "Retries a subset of dbt Cloud job run, filtered by select statuses, "
        "and waits for the triggered retry to complete."
    ),
)
async def retry_dbt_cloud_job_run_subset_and_wait_for_completion(
    dbt_cloud_credentials: DbtCloudCredentials,
    run_id: int,
    trigger_job_run_options: Optional[TriggerJobRunOptions] = None,
    max_wait_seconds: int = 900,
    poll_frequency_seconds: int = 10,
) -> Dict:
    """
    Flow that retrys a subset of dbt Cloud job run, filtered by select statuses,
    and waits for the triggered retry to complete.

    Args:
        dbt_cloud_credentials: Credentials for authenticating with dbt Cloud.
        trigger_job_run_options: An optional TriggerJobRunOptions instance to
            specify overrides for the triggered job run.
        max_wait_seconds: Maximum number of seconds to wait for job to complete
        poll_frequency_seconds: Number of seconds to wait in between checks for
            run completion.
        run_id: The ID of the job run to retry.

    Raises:
        ValueError: If `trigger_job_run_options.steps_override` is set by the user.

    Returns:
        The run data returned by the dbt Cloud administrative API.

    Examples:
        Retry a subset of models in a dbt Cloud job run and wait for completion:
        ```python
        from prefect import flow

        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import retry_dbt_cloud_job_run_subset_and_wait_for_completion

        @flow
        def retry_dbt_cloud_job_run_subset_and_wait_for_completion_flow():
            credentials = DbtCloudCredentials.load("MY_BLOCK_NAME")
            retry_dbt_cloud_job_run_subset_and_wait_for_completion(
                dbt_cloud_credentials=credentials,
                run_id=88640123,
            )

        retry_dbt_cloud_job_run_subset_and_wait_for_completion_flow()
        ```
    """  # noqa
    if trigger_job_run_options and trigger_job_run_options.steps_override is not None:
        raise ValueError(
            "Do not set `steps_override` in `trigger_job_run_options` "
            "because this flow will automatically set it"
        )

    run_info_future = await get_dbt_cloud_run_info.submit(
        dbt_cloud_credentials=dbt_cloud_credentials,
        run_id=run_id,
        include_related=["run_steps"],
    )
    run_info = await run_info_future.result()

    job_id = run_info["job_id"]
    job_info_future = await get_dbt_cloud_job_info.submit(
        dbt_cloud_credentials=dbt_cloud_credentials,
        job_id=job_id,
    )
    job_info = await job_info_future.result()

    trigger_job_run_options_override = await _build_trigger_job_run_options(
        dbt_cloud_credentials=dbt_cloud_credentials,
        trigger_job_run_options=trigger_job_run_options,
        run_id=run_id,
        run_info=run_info,
        job_info=job_info,
    )

    # to circumvent `RuntimeError: The task runner is already started!`
    flow_run_context = FlowRunContext.get()
    task_runner_type = type(flow_run_context.task_runner)

    run_data = await trigger_dbt_cloud_job_run_and_wait_for_completion.with_options(
        task_runner=task_runner_type()
    )(
        dbt_cloud_credentials=dbt_cloud_credentials,
        job_id=job_id,
        retry_filtered_models_attempts=0,
        trigger_job_run_options=trigger_job_run_options_override,
        max_wait_seconds=max_wait_seconds,
        poll_frequency_seconds=poll_frequency_seconds,
    )
    return run_data

run_dbt_cloud_job async

Flow that triggers and waits for a dbt Cloud job run, retrying a subset of failed nodes if necessary.

Parameters:

Name	Type	Description	Default
dbt_cloud_job	DbtCloudJob	Block that holds the information and methods to interact with a dbt Cloud job.	required
targeted_retries	int	The number of times to retry failed steps.	3

Examples:

from prefect import flow
from prefect_dbt.cloud import DbtCloudCredentials, DbtCloudJob
from prefect_dbt.cloud.jobs import run_dbt_cloud_job

@flow
def run_dbt_cloud_job_flow():
    dbt_cloud_credentials = DbtCloudCredentials.load("dbt-token")
    dbt_cloud_job = DbtCloudJob(
        dbt_cloud_credentials=dbt_cloud_credentials, job_id=154217
    )
    return run_dbt_cloud_job(dbt_cloud_job=dbt_cloud_job)

run_dbt_cloud_job_flow()

Source code in prefect_dbt/cloud/jobs.py

@flow
async def run_dbt_cloud_job(
    dbt_cloud_job: DbtCloudJob,
    targeted_retries: int = 3,
) -> Dict[str, Any]:
    """
    Flow that triggers and waits for a dbt Cloud job run, retrying a
    subset of failed nodes if necessary.

    Args:
        dbt_cloud_job: Block that holds the information and
            methods to interact with a dbt Cloud job.
        targeted_retries: The number of times to retry failed steps.

    Examples:
        ```python
        from prefect import flow
        from prefect_dbt.cloud import DbtCloudCredentials, DbtCloudJob
        from prefect_dbt.cloud.jobs import run_dbt_cloud_job

        @flow
        def run_dbt_cloud_job_flow():
            dbt_cloud_credentials = DbtCloudCredentials.load("dbt-token")
            dbt_cloud_job = DbtCloudJob(
                dbt_cloud_credentials=dbt_cloud_credentials, job_id=154217
            )
            return run_dbt_cloud_job(dbt_cloud_job=dbt_cloud_job)

        run_dbt_cloud_job_flow()
        ```
    """
    logger = get_run_logger()

    run = await task(dbt_cloud_job.trigger.aio)(dbt_cloud_job)
    while targeted_retries > 0:
        try:
            await task(run.wait_for_completion.aio)(run)
            result = await task(run.fetch_result.aio)(run)
            return result
        except DbtCloudJobRunFailed:
            logger.info(
                f"Retrying job run with ID: {run.run_id} "
                f"{targeted_retries} more times"
            )
            run = await task(run.retry_failed_steps.aio)(run)
            targeted_retries -= 1

trigger_dbt_cloud_job_run async

A task to trigger a dbt Cloud job run.

Parameters:

Name	Type	Description	Default
dbt_cloud_credentials	DbtCloudCredentials	Credentials for authenticating with dbt Cloud.	required
job_id	int	The ID of the job to trigger.	required
options	Optional[TriggerJobRunOptions]	An optional TriggerJobRunOptions instance to specify overrides for the triggered job run.	None

Returns:

Type	Description
Dict	The run data returned from the dbt Cloud administrative API.

Examples:

Trigger a dbt Cloud job run:

from prefect import flow

from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run


@flow
def trigger_dbt_cloud_job_run_flow():
    credentials = DbtCloudCredentials(api_key="my_api_key", account_id=123456789)

    trigger_dbt_cloud_job_run(dbt_cloud_credentials=credentials, job_id=1)


trigger_dbt_cloud_job_run_flow()

Trigger a dbt Cloud job run with overrides:

from prefect import flow

from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run
from prefect_dbt.cloud.models import TriggerJobRunOptions


@flow
def trigger_dbt_cloud_job_run_flow():
    credentials = DbtCloudCredentials(api_key="my_api_key", account_id=123456789)

    trigger_dbt_cloud_job_run(
        dbt_cloud_credentials=credentials,
        job_id=1,
        options=TriggerJobRunOptions(
            git_branch="staging",
            schema_override="dbt_cloud_pr_123",
            dbt_version_override="0.18.0",
            target_name_override="staging",
            timeout_seconds_override=3000,
            generate_docs_override=True,
            threads_override=8,
            steps_override=[
                "dbt seed",
                "dbt run --fail-fast",
                "dbt test --fail-fast",
            ],
        ),
    )


trigger_dbt_cloud_job_run()

Source code in prefect_dbt/cloud/jobs.py

@task(
    name="Trigger dbt Cloud job run",
    description="Triggers a dbt Cloud job run for the job "
    "with the given job_id and optional overrides.",
    retries=3,
    retry_delay_seconds=10,
)
async def trigger_dbt_cloud_job_run(
    dbt_cloud_credentials: DbtCloudCredentials,
    job_id: int,
    options: Optional[TriggerJobRunOptions] = None,
) -> Dict:
    """
    A task to trigger a dbt Cloud job run.

    Args:
        dbt_cloud_credentials: Credentials for authenticating with dbt Cloud.
        job_id: The ID of the job to trigger.
        options: An optional TriggerJobRunOptions instance to specify overrides
            for the triggered job run.

    Returns:
        The run data returned from the dbt Cloud administrative API.

    Examples:
        Trigger a dbt Cloud job run:
        ```python
        from prefect import flow

        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run


        @flow
        def trigger_dbt_cloud_job_run_flow():
            credentials = DbtCloudCredentials(api_key="my_api_key", account_id=123456789)

            trigger_dbt_cloud_job_run(dbt_cloud_credentials=credentials, job_id=1)


        trigger_dbt_cloud_job_run_flow()
        ```

        Trigger a dbt Cloud job run with overrides:
        ```python
        from prefect import flow

        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run
        from prefect_dbt.cloud.models import TriggerJobRunOptions


        @flow
        def trigger_dbt_cloud_job_run_flow():
            credentials = DbtCloudCredentials(api_key="my_api_key", account_id=123456789)

            trigger_dbt_cloud_job_run(
                dbt_cloud_credentials=credentials,
                job_id=1,
                options=TriggerJobRunOptions(
                    git_branch="staging",
                    schema_override="dbt_cloud_pr_123",
                    dbt_version_override="0.18.0",
                    target_name_override="staging",
                    timeout_seconds_override=3000,
                    generate_docs_override=True,
                    threads_override=8,
                    steps_override=[
                        "dbt seed",
                        "dbt run --fail-fast",
                        "dbt test --fail-fast",
                    ],
                ),
            )


        trigger_dbt_cloud_job_run()
        ```
    """  # noqa
    logger = get_run_logger()

    logger.info(f"Triggering run for job with ID {job_id}")

    try:
        async with dbt_cloud_credentials.get_administrative_client() as client:
            response = await client.trigger_job_run(job_id=job_id, options=options)
    except HTTPStatusError as ex:
        raise DbtCloudJobRunTriggerFailed(extract_user_message(ex)) from ex

    run_data = response.json()["data"]

    if "project_id" in run_data and "id" in run_data:
        logger.info(
            f"Run successfully triggered for job with ID {job_id}. "
            "You can view the status of this run at "
            f"https://{dbt_cloud_credentials.domain}/#/accounts/"
            f"{dbt_cloud_credentials.account_id}/projects/{run_data['project_id']}/"
            f"runs/{run_data['id']}/"
        )

    return run_data

trigger_dbt_cloud_job_run_and_wait_for_completion async

Flow that triggers a job run and waits for the triggered run to complete.

Parameters:

Name	Type	Description	Default
dbt_cloud_credentials	DbtCloudCredentials	Credentials for authenticating with dbt Cloud.	required
job_id	int	The ID of the job to trigger.	required
trigger_job_run_options	Optional[TriggerJobRunOptions]	An optional TriggerJobRunOptions instance to specify overrides for the triggered job run.	None
max_wait_seconds	int	Maximum number of seconds to wait for job to complete	900
poll_frequency_seconds	int	Number of seconds to wait in between checks for run completion.	10
retry_filtered_models_attempts	int	Number of times to retry models selected by retry_status_filters.	3

Raises:

Type	Description
DbtCloudJobRunCancelled	The triggered dbt Cloud job run was cancelled.
DbtCloudJobRunFailed	The triggered dbt Cloud job run failed.
RuntimeError	The triggered dbt Cloud job run ended in an unexpected state.

Returns:

Type	Description
Dict	The run data returned by the dbt Cloud administrative API.

Examples:

Trigger a dbt Cloud job and wait for completion as a stand alone flow:

import asyncio
from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run_and_wait_for_completion

asyncio.run(
    trigger_dbt_cloud_job_run_and_wait_for_completion(
        dbt_cloud_credentials=DbtCloudCredentials(
            api_key="my_api_key",
            account_id=123456789
        ),
        job_id=1
    )
)

Trigger a dbt Cloud job and wait for completion as a sub-flow:

from prefect import flow
from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run_and_wait_for_completion

@flow
def my_flow():
    ...
    run_result = trigger_dbt_cloud_job_run_and_wait_for_completion(
        dbt_cloud_credentials=DbtCloudCredentials(
            api_key="my_api_key",
            account_id=123456789
        ),
        job_id=1
    )
    ...

my_flow()

Trigger a dbt Cloud job with overrides:

import asyncio
from prefect_dbt.cloud import DbtCloudCredentials
from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run_and_wait_for_completion
from prefect_dbt.cloud.models import TriggerJobRunOptions

asyncio.run(
    trigger_dbt_cloud_job_run_and_wait_for_completion(
        dbt_cloud_credentials=DbtCloudCredentials(
            api_key="my_api_key",
            account_id=123456789
        ),
        job_id=1,
        trigger_job_run_options=TriggerJobRunOptions(
            git_branch="staging",
            schema_override="dbt_cloud_pr_123",
            dbt_version_override="0.18.0",
            target_name_override="staging",
            timeout_seconds_override=3000,
            generate_docs_override=True,
            threads_override=8,
            steps_override=[
                "dbt seed",
                "dbt run --fail-fast",
                "dbt test --fail fast",
            ],
        ),
    )
)

Source code in prefect_dbt/cloud/jobs.py

@flow(
    name="Trigger dbt Cloud job run and wait for completion",
    description="Triggers a dbt Cloud job run and waits for the"
    "triggered run to complete.",
)
async def trigger_dbt_cloud_job_run_and_wait_for_completion(
    dbt_cloud_credentials: DbtCloudCredentials,
    job_id: int,
    trigger_job_run_options: Optional[TriggerJobRunOptions] = None,
    max_wait_seconds: int = 900,
    poll_frequency_seconds: int = 10,
    retry_filtered_models_attempts: int = 3,
) -> Dict:
    """
    Flow that triggers a job run and waits for the triggered run to complete.

    Args:
        dbt_cloud_credentials: Credentials for authenticating with dbt Cloud.
        job_id: The ID of the job to trigger.
        trigger_job_run_options: An optional TriggerJobRunOptions instance to
            specify overrides for the triggered job run.
        max_wait_seconds: Maximum number of seconds to wait for job to complete
        poll_frequency_seconds: Number of seconds to wait in between checks for
            run completion.
        retry_filtered_models_attempts: Number of times to retry models selected by `retry_status_filters`.

    Raises:
        DbtCloudJobRunCancelled: The triggered dbt Cloud job run was cancelled.
        DbtCloudJobRunFailed: The triggered dbt Cloud job run failed.
        RuntimeError: The triggered dbt Cloud job run ended in an unexpected state.

    Returns:
        The run data returned by the dbt Cloud administrative API.

    Examples:
        Trigger a dbt Cloud job and wait for completion as a stand alone flow:
        ```python
        import asyncio
        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run_and_wait_for_completion

        asyncio.run(
            trigger_dbt_cloud_job_run_and_wait_for_completion(
                dbt_cloud_credentials=DbtCloudCredentials(
                    api_key="my_api_key",
                    account_id=123456789
                ),
                job_id=1
            )
        )
        ```

        Trigger a dbt Cloud job and wait for completion as a sub-flow:
        ```python
        from prefect import flow
        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run_and_wait_for_completion

        @flow
        def my_flow():
            ...
            run_result = trigger_dbt_cloud_job_run_and_wait_for_completion(
                dbt_cloud_credentials=DbtCloudCredentials(
                    api_key="my_api_key",
                    account_id=123456789
                ),
                job_id=1
            )
            ...

        my_flow()
        ```

        Trigger a dbt Cloud job with overrides:
        ```python
        import asyncio
        from prefect_dbt.cloud import DbtCloudCredentials
        from prefect_dbt.cloud.jobs import trigger_dbt_cloud_job_run_and_wait_for_completion
        from prefect_dbt.cloud.models import TriggerJobRunOptions

        asyncio.run(
            trigger_dbt_cloud_job_run_and_wait_for_completion(
                dbt_cloud_credentials=DbtCloudCredentials(
                    api_key="my_api_key",
                    account_id=123456789
                ),
                job_id=1,
                trigger_job_run_options=TriggerJobRunOptions(
                    git_branch="staging",
                    schema_override="dbt_cloud_pr_123",
                    dbt_version_override="0.18.0",
                    target_name_override="staging",
                    timeout_seconds_override=3000,
                    generate_docs_override=True,
                    threads_override=8,
                    steps_override=[
                        "dbt seed",
                        "dbt run --fail-fast",
                        "dbt test --fail fast",
                    ],
                ),
            )
        )
        ```
    """  # noqa
    logger = get_run_logger()

    triggered_run_data_future = await trigger_dbt_cloud_job_run.submit(
        dbt_cloud_credentials=dbt_cloud_credentials,
        job_id=job_id,
        options=trigger_job_run_options,
    )
    run_id = (await triggered_run_data_future.result()).get("id")
    if run_id is None:
        raise RuntimeError("Unable to determine run ID for triggered job.")

    final_run_status, run_data = await wait_for_dbt_cloud_job_run(
        run_id=run_id,
        dbt_cloud_credentials=dbt_cloud_credentials,
        max_wait_seconds=max_wait_seconds,
        poll_frequency_seconds=poll_frequency_seconds,
    )

    if final_run_status == DbtCloudJobRunStatus.SUCCESS:
        try:
            list_run_artifacts_future = await list_dbt_cloud_run_artifacts.submit(
                dbt_cloud_credentials=dbt_cloud_credentials,
                run_id=run_id,
            )
            run_data["artifact_paths"] = await list_run_artifacts_future.result()
        except DbtCloudListRunArtifactsFailed as ex:
            logger.warning(
                "Unable to retrieve artifacts for job run with ID %s. Reason: %s",
                run_id,
                ex,
            )
        logger.info(
            "dbt Cloud job run with ID %s completed successfully!",
            run_id,
        )
        return run_data
    elif final_run_status == DbtCloudJobRunStatus.CANCELLED:
        raise DbtCloudJobRunCancelled(
            f"Triggered job run with ID {run_id} was cancelled."
        )
    elif final_run_status == DbtCloudJobRunStatus.FAILED:
        while retry_filtered_models_attempts > 0:
            logger.info(
                f"Retrying job run with ID: {run_id} "
                f"{retry_filtered_models_attempts} more times"
            )
            try:
                retry_filtered_models_attempts -= 1
                run_data = await retry_dbt_cloud_job_run_subset_and_wait_for_completion(
                    dbt_cloud_credentials=dbt_cloud_credentials,
                    run_id=run_id,
                    trigger_job_run_options=trigger_job_run_options,
                    max_wait_seconds=max_wait_seconds,
                    poll_frequency_seconds=poll_frequency_seconds,
                )
                return run_data
            except Exception:
                pass
        else:
            raise DbtCloudJobRunFailed(f"Triggered job run with ID: {run_id} failed.")
    else:
        raise RuntimeError(
            f"Triggered job run with ID: {run_id} ended with unexpected"
            f"status {final_run_status.value}."
        )