Files
dify/api/services/dify_builder/native_testing/state_store.py
T

4059 lines
195 KiB
Python

"""Protected native snapshots with a storage-readback + SQL acknowledgement barrier.
The native adapter owns serialization. This owner stores immutable bytes, never
runs Graphon, and never treats a caller's object reference as authorization.
All key-provider and object-storage work happens outside SQL transactions.
"""
import json
import logging
from collections.abc import Generator
from contextlib import contextmanager, suppress
from contextvars import ContextVar
from dataclasses import dataclass, field, replace
from time import perf_counter
from typing import TYPE_CHECKING, Literal
from uuid import UUID, uuid4
from pydantic import TypeAdapter
from sqlalchemy import select
from core.helper import encrypter
from extensions.ext_storage import storage
from models.agent import (
Agent,
AgentConfigSnapshot,
AgentConfigVersionKind,
AgentHomeSnapshot,
AgentWorkingResourceStatus,
AgentWorkspace,
AgentWorkspaceBinding,
AgentWorkspaceOwnerType,
WorkflowAgentBindingType,
WorkflowAgentNodeBinding,
)
from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig
from models.dify_builder import DifyBuilderExecutionRequest, DifyBuilderTestInput
from models.dify_builder_native_tests import (
NativeContinuation,
NativeEffectOperation,
NativeInvocationState,
NativePayloadObject,
NativeTestInvocation,
NativeTestScope,
)
from models.workflow import Workflow
from runtime_contracts.builder.execution import (
BuilderExecutionPolicyError,
canonical_digest,
)
from runtime_contracts.builder.native.agent_contracts import (
AgentCreateClaim,
AgentCreateExisting,
AgentCreateInserted,
AgentDefinitionReceipt,
AgentNodeAssociation,
AgentParentNodeRef,
AgentPreparedConfig,
AgentProtectedState,
AgentTestWorkspaceRef,
AgentWaitAssociation,
)
from runtime_contracts.builder.native.contracts import (
ID,
CheckpointRef,
Counter,
FiniteJSON,
JsonValue,
NativeTestContextRef,
OperationKey,
operation_digest,
)
from runtime_contracts.builder.native.runtime_contracts import (
NativeCheckpointPayload,
ResolvedNativeTestContext,
)
from services.dify_builder.native_testing.private_reads import (
PrivateReadOwner,
validate_invocation_workflow,
validate_private_read,
)
from services.dify_builder.native_testing.repository import (
OwnedNativeTestLoader,
admitted_graph_matches,
admitted_workflow_revision,
native_workflow_revision,
utc_now,
)
from services.dify_builder.native_testing.transitions import require_ancestor_authority
_JSON = TypeAdapter(FiniteJSON)
logger = logging.getLogger(__name__)
PayloadKind = Literal["context", "checkpoint", "result"]
if TYPE_CHECKING:
from dify_agent.protocol.confirmed_recovery import AgentRecoveryBinding
from dify_agent.protocol.snapshot import SessionSnapshot
from core.workflow.nodes.agent_v2.session_store import WorkflowAgentSessionScope
from services.dify_builder.native_testing.agent_recovery import (
ConfirmedSnapshotIdentity,
PreparedRecoverySelection,
RecoveryAllocationReceipt,
SelectedRecovery,
)
from services.dify_builder.native_testing.agent_recovery_state import AgentRecoveryRecord
def encode_payload(
*, tenant_id: str, request_id: str, invocation_id: str, kind: str, payload: JsonValue
) -> tuple[str, str]:
value = _JSON.validate_python(payload)
envelope = {
"schema": "1",
"tenant_id": tenant_id,
"request_id": request_id,
"invocation_id": invocation_id,
"kind": kind,
"payload": value,
}
return json.dumps(
envelope, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False
), canonical_digest(envelope)
def decode_payload(
raw: str, *, tenant_id: str, request_id: str, invocation_id: str, kind: str, digest: str
) -> JsonValue:
try:
envelope = _JSON.validate_python(json.loads(raw))
if not isinstance(envelope, dict) or set(envelope) != {
"schema",
"tenant_id",
"request_id",
"invocation_id",
"kind",
"payload",
}:
raise ValueError("envelope")
expected = {
"schema": "1",
"tenant_id": tenant_id,
"request_id": request_id,
"invocation_id": invocation_id,
"kind": kind,
}
if any(envelope.get(k) != v for k, v in expected.items()) or canonical_digest(envelope) != digest:
raise ValueError("ownership or integrity")
return envelope["payload"]
except (ValueError, TypeError):
raise BuilderExecutionPolicyError("native_payload_invalid") from None
def owner_query(model, ref: NativeTestContextRef, tenant_id: str):
return select(model).where(
model.tenant_id == tenant_id, model.request_id == ref.request_id, model.invocation_id == ref.invocation_id
)
def invocation_state(session, ref: NativeTestContextRef, tenant_id: str) -> NativeInvocationState:
state = session.scalar(owner_query(NativeInvocationState, ref, tenant_id))
if state is None:
raise BuilderExecutionPolicyError("native_state_unavailable")
return state
@dataclass(frozen=True)
class ObjectReceipt:
id: str
tenant_id: str
request_id: str
invocation_id: str
kind: str
logical_key: str
storage_key: str
payload_digest: str
value_digest: str
@classmethod
def from_payload_object(cls, row: NativePayloadObject) -> "ObjectReceipt":
"""Detach the persisted payload identity from its SQLAlchemy session."""
return cls(
id=row.id,
tenant_id=row.tenant_id,
request_id=row.request_id,
invocation_id=row.invocation_id,
kind=row.kind,
logical_key=row.logical_key,
storage_key=row.storage_key,
payload_digest=row.payload_digest,
value_digest=row.value_digest,
)
_PAYLOAD_READ_MAX_ENTRIES = 64
_PAYLOAD_READ_MAX_BYTES = 8 * 1024 * 1024
@dataclass
class _PayloadReads:
encrypted: dict[tuple[ObjectReceipt, int | None], bytes] = field(default_factory=dict)
byte_count: int = 0
active: bool = True
_payload_reads: ContextVar[_PayloadReads | None] = ContextVar("native_payload_reads", default=None)
@contextmanager
def native_payload_read_scope() -> Generator[None]:
"""Reuse exact encrypted objects only during one synchronous native command.
Current owner/permission/resource checks and every decrypt/decode still run.
Nested owner calls share the outer scope; its finally clears even inherited
references. No plaintext, authority, expiry or cross-command reuse is stored.
"""
inherited = _payload_reads.get()
if inherited is not None and inherited.active:
yield
return
reads = _PayloadReads()
token = _payload_reads.set(reads)
try:
yield
finally:
reads.active = False
reads.encrypted.clear()
reads.byte_count = 0
_payload_reads.reset(token)
@dataclass
class _AgentPreparation:
parent: AgentParentNodeRef
context: ResolvedNativeTestContext
authority: PrivateReadOwner
workspace: AgentTestWorkspaceRef
resource_digest: str
objects: list[ObjectReceipt]
settlement: bool = False
for_projection: bool = False
recovery: "SelectedRecovery | None" = None
@contextmanager
def _agent_boundary():
try:
yield
except BuilderExecutionPolicyError:
raise
except Exception:
raise BuilderExecutionPolicyError("native_agent_state_unavailable") from None
class NativeExecutionStateStore:
def __init__(self, loader: OwnedNativeTestLoader):
self.loader = loader
@staticmethod
def recovery_config_request(*, config, parent, create_idempotency_key: str | None):
"""Rebuild the credentials-free executable request for SDK pure validation."""
from dify_agent.protocol.schemas import CreateRunRequest
if create_idempotency_key is None:
raise BuilderExecutionPolicyError("native_agent_recovery_checkpoint_unsupported")
return CreateRunRequest.model_validate(
{
**config,
"idempotency_key": create_idempotency_key,
"execution_mode": "builder_test",
"native_test_profile": "confirmed",
"native_test_ref": parent.ref.model_dump(mode="json"),
"native_test_protocol_version": 1,
"native_parent_node": parent.model_dump(mode="json"),
}
)
@staticmethod
def recovery_home_digest(home: AgentHomeSnapshot) -> str:
"""Bind immutable SQL Home identity/ref; this is not a claim about backend bytes."""
return canonical_digest(
{"id": home.id, "tenant_id": home.tenant_id, "agent_id": home.agent_id, "snapshot_ref": home.snapshot_ref}
)
def prepare_recovery_attempt(self, *, ref, expected):
"""Reload exact retained predecessor and acknowledged evidence outside the writer.
Parsing a supplied record grants nothing. Full owner/config/input chain
and protected bytes are reconstructed, then rechecked by the caller.
"""
from dify_agent.protocol import InspectExecutionBindingResponse
from dify_agent.protocol.confirmed_recovery import AgentRecoveryBinding
from dify_agent.protocol.native_continuation import AgentCheckpoint
evidence = self.load_result(ref=ref, result_ref=expected.evidence_object_ref)
if not isinstance(evidence, dict) or set(evidence) != {
"parent_node",
"wait",
"parent_checkpoint",
"pause_id",
"pause_state_object_key",
"native_run_id",
"inspection",
"agent_run_id",
"pending_ids",
"allocation_owner",
}:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
parent = AgentParentNodeRef.model_validate(evidence["parent_node"])
wait = AgentWaitAssociation.model_validate(evidence["wait"])
inspection = InspectExecutionBindingResponse.model_validate(evidence["inspection"])
prepared = self._agent_prepare(
ref=ref, parent_node=parent, writing=False, expected_generation=expected.parent_generation
)
association = self._agent_binding(prepared, expected.predecessor.association_object_ref)
config = self._agent_prepared_config(prepared, expected.predecessor.config_ref)
workspace = prepared.workspace
predecessor = AgentRecoveryBinding(
binding_id=workspace.binding_id,
workspace_id=workspace.workspace_id,
binding_ref=workspace.backend_binding_ref,
workspace_ref=workspace.backend_workspace_ref,
workspace_generation=workspace.workspace_generation,
association_object_ref=expected.predecessor.association_object_ref,
config_ref=config.config_ref,
config_digest=config.config_digest,
)
if (
predecessor,
parent.execution_id,
parent.input_digest,
prepared.resource_digest,
wait.parent_node,
wait.association_object_ref,
wait.agent_generation,
wait.checkpoint_object_ref,
wait.checkpoint_digest,
association.agent_run_id,
inspection.binding_ref,
inspection.workspace_ref,
# The DTO validates the exact adapter/evidence pair before this
# complete protected predecessor identity comparison.
inspection.state,
) != (
expected.predecessor,
expected.identity.parent_execution_id,
expected.input_digest,
expected.definition_digest,
parent,
expected.predecessor.association_object_ref,
expected.agent_generation,
expected.predecessor_checkpoint.checkpoint_id,
expected.predecessor_checkpoint.payload_digest,
evidence["agent_run_id"],
predecessor.binding_ref,
predecessor.workspace_ref,
"definitively_gone",
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
checkpoint = self._agent_wait_relation(prepared, association, wait)
if [call.pending_operation_id for call in AgentCheckpoint.model_validate(checkpoint).pending_calls] != evidence[
"pending_ids"
]:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
self._agent_object(
prepared,
logical_key="agent-recovery:" + expected.attempt_id + ":evidence",
object_ref=expected.evidence_object_ref,
)
return prepared, evidence
@staticmethod
def recheck_recovery_wait(*, session, owned, prepared, evidence, expected) -> None:
"""Recheck durable pause, never-dispatched wait and exclusive old participant.
Preconditions: root lock and refreshed protected preparation. No external
I/O occurs; all refusals precede any reserved/creating/final selection write.
"""
from models.workflow import WorkflowNodeExecutionModel, WorkflowPause, WorkflowRun
from services.dify_builder.native_testing.agent_recovery import (
require_never_dispatched_pending,
waiting_confirmed_checkpoint,
)
request = session.get(DifyBuilderExecutionRequest, prepared.parent.ref.request_id)
scope = session.scalar(
select(NativeTestScope).where(
NativeTestScope.request_id == prepared.parent.ref.request_id,
NativeTestScope.tenant_id == owned.tenant_id,
)
)
if request is None or scope is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
checkpoint, generation = waiting_confirmed_checkpoint(
session=session, ref=prepared.parent.ref, request=request, owned=owned, scope=scope
)
state = invocation_state(session, prepared.parent.ref, owned.tenant_id)
pause = session.get(WorkflowPause, evidence["pause_id"])
run = session.get(WorkflowRun, evidence["native_run_id"])
if (
(checkpoint.model_dump(mode="json"), generation, state.wait_reason, set(state.pending_ids))
!= (
evidence["parent_checkpoint"],
expected.parent_generation,
"approval",
set(evidence["pending_ids"]),
)
or pause is None
or run is None
or (
pause.state_object_key,
pause.workflow_run_id,
pause.resumed_at,
run.status,
)
!= (evidence["pause_state_object_key"], evidence["native_run_id"], None, "paused")
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
require_never_dispatched_pending(
session=session,
tenant_id=owned.tenant_id,
ref=prepared.parent.ref,
pending_ids=tuple(evidence["pending_ids"]),
generation=expected.parent_generation,
)
participants = list(
session.scalars(
select(AgentWorkspaceBinding.id)
.where(
AgentWorkspaceBinding.workspace_id == expected.predecessor.workspace_id,
AgentWorkspaceBinding.status == AgentWorkingResourceStatus.ACTIVE,
)
.with_for_update()
)
)
executions = list(
session.scalars(
select(WorkflowNodeExecutionModel.id)
.where(WorkflowNodeExecutionModel.agent_workspace_binding_id == expected.predecessor.binding_id)
.with_for_update()
)
)
if participants != [expected.predecessor.binding_id] or executions != [expected.identity.parent_execution_id]:
raise BuilderExecutionPolicyError("native_agent_single_workspace_limit")
home = None if expected.home_snapshot_id is None else session.get(AgentHomeSnapshot, expected.home_snapshot_id)
binding = session.get(AgentWorkspaceBinding, expected.predecessor.binding_id)
if binding is None or evidence["allocation_owner"] != {
"tenant_id": owned.tenant_id,
"agent_id": binding.agent_id,
"home_snapshot_id": expected.home_snapshot_id,
"home_snapshot_ref": None if home is None else home.snapshot_ref,
"home_snapshot_digest": None if home is None else NativeExecutionStateStore.recovery_home_digest(home),
}:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
if (
None if home is None else NativeExecutionStateStore.recovery_home_digest(home)
) != expected.home_snapshot_digest:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
def block_recovery_allocation(
self, *, ref: NativeTestContextRef, expected: "AgentRecoveryRecord"
) -> "AgentRecoveryRecord":
"""Inert settlement port for 3C's known authority/pin/scope/reconstruction drift.
No new allocation, retry or execution authority is granted. Exact allocated
ACK is retained; creating/unknown/ready are never rewritten. Caller must
distinguish known drift from transient persistence failure.
"""
from services.dify_builder.native_testing.agent_recovery import block_recovery_allocation_record
with self.loader.locked(ref=ref, settlement=True) as (session, owned, _scope):
return block_recovery_allocation_record(
session=session, tenant_id=owned.tenant_id, ref=ref, expected=expected
)
def _selected_recovery_graph(self, *, owned, context, parent):
"""Decode exactly the SQL-selected ready graph outside every writer lock.
Unselected staging objects grant no authority. This reconstructs both
retained and successor tuples, full parent/input/run/pins and canonical
receipt values, then SDK pure validators prove the allowed substitution.
The caller must retain this proof for the SQL-only current-row recheck.
"""
from dify_agent.protocol.confirmed_recovery import (
AgentRecoveryReconstruction,
AgentRecoverySuccessor,
recovery_successor_digest,
)
from dify_agent.protocol.native_continuation import AgentCheckpoint
from dify_agent.runtime.confirmed_recovery import validate_recovery_checkpoint, validate_recovery_configs
from services.dify_builder.native_testing.agent_recovery import (
RecoveryAllocationReceipt,
SelectedRecovery,
recovery_record,
validate_allocation_receipt,
)
record = recovery_record(owned.bindings)
if record is None or record.phase != "ready":
return None
receipts = []
def read(logical_key, object_ref):
with self.loader.session_factory() as session:
row = session.scalar(
owner_query(NativePayloadObject, parent.ref, owned.tenant_id).where(
NativePayloadObject.kind == "result",
NativePayloadObject.logical_key == logical_key,
NativePayloadObject.id == object_ref,
)
)
if row is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
receipt = ObjectReceipt.from_payload_object(row)
receipts.append(receipt)
return receipt, self._read_object(receipt)
prefix = "agent-recovery:" + record.attempt_id + ":"
relation_receipt, raw_relation = read(prefix + "relation", record.successor_object_ref)
relation = AgentRecoverySuccessor.model_validate(raw_relation)
if (record.identity.invocation_id, record.identity.parent_execution_id, record.input_digest) != (
parent.ref.invocation_id,
parent.execution_id,
parent.input_digest,
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
if relation_receipt.value_digest != recovery_successor_digest(relation) or (
relation.parent_node.model_dump(mode="json"),
relation.attempt_id,
relation.expected_parent_generation,
relation.expected_agent_generation,
relation.predecessor,
relation.predecessor_checkpoint,
relation.successor.binding_id,
relation.successor.workspace_id,
relation.successor.workspace_generation,
relation_receipt.value_digest,
) != (
parent.model_dump(mode="json"),
record.attempt_id,
record.parent_generation,
record.agent_generation,
record.predecessor,
record.predecessor_checkpoint,
record.reserved_binding_id,
record.reserved_workspace_id,
record.next_workspace_generation,
record.successor_digest,
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
input_receipt, inputs = read("node-input:" + parent.execution_id, parent.input_result_ref)
if input_receipt.value_digest != parent.input_digest or canonical_digest(inputs) != record.input_digest:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
_evidence_receipt, evidence = read(prefix + "evidence", record.evidence_object_ref)
if not isinstance(evidence, dict) or evidence.get("parent_node") != parent.model_dump(mode="json"):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_wait = AgentWaitAssociation.model_validate(evidence["wait"])
_old_config_receipt, old_config_value = read(
self._agent_key("agent-config", parent), relation.predecessor.config_ref
)
if not isinstance(old_config_value, dict) or (
old_config_value.get("schema"),
old_config_value.get("parent_node"),
old_config_value.get("workflow_native_run_id"),
old_config_value.get("resource_digest"),
) != ("agent-config-v2", parent.model_dump(mode="json"), context.native_run_id, record.definition_digest):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_workspace = AgentTestWorkspaceRef.model_validate(old_config_value["workspace"])
old_config = AgentPreparedConfig(
parent_node=parent,
workflow_native_run_id=context.native_run_id,
workspace=old_workspace,
config_ref=relation.predecessor.config_ref,
config_digest=canonical_digest(self._agent_executable_config(old_config_value["config"], parent)),
config=old_config_value["config"],
)
_old_association_receipt, old_association_value = read(
self._agent_key("agent-bind", parent), relation.predecessor.association_object_ref
)
old_association = AgentNodeAssociation.model_validate(old_association_value)
_old_checkpoint_receipt, old_checkpoint_value = read(
self._agent_key("agent-cp", parent, record.agent_generation), record.predecessor_checkpoint.checkpoint_id
)
if not isinstance(old_checkpoint_value, dict) or old_checkpoint_value != {
"schema": "agent-checkpoint-v1",
"parent_node": parent.model_dump(mode="json"),
"association_object_ref": relation.predecessor.association_object_ref,
"agent_generation": record.agent_generation,
"backend_wait_event_id": relation.predecessor_waiting_event_id,
"checkpoint": old_checkpoint_value.get("checkpoint"),
}:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_checkpoint = AgentCheckpoint.model_validate(old_checkpoint_value["checkpoint"])
config_receipt, config_value = read(prefix + "config", relation.successor.config_ref)
executable = self._recovery_prepared_config(receipt=config_receipt, value=config_value)
_association_receipt, association_value = read(
prefix + "association", relation.successor.association_object_ref
)
association = AgentNodeAssociation.model_validate(association_value)
checkpoint_receipt, checkpoint_value = read(prefix + "checkpoint", relation.successor_checkpoint.checkpoint_id)
checkpoint = AgentCheckpoint.model_validate(checkpoint_value)
reconstruction_receipt, reconstruction_value = read(
prefix + "reconstruction", relation.reconstruction_object_ref
)
reconstruction = AgentRecoveryReconstruction.model_validate(reconstruction_value)
_allocation_receipt, allocation_value = read(prefix + "allocation", record.allocation_receipt_ref)
allocation = RecoveryAllocationReceipt.model_validate(allocation_value)
allocation_owner = evidence["allocation_owner"]
if not isinstance(allocation_owner, dict):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
validate_allocation_receipt(
expected=record,
receipt=allocation,
tenant_id=context.tenant_id,
agent_id=allocation_owner["agent_id"],
home_snapshot_ref=allocation_owner["home_snapshot_ref"],
)
predecessor = relation.predecessor
successor = relation.successor
if (
(
old_workspace.binding_id,
old_workspace.workspace_id,
old_workspace.backend_binding_ref,
old_workspace.backend_workspace_ref,
old_workspace.workspace_generation,
old_config.config_digest,
)
!= (
predecessor.binding_id,
predecessor.workspace_id,
predecessor.binding_ref,
predecessor.workspace_ref,
predecessor.workspace_generation,
predecessor.config_digest,
)
or (
old_association.parent_node,
old_association.workflow_native_run_id,
old_association.agent_run_id,
old_association.workspace,
old_association.config_ref,
old_association.config_digest,
)
!= (
parent,
context.native_run_id,
relation.agent_run_id,
old_workspace,
old_config.config_ref,
old_config.config_digest,
)
or old_wait
!= AgentWaitAssociation(
parent_node=parent,
association_object_ref=predecessor.association_object_ref,
agent_generation=record.agent_generation,
checkpoint_object_ref=record.predecessor_checkpoint.checkpoint_id,
checkpoint_digest=record.predecessor_checkpoint.payload_digest,
backend_wait_event_id=relation.predecessor_waiting_event_id,
)
or canonical_digest(old_checkpoint.model_dump(mode="json")) != record.predecessor_checkpoint.payload_digest
or (executable.parent_node.model_dump(mode="json"), executable.config_ref, executable.config_digest)
!= (parent.model_dump(mode="json"), successor.config_ref, successor.config_digest)
or (
association.parent_node,
association.workflow_native_run_id,
association.agent_run_id,
association.config_ref,
association.config_digest,
association.pinned_snapshot_id,
association.home_snapshot_id,
association.workspace.binding_id,
association.workspace.workspace_id,
association.workspace.backend_binding_ref,
association.workspace.backend_workspace_ref,
association.workspace.workspace_generation,
)
!= (
parent,
context.native_run_id,
relation.agent_run_id,
successor.config_ref,
successor.config_digest,
old_workspace.pinned_snapshot_id,
record.home_snapshot_id,
successor.binding_id,
successor.workspace_id,
successor.binding_ref,
successor.workspace_ref,
successor.workspace_generation,
)
or (
checkpoint_receipt.value_digest,
reconstruction_receipt.value_digest,
reconstruction.home_snapshot_id,
reconstruction.home_snapshot_digest,
allocation.response.binding_ref,
allocation.response.workspace_ref,
)
!= (
relation.successor_checkpoint.payload_digest,
relation.reconstruction_digest,
record.home_snapshot_id,
record.home_snapshot_digest,
successor.binding_ref,
successor.workspace_ref,
)
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
validate_recovery_checkpoint(relation, old_checkpoint, checkpoint)
validate_recovery_configs(
relation,
self.recovery_config_request(
config=old_config.config, parent=parent, create_idempotency_key=old_checkpoint.create_idempotency_key
),
self.recovery_config_request(
config=executable.config, parent=parent, create_idempotency_key=checkpoint.create_idempotency_key
),
checkpoint,
)
config = AgentPreparedConfig(
parent_node=parent,
workflow_native_run_id=context.native_run_id,
workspace=association.workspace,
config_ref=executable.config_ref,
config_digest=executable.config_digest,
config=executable.config,
)
return SelectedRecovery(
record,
relation,
config,
association,
checkpoint,
old_config,
old_association,
old_wait,
old_checkpoint,
tuple(receipts),
)
@staticmethod
def _recheck_selected_recovery(*, session, owned, parent, proof):
"""SQL-only exact record/receipt/input comparison after decoded graph reads."""
from services.dify_builder.native_testing.agent_recovery import recovery_record
if recovery_record(owned.bindings) != proof.record or proof.relation.parent_node.model_dump(
mode="json"
) != parent.model_dump(mode="json"):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
from dify_agent.protocol.confirmed_recovery import AgentRecoveryReconstruction, recovery_successor_digest
from services.dify_builder.native_testing.agent_recovery import RecoveryPreparedConfigPayload
record, relation, config = proof.record, proof.relation, proof.config
successor = relation.successor
if (
recovery_successor_digest(relation),
config.parent_node,
config.config_ref,
config.config_digest,
canonical_digest(config.config),
config.workspace,
proof.association.parent_node,
proof.association.workflow_native_run_id,
proof.association.agent_run_id,
proof.association.config_ref,
proof.association.config_digest,
proof.checkpoint.config_ref,
proof.checkpoint.config_digest,
canonical_digest(proof.checkpoint.model_dump(mode="json")),
) != (
record.successor_digest,
parent,
successor.config_ref,
successor.config_digest,
successor.config_digest,
proof.association.workspace,
parent,
config.workflow_native_run_id,
relation.agent_run_id,
successor.config_ref,
successor.config_digest,
successor.config_ref,
successor.config_digest,
relation.successor_checkpoint.payload_digest,
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
prefix = "agent-recovery:" + record.attempt_id + ":"
roles = {
record.successor_object_ref: prefix + "relation",
parent.input_result_ref: "node-input:" + parent.execution_id,
record.evidence_object_ref: prefix + "evidence",
record.allocation_receipt_ref: prefix + "allocation",
relation.predecessor.config_ref: NativeExecutionStateStore._agent_key("agent-config", parent),
relation.predecessor.association_object_ref: NativeExecutionStateStore._agent_key("agent-bind", parent),
relation.predecessor_checkpoint.checkpoint_id: NativeExecutionStateStore._agent_key(
"agent-cp", parent, record.agent_generation
),
successor.config_ref: prefix + "config",
successor.association_object_ref: prefix + "association",
relation.successor_checkpoint.checkpoint_id: prefix + "checkpoint",
relation.reconstruction_object_ref: prefix + "reconstruction",
}
values = {
record.successor_object_ref: record.successor_digest,
parent.input_result_ref: parent.input_digest,
successor.config_ref: canonical_digest(
RecoveryPreparedConfigPayload(
schema_version="agent-recovery-config-v1", parent_node=parent, config=config.config
).model_dump(mode="json")
),
successor.association_object_ref: canonical_digest(proof.association.model_dump(mode="json")),
relation.successor_checkpoint.checkpoint_id: relation.successor_checkpoint.payload_digest,
relation.reconstruction_object_ref: canonical_digest(
AgentRecoveryReconstruction(
kind="empty-bootstrap-v1",
home_snapshot_id=record.home_snapshot_id,
home_snapshot_digest=record.home_snapshot_digest,
).model_dump(mode="json")
),
relation.predecessor.config_ref: canonical_digest(
{
"schema": "agent-config-v2",
"parent_node": parent.model_dump(mode="json"),
"workflow_native_run_id": proof.predecessor_config.workflow_native_run_id,
"workspace": proof.predecessor_config.workspace.model_dump(mode="json"),
"resource_digest": record.definition_digest,
"config": proof.predecessor_config.config,
}
),
relation.predecessor.association_object_ref: canonical_digest(
proof.predecessor_association.model_dump(mode="json")
),
relation.predecessor_checkpoint.checkpoint_id: canonical_digest(
{
"schema": "agent-checkpoint-v1",
"parent_node": parent.model_dump(mode="json"),
"association_object_ref": relation.predecessor.association_object_ref,
"agent_generation": record.agent_generation,
"backend_wait_event_id": relation.predecessor_waiting_event_id,
"checkpoint": proof.predecessor_checkpoint.model_dump(mode="json"),
}
),
}
if len(roles) != 11 or len(proof.receipts) != 11 or {receipt.id for receipt in proof.receipts} != set(roles):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
for receipt in proof.receipts:
if receipt.logical_key != roles[receipt.id] or (
receipt.id in values and receipt.value_digest != values[receipt.id]
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
row = session.scalar(
owner_query(NativePayloadObject, parent.ref, owned.tenant_id).where(
NativePayloadObject.id == receipt.id
)
)
if row is None or ObjectReceipt.from_payload_object(row) != receipt:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
def load_selected_recovery(self, *, ref, parent_node):
"""Read one exact current relation; incomplete staging stays unselected.
Existing access, scope, revision, pin, complete input and SQL resource
checks run before returning. No SDK handoff or mutation is authorized.
"""
prepared = self._agent_prepare(ref=ref, parent_node=parent_node, writing=False)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
return None if prepared.recovery is None else prepared.recovery.relation
def _selected_recovery_staged_pending(self, *, prepared, graph):
"""Read exact selected staged disclosures before the root writer.
The graph's validated checkpoints and relation supply both ends of each
mapping; actual protected snapshots supply fresh intents and proposals.
The caller must recheck the recorded receipts and ledger under root.
"""
from runtime_contracts.builder.native.approval_contracts import NativeApprovalProposal
from services.dify_builder.native_testing.agent_bridge import api_intent
from services.dify_builder.native_testing.agent_operations import _resource_key
from services.dify_builder.native_testing.agent_recovery import RecoveryStagedPending
relation = graph.relation
old_calls = {call.pending_operation_id: call for call in graph.predecessor_checkpoint.pending_calls}
fresh_calls = {call.pending_operation_id: call for call in graph.checkpoint.pending_calls}
substitutions = relation.pending_operation_substitutions
if (
len(old_calls) != len(substitutions)
or len(fresh_calls) != len(substitutions)
or set(old_calls) != {item.predecessor_operation_id for item in substitutions}
or set(fresh_calls) != {item.successor_operation_id for item in substitutions}
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
pending = []
for item in substitutions:
intent = api_intent(fresh_calls[item.successor_operation_id].intent)
selected = self._agent_object(prepared, logical_key=_resource_key(intent.key))
if (
selected is None
or not isinstance(selected[1], dict)
or selected[1].get("intent") != intent.model_dump(mode="json")
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
pending.append(
RecoveryStagedPending(
predecessor_pending_id=item.predecessor_operation_id,
pending_id=item.successor_operation_id,
intent=intent,
proposal=NativeApprovalProposal.model_validate(selected[1]["proposal"]),
)
)
return tuple(pending)
def read_recovery_predecessor_result(self, *, ref, parent_node, selection, result_ref):
"""Read only the current selected graph for a verified predecessor caller.
The caller must separately authenticate the predecessor runtime. This
core port checks the complete selected owner and both pending ledgers;
it confers no mutation or execution authority. Storage reads occur
before the root lock and all selected receipts are rechecked under it.
"""
from dify_agent.protocol.native_testing import AgentParentNodeRef as WireParent
from dify_agent.protocol.native_testing import AgentPreparedConfigResponse
from services.dify_builder.native_testing.agent_recovery import (
recheck_staged_recovery_pending,
require_never_dispatched_pending,
)
prepared = self._agent_prepare(ref=ref, parent_node=parent_node, writing=False)
proof = prepared.recovery
if proof is None or proof.relation != selection or proof.record.sdk_receipt_ref is not None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
relation = proof.relation
pending = self._selected_recovery_staged_pending(prepared=prepared, graph=proof)
prefix = "agent-recovery:" + proof.record.attempt_id + ":"
roles = {
proof.record.successor_object_ref: prefix + "relation",
relation.reconstruction_object_ref: prefix + "reconstruction",
relation.successor_checkpoint.checkpoint_id: prefix + "checkpoint",
}
if result_ref in roles:
selected = self._agent_object(prepared, logical_key=roles[result_ref], object_ref=result_ref)
if selected is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
value = selected[1]
elif result_ref in (relation.predecessor.config_ref, relation.successor.config_ref):
config = proof.predecessor_config if result_ref == relation.predecessor.config_ref else proof.config
value = AgentPreparedConfigResponse(
parent_node=WireParent.model_validate(parent_node.model_dump(mode="json")),
config_ref=config.config_ref,
config_digest=config.config_digest,
config=self._agent_executable_config(config.config, parent_node),
).model_dump(mode="json")
else:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
self._recheck_selected_recovery(session=session, owned=owned, parent=parent_node, proof=proof)
recheck_staged_recovery_pending(session=session, owned=owned, expected=proof.record, pending=pending)
require_never_dispatched_pending(
session=session,
tenant_id=owned.tenant_id,
ref=ref,
pending_ids=tuple(item.predecessor_pending_id for item in pending),
generation=proof.record.parent_generation,
)
return value
def acknowledge_recovery_sdk_receipt(self, *, ref, expected, receipt):
"""Persist the trusted SDK install observation without selecting a wait.
The internal caller supplies the actual durable SDK receipt; this port
validates its complete selected tuple and cannot itself authenticate an
SDK transport. Identical replay returns the same protected receipt.
All SQL authority remains closed until the separate wait ACK.
"""
from dify_agent.protocol.confirmed_recovery import InstallConfirmedRecoverySuccessorReceipt
from services.dify_builder.native_testing.agent_recovery import (
recheck_staged_recovery_pending,
recovery_record,
require_never_dispatched_pending,
)
from services.dify_builder.native_testing.agent_recovery_state import (
RECOVERY_RECORD_KEY,
advance_recovery_record,
)
receipt = InstallConfirmedRecoverySuccessorReceipt.model_validate(receipt.model_dump(mode="json"))
evidence = self.load_result(ref=ref, result_ref=expected.evidence_object_ref)
if not isinstance(evidence, dict):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
parent = AgentParentNodeRef.model_validate(evidence["parent_node"])
prepared = self._agent_prepare(ref=ref, parent_node=parent, writing=False)
proof = prepared.recovery
if proof is None or expected.phase != "ready" or expected.sdk_receipt_ref is not None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
relation = proof.relation
pending = self._selected_recovery_staged_pending(prepared=prepared, graph=proof)
if (
receipt.attempt_id,
receipt.successor_digest,
receipt.agent_run_id,
receipt.parent_generation,
receipt.agent_generation,
receipt.workspace_generation,
receipt.config_ref,
receipt.config_digest,
receipt.checkpoint,
) != (
expected.attempt_id,
expected.successor_digest,
relation.agent_run_id,
expected.parent_generation,
expected.agent_generation,
expected.next_workspace_generation,
relation.successor.config_ref,
relation.successor.config_digest,
relation.successor_checkpoint,
) or receipt.waiting_event_id == relation.predecessor_waiting_event_id:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
logical_key = "agent-recovery:" + expected.attempt_id + ":sdk"
value = receipt.model_dump(mode="json")
existing = self._agent_object(prepared, logical_key=logical_key, required=False)
if existing is not None and existing[1] != value:
raise BuilderExecutionPolicyError("native_payload_conflict")
protected = (
existing[0]
if existing is not None
else self.prepare_object(
ref=ref, tenant_id=prepared.context.tenant_id, kind="result", logical_key=logical_key, payload=value
)
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
self._recheck_selected_recovery(session=session, owned=owned, parent=parent, proof=proof)
current = recovery_record(owned.bindings)
if current is None or current.model_copy(update={"sdk_receipt_ref": None}) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
recheck_staged_recovery_pending(session=session, owned=owned, expected=current, pending=pending)
require_never_dispatched_pending(
session=session,
tenant_id=owned.tenant_id,
ref=ref,
pending_ids=tuple(item.predecessor_pending_id for item in pending),
generation=expected.parent_generation,
)
acknowledged = ObjectReceipt.from_payload_object(self.acknowledge_object(session, protected))
if current.sdk_receipt_ref is not None and current.sdk_receipt_ref != acknowledged.id:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
proposed = advance_recovery_record(current, current.model_copy(update={"sdk_receipt_ref": acknowledged.id}))
invocation = session.scalar(owner_query(NativeTestInvocation, ref, owned.tenant_id).with_for_update())
if invocation is None:
raise BuilderExecutionPolicyError("native_invocation_mismatch")
invocation.bindings = {**invocation.bindings, RECOVERY_RECORD_KEY: proposed.model_dump(mode="json")}
if self._read_object(acknowledged) != value:
raise BuilderExecutionPolicyError("native_payload_conflict")
return proposed
def acknowledge_recovery_wait(self, *, ref, expected, receipt, checkpoint, event):
"""Atomically select the verified fresh wait and retire unattempted old operations.
The caller supplies a real SDK checkpoint/event observation. The protected
SDK receipt and staged disclosures are read before the root lock; all
selected graph, pending and parent identities are rechecked under it.
Success preserves the original continuation and old pending rows, and
settles only proved never-dispatched predecessors as not_dispatched.
Failure leaves the selected wait and predecessor operations unchanged.
"""
from dify_agent.protocol.confirmed_recovery import InstallConfirmedRecoverySuccessorReceipt
from dify_agent.protocol.native_continuation import SelectedNativeCheckpoint
from models.dify_builder_native_tests import NativePendingAction
from services.dify_builder.native_testing.agent_recovery import (
RecoveryApprovalEntry,
RecoveryPendingProof,
RecoveryPendingSelection,
recheck_staged_recovery_pending,
recovery_record,
require_never_dispatched_pending,
)
from services.dify_builder.native_testing.agent_recovery_state import (
RECOVERY_RECORD_KEY,
advance_recovery_record,
)
receipt = InstallConfirmedRecoverySuccessorReceipt.model_validate(receipt.model_dump(mode="json"))
observed = SelectedNativeCheckpoint(checkpoint=checkpoint, waiting_event=event)
evidence = self.load_result(ref=ref, result_ref=expected.evidence_object_ref)
if not isinstance(evidence, dict) or expected.phase != "ready" or expected.sdk_receipt_ref is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
parent = AgentParentNodeRef.model_validate(evidence["parent_node"])
original = CheckpointRef.model_validate(evidence["parent_checkpoint"])
current = recovery_record(self.loader.load(ref=ref).bindings)
if current is not None and current.successor_wait_ref is not None:
if current.model_copy(update={"successor_wait_ref": None}) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
selected = self.load_recovery_pending_state(ref=ref, original_checkpoint=original)
if (
selected is None
or selected.sdk_receipt != receipt
or selected.graph.checkpoint != observed.checkpoint
or selected.proof.selection.wait.backend_wait_event_id != observed.waiting_event.id
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
return selected.proof
original_payload = self.read_checkpoint(ref=ref, checkpoint=original)
prepared = self._agent_prepare(ref=ref, parent_node=parent, writing=False)
graph = prepared.recovery
if (
graph is None
or graph.record != expected
or observed.checkpoint != graph.checkpoint
or (observed.waiting_event.id, observed.waiting_event.run_id)
!= (receipt.waiting_event_id, receipt.agent_run_id)
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
if original_payload.generation != expected.parent_generation or [
item for item in original_payload.agent_associations if item.parent_node == parent
] != [graph.predecessor_wait]:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
with self.loader.session_factory() as session:
original_row = session.scalar(
owner_query(NativePayloadObject, ref, prepared.context.tenant_id).where(
NativePayloadObject.id == original.checkpoint_id, NativePayloadObject.kind == "checkpoint"
)
)
if original_row is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
original_receipt = ObjectReceipt.from_payload_object(original_row)
sdk = self._agent_object(
prepared, logical_key="agent-recovery:" + expected.attempt_id + ":sdk", object_ref=expected.sdk_receipt_ref
)
if sdk is None or sdk[1] != receipt.model_dump(mode="json"):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
relation = graph.relation
staged = self._selected_recovery_staged_pending(prepared=prepared, graph=graph)
entries = tuple(RecoveryApprovalEntry.model_validate(entry.model_dump(mode="json")) for entry in staged)
wait = AgentWaitAssociation(
parent_node=parent,
association_object_ref=relation.successor.association_object_ref,
agent_generation=receipt.agent_generation,
checkpoint_object_ref=receipt.checkpoint.checkpoint_id,
checkpoint_digest=receipt.checkpoint.payload_digest,
backend_wait_event_id=receipt.waiting_event_id,
)
selection = RecoveryPendingSelection(
schema_version="agent-recovery-pending-v1",
attempt_id=expected.attempt_id,
original_checkpoint=original,
parent_generation=expected.parent_generation,
relation_ref=expected.successor_object_ref,
relation_digest=expected.successor_digest,
sdk_receipt_ref=expected.sdk_receipt_ref,
wait=wait,
approvals=entries,
)
value = selection.model_dump(mode="json")
logical_key = "agent-recovery:" + expected.attempt_id + ":wait"
existing = self._agent_object(prepared, logical_key=logical_key, required=False)
if existing is not None and existing[1] != value:
raise BuilderExecutionPolicyError("native_payload_conflict")
protected = (
existing[0]
if existing is not None
else self.prepare_object(
ref=ref, tenant_id=prepared.context.tenant_id, kind="result", logical_key=logical_key, payload=value
)
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
self._recheck_selected_recovery(session=session, owned=owned, parent=parent, proof=graph)
if recovery_record(owned.bindings) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
recheck_staged_recovery_pending(session=session, owned=owned, expected=expected, pending=staged)
old_ids = tuple(entry.predecessor_pending_id for entry in entries)
require_never_dispatched_pending(
session=session,
tenant_id=owned.tenant_id,
ref=ref,
pending_ids=old_ids,
generation=expected.parent_generation,
)
state = invocation_state(session, ref, owned.tenant_id)
continuation = session.scalar(
owner_query(NativeContinuation, ref, owned.tenant_id).where(
NativeContinuation.generation == expected.parent_generation
)
)
if continuation is None or (
state.checkpoint_id,
continuation.checkpoint_id,
continuation.payload_digest,
) != (original.checkpoint_id, original.checkpoint_id, original.payload_digest):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
original_row = session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.id == original.checkpoint_id, NativePayloadObject.kind == "checkpoint"
)
)
if (
original_row is None
or ObjectReceipt.from_payload_object(original_row) != original_receipt
or continuation.payload_ref != original_receipt.storage_key
or original_receipt.value_digest != canonical_digest(original_payload.model_dump(mode="json"))
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
acknowledged = ObjectReceipt.from_payload_object(self.acknowledge_object(session, protected))
for old_id in old_ids:
pending_row = session.scalar(
owner_query(NativePendingAction, ref, owned.tenant_id).where(NativePendingAction.id == old_id)
)
if pending_row is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
operation = session.scalar(
owner_query(NativeEffectOperation, ref, owned.tenant_id).where(
NativeEffectOperation.id == pending_row.operation_id
)
)
if operation is None or operation.state != "pending":
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
operation.state = "not_dispatched"
state.pending_ids = [entry.pending_id for entry in entries]
proposed = advance_recovery_record(
expected, expected.model_copy(update={"successor_wait_ref": acknowledged.id})
)
invocation = session.scalar(owner_query(NativeTestInvocation, ref, owned.tenant_id).with_for_update())
request = session.get(DifyBuilderExecutionRequest, ref.request_id)
if invocation is None or request is None or len(request.observations) >= 4096:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
invocation.bindings = {**invocation.bindings, RECOVERY_RECORD_KEY: proposed.model_dump(mode="json")}
request.observations = [
*request.observations,
{
"recovery_pending_replaced": True,
"attempt_id": expected.attempt_id,
"wait_digest": acknowledged.value_digest,
},
]
return RecoveryPendingProof(
object_ref=acknowledged.id, value_digest=acknowledged.value_digest, selection=selection
)
def load_recovery_pending_chain(self, *, ref, original_checkpoint, for_projection=False):
"""Load the immutable selected SDK/wait/graph and retained ledger chain.
No recovery returns None. Incomplete or inconsistent recovery fails
closed. This preserves historical evidence as ordinary pending moves;
the active replacement set has its separate complete predicate below.
Current pins/access and canonical receipts are rechecked under the lock.
"""
from dify_agent.protocol.confirmed_recovery import InstallConfirmedRecoverySuccessorReceipt
from services.dify_builder.native_testing.agent_operations import _resource_key
from services.dify_builder.native_testing.agent_recovery import (
RecoveryPendingProof,
RecoveryPendingSelection,
RecoveryPendingState,
recovery_record,
)
record = recovery_record(self.loader.load(ref=ref).bindings)
if record is None:
return None
if record.phase != "ready" or record.sdk_receipt_ref is None or record.successor_wait_ref is None:
raise BuilderExecutionPolicyError("native_agent_recovery_unready")
evidence = self.load_result(ref=ref, result_ref=record.evidence_object_ref)
if not isinstance(evidence, dict):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
parent = AgentParentNodeRef.model_validate(evidence["parent_node"])
original_payload = self.read_checkpoint(ref=ref, checkpoint=original_checkpoint, for_projection=for_projection)
prepared = self._agent_prepare(ref=ref, parent_node=parent, writing=False, for_projection=for_projection)
graph = prepared.recovery
if graph is None or graph.record != record:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
if original_payload.generation != record.parent_generation or [
item for item in original_payload.agent_associations if item.parent_node == parent
] != [graph.predecessor_wait]:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
with self.loader.session_factory() as session:
original_row = session.scalar(
owner_query(NativePayloadObject, ref, prepared.context.tenant_id).where(
NativePayloadObject.id == original_checkpoint.checkpoint_id,
NativePayloadObject.kind == "checkpoint",
)
)
if original_row is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
original_receipt = ObjectReceipt.from_payload_object(original_row)
prefix = "agent-recovery:" + record.attempt_id + ":"
sdk = self._agent_object(prepared, logical_key=prefix + "sdk", object_ref=record.sdk_receipt_ref)
wait = self._agent_object(prepared, logical_key=prefix + "wait", object_ref=record.successor_wait_ref)
if sdk is None or wait is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
receipt = InstallConfirmedRecoverySuccessorReceipt.model_validate(sdk[1])
selection = RecoveryPendingSelection.model_validate(wait[1])
proof = RecoveryPendingProof(object_ref=wait[0].id, value_digest=wait[0].value_digest, selection=selection)
snapshots = []
for entry in selection.approvals:
snapshot = self._agent_object(prepared, logical_key=_resource_key(entry.intent.key))
if (
snapshot is None
or not isinstance(snapshot[1], dict)
or (snapshot[1].get("intent"), snapshot[1].get("proposal"))
!= (entry.intent.model_dump(mode="json"), entry.proposal.model_dump(mode="json"))
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
snapshots.append(snapshot[0])
state = RecoveryPendingState(
proof,
graph,
receipt,
sdk[0],
wait[0],
original_receipt,
original_payload,
tuple(snapshots),
prepared.context,
prepared.workspace,
prepared.resource_digest,
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
self.recheck_recovery_pending_chain(
session=session, owned=owned, state=state, original_checkpoint=original_checkpoint
)
return state
def load_recovery_pending_state(self, *, ref, original_checkpoint, for_projection=False):
"""Read the immutable chain and require the active replacement pending set."""
state = self.load_recovery_pending_chain(
ref=ref, original_checkpoint=original_checkpoint, for_projection=for_projection
)
if state is not None:
with self.loader.locked(ref=ref) as (session, owned, _scope):
self.recheck_recovery_pending_state(
session=session, owned=owned, state=state, original_checkpoint=original_checkpoint
)
return state
def load_recovery_checkpoint_state(self, *, ref, checkpoint, for_projection=False):
"""Prove the recovery chain and this actual current protected checkpoint.
The original replaced checkpoint retains the complete active-overlay
checks. A later ordinary checkpoint uses its own payload/continuation,
while immutable SDK/wait/graph receipts remain mandatory.
"""
from services.dify_builder.native_testing.agent_recovery import recovery_record
record = recovery_record(self.loader.load(ref=ref).bindings)
if record is None:
return None
evidence = self.load_result(ref=ref, result_ref=record.evidence_object_ref)
if not isinstance(evidence, dict):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
original = CheckpointRef.model_validate(evidence["parent_checkpoint"])
state = self.load_recovery_pending_chain(ref=ref, original_checkpoint=original, for_projection=for_projection)
if state is None:
raise BuilderExecutionPolicyError("native_agent_recovery_unready")
payload = self.read_checkpoint(ref=ref, checkpoint=checkpoint, for_projection=for_projection)
if checkpoint != original and (
payload.generation <= state.proof.selection.parent_generation
or len(
[
wait
for wait in payload.agent_associations
if wait.parent_node == state.proof.selection.wait.parent_node
and wait.association_object_ref == state.proof.selection.wait.association_object_ref
and wait.agent_generation > state.proof.selection.wait.agent_generation
]
)
!= 1
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
with self.loader.locked(ref=ref) as (session, owned, _scope):
self.recheck_recovery_checkpoint_state(session=session, owned=owned, state=state, checkpoint=checkpoint)
return state
@staticmethod
def recheck_recovery_checkpoint_state(*, session, owned, state, checkpoint):
"""SQL-only exact current continuation plus immutable recovery receipt proof."""
original = state.proof.selection.original_checkpoint
if checkpoint == original:
NativeExecutionStateStore.recheck_recovery_pending_state(
session=session, owned=owned, state=state, original_checkpoint=original
)
return
NativeExecutionStateStore.recheck_recovery_pending_chain(
session=session, owned=owned, state=state, original_checkpoint=original
)
current = invocation_state(session, owned.ref, owned.tenant_id)
continuation = session.scalar(
owner_query(NativeContinuation, owned.ref, owned.tenant_id).where(
NativeContinuation.checkpoint_id == checkpoint.checkpoint_id,
NativeContinuation.payload_digest == checkpoint.payload_digest,
NativeContinuation.generation > state.proof.selection.parent_generation,
)
)
row = session.scalar(
owner_query(NativePayloadObject, owned.ref, owned.tenant_id).where(
NativePayloadObject.id == checkpoint.checkpoint_id,
NativePayloadObject.kind == "checkpoint",
NativePayloadObject.payload_digest == checkpoint.payload_digest,
)
)
if (
continuation is None
or row is None
or current.checkpoint_id != checkpoint.checkpoint_id
or continuation.payload_ref != row.storage_key
or row.logical_key != str(continuation.generation)
or not continuation.generation <= current.generation <= continuation.generation + 1
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
def load_recovery_pending_proof(self, *, ref, original_checkpoint, for_projection=False):
"""Return the selected overlay only after its full protected state checks."""
state = self.load_recovery_pending_state(
ref=ref, original_checkpoint=original_checkpoint, for_projection=for_projection
)
return None if state is None else state.proof
@staticmethod
def recheck_recovery_pending_chain(*, session, owned, state, original_checkpoint):
"""Recheck canonical per-call evidence and current SQL owners without I/O."""
from models.dify_builder_native_tests import NativeEffectAttempt, NativeEffectDispatchClaim, NativePendingAction
from services.dify_builder.native_testing.agent_bridge import api_intent
from services.dify_builder.native_testing.agent_recovery import recovery_record
from services.dify_builder.native_testing.authorization import _scope_digest, validate_binding
from services.dify_builder.native_testing.repository import NativeBinding
proof, graph, receipt = state.proof, state.graph, state.sdk_receipt
selection, record, relation = proof.selection, graph.record, graph.relation
NativeExecutionStateStore._recheck_selected_recovery(
session=session, owned=owned, parent=selection.wait.parent_node, proof=graph
)
if (
recovery_record(owned.bindings) != record
or record.phase != "ready"
or (proof.object_ref, proof.value_digest, canonical_digest(selection.model_dump(mode="json")))
!= (record.successor_wait_ref, state.wait_object.value_digest, state.wait_object.value_digest)
or (
selection.attempt_id,
selection.original_checkpoint,
selection.parent_generation,
selection.relation_ref,
selection.relation_digest,
selection.sdk_receipt_ref,
)
!= (
record.attempt_id,
original_checkpoint,
record.parent_generation,
record.successor_object_ref,
record.successor_digest,
record.sdk_receipt_ref,
)
or (state.sdk_object.id, state.sdk_object.value_digest)
!= (record.sdk_receipt_ref, canonical_digest(receipt.model_dump(mode="json")))
or (
receipt.attempt_id,
receipt.successor_digest,
receipt.agent_run_id,
receipt.parent_generation,
receipt.agent_generation,
receipt.workspace_generation,
receipt.config_ref,
receipt.config_digest,
receipt.checkpoint,
receipt.waiting_event_id,
)
!= (
record.attempt_id,
record.successor_digest,
relation.agent_run_id,
record.parent_generation,
record.agent_generation,
record.next_workspace_generation,
relation.successor.config_ref,
relation.successor.config_digest,
relation.successor_checkpoint,
selection.wait.backend_wait_event_id,
)
or (
selection.wait.association_object_ref,
selection.wait.agent_generation,
selection.wait.checkpoint_object_ref,
selection.wait.checkpoint_digest,
)
!= (
relation.successor.association_object_ref,
receipt.agent_generation,
receipt.checkpoint.checkpoint_id,
receipt.checkpoint.payload_digest,
)
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
prefix = "agent-recovery:" + record.attempt_id + ":"
if (state.sdk_object.logical_key, state.wait_object.logical_key, state.wait_object.id) != (
prefix + "sdk",
prefix + "wait",
proof.object_ref,
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
for saved in (state.sdk_object, state.wait_object, state.original_object, *state.snapshot_objects):
row = session.scalar(
owner_query(NativePayloadObject, owned.ref, owned.tenant_id).where(NativePayloadObject.id == saved.id)
)
if row is None or ObjectReceipt.from_payload_object(row) != saved:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
if (
state.original_object.id != original_checkpoint.checkpoint_id
or state.original_object.kind != "checkpoint"
or state.original_object.payload_digest != original_checkpoint.payload_digest
or state.original_object.value_digest != canonical_digest(state.original_payload.model_dump(mode="json"))
or state.original_payload.generation != record.parent_generation
or [
item
for item in state.original_payload.agent_associations
if item.parent_node == selection.wait.parent_node
]
!= [graph.predecessor_wait]
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
original_continuation = session.scalar(
owner_query(NativeContinuation, owned.ref, owned.tenant_id).where(
NativeContinuation.generation == record.parent_generation,
NativeContinuation.checkpoint_id == original_checkpoint.checkpoint_id,
NativeContinuation.payload_digest == original_checkpoint.payload_digest,
)
)
if original_continuation is None or original_continuation.payload_ref != state.original_object.storage_key:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
current_workspace, current_digest = NativeExecutionStateStore._agent_resources(
session, owned, state.context, recovery=graph
)
if (current_workspace, current_digest) != (state.workspace, state.resource_digest):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_calls = {
call.pending_operation_id: api_intent(call.intent) for call in graph.predecessor_checkpoint.pending_calls
}
fresh_calls = {call.pending_operation_id: api_intent(call.intent) for call in graph.checkpoint.pending_calls}
if (
set(old_calls) != {entry.predecessor_pending_id for entry in selection.approvals}
or set(fresh_calls) != {entry.pending_id for entry in selection.approvals}
or len(state.snapshot_objects) != len(selection.approvals)
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
for entry in selection.approvals:
old = session.scalar(
owner_query(NativePendingAction, owned.ref, owned.tenant_id).where(
NativePendingAction.id == entry.predecessor_pending_id
)
)
fresh = session.scalar(
owner_query(NativePendingAction, owned.ref, owned.tenant_id).where(
NativePendingAction.id == entry.pending_id
)
)
old_op = (
None
if old is None
else session.scalar(
owner_query(NativeEffectOperation, owned.ref, owned.tenant_id).where(
NativeEffectOperation.id == old.operation_id
)
)
)
op = (
None
if fresh is None
else session.scalar(
owner_query(NativeEffectOperation, owned.ref, owned.tenant_id).where(
NativeEffectOperation.id == fresh.operation_id
)
)
)
raw = owned.bindings.get(operation_digest(entry.intent.key))
if old is None or fresh is None or old_op is None or op is None or raw is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
binding = NativeBinding.model_validate(raw)
validate_binding(entry.intent, binding)
digest = _scope_digest(entry.intent, binding)
old_intent = old_calls[entry.predecessor_pending_id]
old_raw = owned.bindings.get(operation_digest(old_intent.key))
if old_raw is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_binding = NativeBinding.model_validate(old_raw)
validate_binding(old_intent, old_binding)
old_digest = _scope_digest(old_intent, old_binding)
if (
entry.intent != fresh_calls[entry.pending_id]
# Retain reads of legacy ACKed chains without rewriting their
# history. Terminal projection still refuses legacy cancelled.
or old_op.state not in {"not_dispatched", "cancelled"}
or old_op.result_ref is not None
or op.state not in {"pending", "claimed", "completed"}
or old.reference != old_op.id
or old.reference_digest != canonical_digest(old_op.id)
or old.owner_ref != owned.app_id
or old.owner_version != old_digest
or old_op.scope_digest != old_digest
or old_op.operation_digest != operation_digest(old_intent.key)
or old_op.intent_digest != canonical_digest(old_intent.model_dump(mode="json"))
or old.category != "approval"
or fresh.category != "approval"
or old.generation != record.parent_generation
or fresh.generation != record.parent_generation
or fresh.reference != op.id
or fresh.reference_digest != canonical_digest(op.id)
or fresh.owner_ref != owned.app_id
or fresh.owner_version != digest
or op.scope_digest != digest
or op.operation_digest != operation_digest(entry.intent.key)
or op.intent_digest != canonical_digest(entry.intent.model_dump(mode="json"))
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
for model in (NativeEffectAttempt, NativeEffectDispatchClaim):
operation_ids = [old_op.id] + ([op.id] if op.state == "pending" else [])
if (
session.scalar(
owner_query(model, owned.ref, owned.tenant_id)
.where(model.operation_id.in_(operation_ids))
.limit(1)
)
is not None
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
@staticmethod
def recheck_recovery_pending_state(*, session, owned, state, original_checkpoint):
"""Recheck the complete active overlay; later checkpoints use their own owner."""
NativeExecutionStateStore.recheck_recovery_pending_chain(
session=session, owned=owned, state=state, original_checkpoint=original_checkpoint
)
graph, selection = state.graph, state.proof.selection
record, relation = graph.record, graph.relation
current = invocation_state(session, owned.ref, owned.tenant_id)
continuation = session.scalar(
owner_query(NativeContinuation, owned.ref, owned.tenant_id).where(
NativeContinuation.generation == record.parent_generation
)
)
if (
continuation is None
or current.checkpoint_id != original_checkpoint.checkpoint_id
or (continuation.checkpoint_id, continuation.payload_digest)
!= (original_checkpoint.checkpoint_id, original_checkpoint.payload_digest)
or continuation.payload_ref != state.original_object.storage_key
or current.pending_ids != [entry.pending_id for entry in selection.approvals]
or [(e.predecessor_pending_id, e.pending_id) for e in selection.approvals]
!= [
(s.predecessor_operation_id, s.successor_operation_id) for s in relation.pending_operation_substitutions
]
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
def prepare_recovery_selection(self, *, ref, expected, pending):
"""Build a stable protected successor graph from acknowledged inert pending.
Preconditions: current exact allocated attempt, old approved wait and
server-produced fresh pending snapshots. Each dependency is read/ACKed
before its ID enters the next object. SDK pure validators own the state
whitelist. No allocation, grant, selected pointer or parent checkpoint
change occurs; partial graphs remain inert and retries reuse role IDs.
"""
from dify_agent.protocol.confirmed_recovery import (
AgentRecoveryBinding,
AgentRecoveryReconstruction,
AgentRecoverySuccessor,
)
from dify_agent.protocol.native_continuation import AgentCheckpoint
from dify_agent.runtime.confirmed_recovery import (
recovery_snapshot,
validate_recovery_checkpoint,
validate_recovery_configs,
)
from services.dify_builder.native_testing.agent_operations import _resource_key
from services.dify_builder.native_testing.agent_recovery import (
PreparedRecoverySelection,
RecoveryAllocationReceipt,
RecoveryPreparedConfigPayload,
RecoveryStagedPending,
recheck_staged_recovery_pending,
recovery_record,
)
pending = tuple(RecoveryStagedPending.model_validate(entry.model_dump(mode="json")) for entry in pending)
if expected.phase != "allocated" or recovery_record(self.loader.load(ref=ref).bindings) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
prepared, evidence = self.prepare_recovery_attempt(ref=ref, expected=expected)
old_association = self._agent_binding(prepared, expected.predecessor.association_object_ref)
old_config = self._agent_prepared_config(prepared, expected.predecessor.config_ref)
old_wait = AgentWaitAssociation.model_validate(evidence["wait"])
old_checkpoint = AgentCheckpoint.model_validate(self._agent_wait_relation(prepared, old_association, old_wait))
if tuple(entry.predecessor_pending_id for entry in pending) != tuple(
call.pending_operation_id for call in old_checkpoint.pending_calls
) or len({entry.pending_id for entry in pending}) != len(pending):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
for entry in pending:
snapshot = self._agent_object(prepared, logical_key=_resource_key(entry.intent.key))
assert snapshot is not None
if not isinstance(snapshot[1], dict) or (snapshot[1].get("intent"), snapshot[1].get("proposal")) != (
entry.intent.model_dump(mode="json"),
entry.proposal.model_dump(mode="json"),
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
allocation = self._agent_object(
prepared,
logical_key="agent-recovery:" + expected.attempt_id + ":allocation",
object_ref=expected.allocation_receipt_ref,
)
assert allocation is not None
allocated = RecoveryAllocationReceipt.model_validate(allocation[1])
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
if recovery_record(owned.bindings) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
self._agent_recheck(session, owned, scope, prepared, writing=False)
self.recheck_recovery_wait(
session=session, owned=owned, prepared=prepared, evidence=evidence, expected=expected
)
recheck_staged_recovery_pending(session=session, owned=owned, expected=expected, pending=pending)
snapshot = recovery_snapshot(old_checkpoint)
config = json.loads(json.dumps(old_config.config))
for layer in config["composition"]["layers"]:
if layer["name"] == "runtime":
layer["config"]["backend_binding_ref"] = allocated.response.binding_ref
config["session_snapshot"] = snapshot.model_dump(mode="json")
payload = RecoveryPreparedConfigPayload(
schema_version="agent-recovery-config-v1", parent_node=prepared.parent, config=config
)
config_receipt = self.stage_recovery_object(
ref=ref, expected=expected, role="config", value=payload.model_dump(mode="json")
)
executable = self._recovery_prepared_config(receipt=config_receipt, value=payload.model_dump(mode="json"))
workspace = AgentTestWorkspaceRef(
binding_id=expected.reserved_binding_id,
workspace_id=expected.reserved_workspace_id,
backend_binding_ref=allocated.response.binding_ref,
backend_workspace_ref=allocated.response.workspace_ref,
pinned_snapshot_id=prepared.workspace.pinned_snapshot_id,
home_snapshot_id=expected.home_snapshot_id,
workspace_generation=expected.next_workspace_generation,
)
association = AgentNodeAssociation(
parent_node=prepared.parent,
workflow_native_run_id=old_association.workflow_native_run_id,
agent_run_id=old_association.agent_run_id,
config_ref=config_receipt.id,
config_digest=executable.config_digest,
pinned_snapshot_id=workspace.pinned_snapshot_id,
home_snapshot_id=workspace.home_snapshot_id,
workspace=workspace,
)
association_receipt = self.stage_recovery_object(
ref=ref, expected=expected, role="association", value=association.model_dump(mode="json")
)
calls = [
{
**call.model_dump(mode="json"),
"pending_operation_id": entry.pending_id,
"intent": entry.intent.model_dump(mode="json"),
}
for call, entry in zip(old_checkpoint.pending_calls, pending, strict=True)
]
checkpoint = AgentCheckpoint.model_validate(
{
**old_checkpoint.model_dump(mode="json"),
"pending_calls": calls,
"config_ref": config_receipt.id,
"config_digest": executable.config_digest,
"session_snapshot": snapshot.model_dump(mode="json"),
}
)
checkpoint_receipt = self.stage_recovery_object(
ref=ref, expected=expected, role="checkpoint", value=checkpoint.model_dump(mode="json")
)
reconstruction = AgentRecoveryReconstruction(
kind="empty-bootstrap-v1",
home_snapshot_id=expected.home_snapshot_id,
home_snapshot_digest=expected.home_snapshot_digest,
)
reconstruction_receipt = self.stage_recovery_object(
ref=ref, expected=expected, role="reconstruction", value=reconstruction.model_dump(mode="json")
)
successor = AgentRecoveryBinding(
binding_id=workspace.binding_id,
workspace_id=workspace.workspace_id,
binding_ref=workspace.backend_binding_ref,
workspace_ref=workspace.backend_workspace_ref,
workspace_generation=workspace.workspace_generation,
association_object_ref=association_receipt.id,
config_ref=config_receipt.id,
config_digest=executable.config_digest,
)
relation = AgentRecoverySuccessor.model_validate(
{
"attempt_id": expected.attempt_id,
"parent_node": prepared.parent.model_dump(mode="json"),
"agent_run_id": association.agent_run_id,
"expected_parent_generation": expected.parent_generation,
"expected_agent_generation": expected.agent_generation,
"predecessor": expected.predecessor.model_dump(mode="json"),
"successor": successor.model_dump(mode="json"),
"predecessor_checkpoint": expected.predecessor_checkpoint.model_dump(mode="json"),
"successor_checkpoint": {
"checkpoint_id": checkpoint_receipt.id,
"payload_digest": checkpoint_receipt.value_digest,
},
"predecessor_waiting_event_id": old_wait.backend_wait_event_id,
"reconstruction_object_ref": reconstruction_receipt.id,
"reconstruction_digest": reconstruction_receipt.value_digest,
"pending_operation_substitutions": [
{
"predecessor_operation_id": entry.predecessor_pending_id,
"successor_operation_id": entry.pending_id,
}
for entry in pending
],
}
)
validate_recovery_checkpoint(relation, old_checkpoint, checkpoint)
validate_recovery_configs(
relation,
self.recovery_config_request(
config=old_config.config,
parent=prepared.parent,
create_idempotency_key=old_checkpoint.create_idempotency_key,
),
self.recovery_config_request(
config=executable.config,
parent=prepared.parent,
create_idempotency_key=checkpoint.create_idempotency_key,
),
checkpoint,
)
relation_receipt = self.stage_recovery_object(
ref=ref, expected=expected, role="relation", value=relation.model_dump(mode="json")
)
return PreparedRecoverySelection(
expected,
relation,
relation_receipt,
config_receipt,
association_receipt,
checkpoint_receipt,
reconstruction_receipt,
)
def stage_recovery_object(
self,
*,
ref: NativeTestContextRef,
expected: "AgentRecoveryRecord",
role: Literal["config", "association", "checkpoint", "reconstruction", "relation"],
value: JsonValue,
) -> ObjectReceipt:
"""Acknowledge one inert allocated-only role and return its stable stored ID.
Preconditions: exact durable allocated record and retained original wait.
Storage/readback/access refresh occur outside the root writer. Identical
concurrent producers reuse the acknowledged receipt, including its ID;
conflicts or selection changes refuse without adopting arbitrary bytes.
"""
from services.dify_builder.native_testing.agent_recovery import recovery_record
role = TypeAdapter(
Literal["config", "association", "checkpoint", "reconstruction", "relation"]
).validate_python(role)
owned = self.loader.load(ref=ref)
if expected.phase != "allocated" or recovery_record(owned.bindings) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
prepared, evidence = self.prepare_recovery_attempt(ref=ref, expected=expected)
value = _JSON.validate_python(value)
logical_key = "agent-recovery:" + expected.attempt_id + ":" + role
existing = self._agent_object(prepared, logical_key=logical_key, required=False)
if existing is not None and existing[1] != value:
raise BuilderExecutionPolicyError("native_payload_conflict")
receipt = (
existing[0]
if existing is not None
else self.prepare_object(
ref=ref, tenant_id=owned.tenant_id, kind="result", logical_key=logical_key, payload=value
)
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, current, scope):
if recovery_record(current.bindings) != expected:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
self._agent_recheck(session, current, scope, prepared, writing=False)
self.recheck_recovery_wait(
session=session, owned=current, prepared=prepared, evidence=evidence, expected=expected
)
acknowledged = ObjectReceipt.from_payload_object(self.acknowledge_object(session, receipt))
if self._read_object(acknowledged) != value:
raise BuilderExecutionPolicyError("native_payload_conflict")
return acknowledged
def _recovery_prepared_config(self, *, receipt: ObjectReceipt, value: JsonValue):
"""Decode protected recovery config and derive the unchanged wire response.
This is data validation only. The finalizer/current owner must separately
prove the full parent/run, exact receipt and executable owner chain.
"""
from dify_agent.protocol.native_testing import AgentParentNodeRef as WireParent
from dify_agent.protocol.native_testing import AgentPreparedConfigResponse
from services.dify_builder.native_testing.agent_recovery import RecoveryPreparedConfigPayload
payload = RecoveryPreparedConfigPayload.model_validate(value)
config = self._agent_executable_config(payload.config, payload.parent_node)
return AgentPreparedConfigResponse(
parent_node=WireParent.model_validate(payload.parent_node.model_dump(mode="json")),
config_ref=receipt.id,
config_digest=canonical_digest(config),
config=config,
)
def finalize_recovery_successor(self, *, ref, selection: "PreparedRecoverySelection") -> "AgentRecoveryRecord":
"""Validate acknowledged immutable graph, then SQL-only finalize under root lock.
No interaction is wired to this primitive. All decrypt/SDK pure checks
precede the writer. Failure rolls back SQL selection while allocation ACK
remains durable. Full handshake is still separately required by 3C.
"""
from dify_agent.protocol.confirmed_recovery import AgentRecoveryReconstruction
from dify_agent.protocol.native_continuation import AgentCheckpoint
from dify_agent.runtime.confirmed_recovery import validate_recovery_checkpoint, validate_recovery_configs
from core.workflow.nodes.agent_v2.session_store import WorkflowAgentSessionScope
from services.dify_builder.native_testing.agent_recovery import (
RecoveryAllocationReceipt,
finalize_recovery_selection,
recovery_record,
)
owner = self.loader.load(ref=ref)
selected_record = recovery_record(owner.bindings)
if selected_record is not None and selected_record.phase == "ready":
# Exact completed-attempt readback only: never reaches the generation-zero
# resource admission path and never grants execution or adopts rows.
with self.loader.locked(ref=ref) as (session, owned, _scope):
invocation = session.scalar(owner_query(NativeTestInvocation, ref, owned.tenant_id))
if invocation is None:
raise BuilderExecutionPolicyError("native_invocation_mismatch")
session_scope = WorkflowAgentSessionScope(
**invocation.bindings["_agent_confirmed_workspace_v1"]["scope"]
)
return finalize_recovery_selection(
session=session,
tenant_id=owned.tenant_id,
ref=ref,
selection=selection,
session_scope=session_scope,
)
prepared, evidence = self.prepare_recovery_attempt(ref=ref, expected=selection.expected)
relation = selection.relation
if relation.parent_node.model_dump(mode="json") != prepared.parent.model_dump(mode="json"):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
values = {}
for role, receipt in (
("relation", selection.relation_receipt),
("config", selection.config_receipt),
("association", selection.association_receipt),
("checkpoint", selection.checkpoint_receipt),
("reconstruction", selection.reconstruction_receipt),
):
selected = self._agent_object(
prepared,
logical_key="agent-recovery:" + selection.expected.attempt_id + ":" + role,
object_ref=receipt.id,
)
if selected is None or selected[0] != receipt:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
values[role] = selected[1]
allocation = self._agent_object(
prepared,
logical_key="agent-recovery:" + selection.expected.attempt_id + ":allocation",
object_ref=selection.expected.allocation_receipt_ref,
)
assert allocation is not None
allocated = RecoveryAllocationReceipt.model_validate(allocation[1])
config = self._recovery_prepared_config(receipt=selection.config_receipt, value=values["config"])
association = AgentNodeAssociation.model_validate(values["association"])
checkpoint = AgentCheckpoint.model_validate(values["checkpoint"])
reconstruction = AgentRecoveryReconstruction.model_validate(values["reconstruction"])
if (
values["relation"] != relation.model_dump(mode="json")
or (
allocated.attempt_id,
allocated.response.binding_ref,
allocated.response.workspace_ref,
config.parent_node.model_dump(mode="json"),
config.config_ref,
config.config_digest,
association.parent_node,
association.workflow_native_run_id,
association.agent_run_id,
association.config_ref,
association.config_digest,
association.workspace.binding_id,
association.workspace.workspace_id,
association.workspace.backend_binding_ref,
association.workspace.backend_workspace_ref,
association.workspace.workspace_generation,
association.workspace.pinned_snapshot_id,
association.workspace.home_snapshot_id,
reconstruction.home_snapshot_id,
reconstruction.home_snapshot_digest,
)
!= (
selection.expected.attempt_id,
relation.successor.binding_ref,
relation.successor.workspace_ref,
prepared.parent.model_dump(mode="json"),
relation.successor.config_ref,
relation.successor.config_digest,
prepared.parent,
prepared.context.native_run_id,
relation.agent_run_id,
relation.successor.config_ref,
relation.successor.config_digest,
relation.successor.binding_id,
relation.successor.workspace_id,
relation.successor.binding_ref,
relation.successor.workspace_ref,
relation.successor.workspace_generation,
prepared.workspace.pinned_snapshot_id,
selection.expected.home_snapshot_id,
selection.expected.home_snapshot_id,
selection.expected.home_snapshot_digest,
)
or canonical_digest(self._agent_executable_config(config.config, prepared.parent)) != config.config_digest
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_association = self._agent_binding(prepared, selection.expected.predecessor.association_object_ref)
original_wait = AgentWaitAssociation.model_validate(evidence["wait"])
if relation.predecessor_waiting_event_id != original_wait.backend_wait_event_id:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
old_checkpoint = AgentCheckpoint.model_validate(
self._agent_wait_relation(prepared, old_association, original_wait)
)
old_config = self._agent_prepared_config(prepared, selection.expected.predecessor.config_ref)
validate_recovery_checkpoint(relation, old_checkpoint, checkpoint)
validate_recovery_configs(
relation,
self.recovery_config_request(
config=old_config.config,
parent=prepared.parent,
create_idempotency_key=old_checkpoint.create_idempotency_key,
),
self.recovery_config_request(
config=config.config, parent=prepared.parent, create_idempotency_key=checkpoint.create_idempotency_key
),
checkpoint,
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
self.recheck_recovery_wait(
session=session, owned=owned, prepared=prepared, evidence=evidence, expected=selection.expected
)
invocation = session.scalar(owner_query(NativeTestInvocation, ref, owned.tenant_id))
if invocation is None:
raise BuilderExecutionPolicyError("native_invocation_mismatch")
session_scope = WorkflowAgentSessionScope(**invocation.bindings["_agent_confirmed_workspace_v1"]["scope"])
return finalize_recovery_selection(
session=session, tenant_id=owned.tenant_id, ref=ref, selection=selection, session_scope=session_scope
)
def _confirmed_snapshot_preparation(self, *, identity: "ConfirmedSnapshotIdentity"):
"""Read exact immutable inputs/config/association outside the SQL writer."""
from dify_agent.protocol.confirmed_recovery import AgentRecoveryBinding
from services.dify_builder.native_testing.agent_recovery import ConfirmedSnapshotIdentity
prepared = self._agent_prepare(
ref=identity.parent_node.ref,
parent_node=identity.parent_node,
writing=False,
expected_generation=identity.parent_generation,
)
association = self._agent_binding(prepared, identity.binding.association_object_ref)
config = self._agent_prepared_config(prepared, identity.binding.config_ref)
workspace = prepared.workspace
actual = ConfirmedSnapshotIdentity(
prepared.parent,
prepared.context.generation,
association.agent_run_id,
TypeAdapter(Counter).validate_python(identity.agent_generation),
AgentRecoveryBinding(
binding_id=workspace.binding_id,
workspace_id=workspace.workspace_id,
binding_ref=workspace.backend_binding_ref,
workspace_ref=workspace.backend_workspace_ref,
workspace_generation=workspace.workspace_generation,
association_object_ref=identity.binding.association_object_ref,
config_ref=config.config_ref,
config_digest=config.config_digest,
),
)
if actual != replace(identity, recovery=None):
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
if actual.agent_generation > 0:
graph = prepared.recovery
if graph is not None and actual.agent_generation == graph.record.agent_generation + 1:
evidence = self.load_result(ref=prepared.parent.ref, result_ref=graph.record.evidence_object_ref)
if not isinstance(evidence, dict):
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
original = CheckpointRef.model_validate(evidence["parent_checkpoint"])
recovery = identity.recovery or self.load_recovery_pending_state(
ref=prepared.parent.ref, original_checkpoint=original
)
if (
recovery is None
or recovery.graph != graph
or recovery.proof.selection.original_checkpoint != original
or actual.parent_generation != recovery.proof.selection.parent_generation + 1
or actual.agent_generation != recovery.proof.selection.wait.agent_generation + 1
or actual.binding.association_object_ref != recovery.proof.selection.wait.association_object_ref
):
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
actual = replace(actual, recovery=recovery)
else:
if identity.recovery is not None:
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
prior = self._agent_object(
prepared, logical_key=self._agent_key("agent-cp", prepared.parent, actual.agent_generation - 1)
)
assert prior is not None
value = prior[1]
if not isinstance(value, dict) or (
value.get("parent_node"),
value.get("association_object_ref"),
value.get("agent_generation"),
) != (
prepared.parent.model_dump(mode="json"),
actual.binding.association_object_ref,
actual.agent_generation - 1,
):
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
return prepared, actual
@staticmethod
def _confirmed_snapshot_execution(session, *, owned, prepared, identity, session_scope=None) -> None:
"""Recheck exact saved scope/execution and generation authority under root lock."""
from core.workflow.nodes.agent_v2.session_store import WorkflowAgentSessionScope
from models.dify_builder_native_tests import NativeDispatch
from models.workflow import WorkflowNodeExecutionModel
from services.dify_builder.native_testing.agent_confirmed_workspace import ALLOCATION
recovery = identity.recovery
if recovery is not None:
# Capture proved current approval state. Save proves the immutable
# selected chain without freezing a later ordinary pending/checkpoint.
NativeExecutionStateStore.recheck_recovery_pending_chain(
session=session,
owned=owned,
state=recovery,
original_checkpoint=recovery.proof.selection.original_checkpoint,
)
selection = recovery.proof.selection
if (
selection.wait.association_object_ref != identity.binding.association_object_ref
or selection.wait.parent_node != identity.parent_node
or selection.parent_generation + 1 != identity.parent_generation
or selection.wait.agent_generation + 1 != identity.agent_generation
):
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
invocation = session.scalar(owner_query(NativeTestInvocation, prepared.parent.ref, prepared.context.tenant_id))
allocation = None if invocation is None else invocation.bindings.get(ALLOCATION)
if not isinstance(allocation, dict) or set(allocation) != {"scope", "workspace"}:
raise BuilderExecutionPolicyError("native_workspace_owner_mismatch")
scope = WorkflowAgentSessionScope(**allocation["scope"])
if session_scope is not None and scope != session_scope:
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
execution = session.scalar(
select(WorkflowNodeExecutionModel)
.where(
WorkflowNodeExecutionModel.id == scope.node_execution_id,
WorkflowNodeExecutionModel.tenant_id == scope.tenant_id,
WorkflowNodeExecutionModel.app_id == scope.app_id,
WorkflowNodeExecutionModel.workflow_id == scope.workflow_id,
WorkflowNodeExecutionModel.workflow_run_id == scope.workflow_run_id,
WorkflowNodeExecutionModel.node_id == scope.node_id,
WorkflowNodeExecutionModel.agent_workspace_binding_id == identity.binding.binding_id,
)
.with_for_update()
.execution_options(populate_existing=True)
)
process = None if execution is None else execution.process_data_dict
if (
scope.node_execution_id != identity.parent_node.execution_id
or not isinstance(process, dict)
or (process.get("workflow_agent_binding_id"), process.get("workflow_tool_invocation_id"))
!= (scope.workflow_agent_binding_id, scope.workflow_tool_invocation_id)
):
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
if identity.agent_generation > 0:
dispatch = session.scalar(
owner_query(NativeDispatch, prepared.parent.ref, prepared.context.tenant_id).where(
NativeDispatch.generation == identity.parent_generation, NativeDispatch.state == "claimed"
)
)
continuation = session.scalar(
owner_query(NativeContinuation, prepared.parent.ref, prepared.context.tenant_id).where(
NativeContinuation.generation == identity.parent_generation - 1
)
)
if dispatch is None or continuation is None:
raise BuilderExecutionPolicyError("native_agent_snapshot_identity_mismatch")
def capture_confirmed_snapshot_identity(
self,
*,
parent_node: AgentParentNodeRef,
parent_generation: int,
agent_run_id: str,
agent_generation: int,
binding: "AgentRecoveryBinding",
) -> "ConfirmedSnapshotIdentity":
"""Capture the exact executable tuple before stream; no save-time relabeling.
Requires original protected ownership and current generation, rechecked
under the root writer. Commands additionally verifies actual SDK identity.
Failure grants no snapshot authority and does not alter any state.
"""
from dify_agent.protocol.confirmed_recovery import AgentRecoveryBinding
from services.dify_builder.native_testing.agent_recovery import (
ConfirmedSnapshotIdentity,
require_recovery_mutation,
)
with _agent_boundary():
identity = ConfirmedSnapshotIdentity(
AgentParentNodeRef.model_validate(parent_node.model_dump(mode="json")),
TypeAdapter(Counter).validate_python(parent_generation),
TypeAdapter(ID).validate_python(agent_run_id),
TypeAdapter(Counter).validate_python(agent_generation),
AgentRecoveryBinding.model_validate(binding.model_dump(mode="json")),
)
prepared, actual = self._confirmed_snapshot_preparation(identity=identity)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=parent_node.ref) as (session, owned, scope):
require_recovery_mutation(
bindings=owned.bindings,
binding_id=identity.binding.binding_id,
workspace_generation=identity.binding.workspace_generation,
)
self._agent_recheck(session, owned, scope, prepared, writing=True)
if actual.recovery is not None:
self.recheck_recovery_pending_state(
session=session,
owned=owned,
state=actual.recovery,
original_checkpoint=actual.recovery.proof.selection.original_checkpoint,
)
self._confirmed_snapshot_execution(session, owned=owned, prepared=prepared, identity=actual)
return actual
def save_confirmed_session_snapshot(
self,
*,
identity: "ConfirmedSnapshotIdentity",
session_scope: "WorkflowAgentSessionScope",
snapshot: "SessionSnapshot | None",
) -> None:
"""Serialize/read/refresh before locking, then recheck and write in one root transaction.
None performs no write or authority acquisition. Reservation and this
write share the same SQL owner. Generic ordinary snapshot APIs are unused.
"""
from services.dify_builder.native_testing.agent_recovery import write_confirmed_snapshot
if snapshot is None:
return
serialized = snapshot.model_dump_json()
with _agent_boundary():
prepared, actual = self._confirmed_snapshot_preparation(identity=identity)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=identity.parent_node.ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=True)
self._confirmed_snapshot_execution(
session, owned=owned, prepared=prepared, identity=actual, session_scope=session_scope
)
write_confirmed_snapshot(
session=session,
tenant_id=owned.tenant_id,
bindings=owned.bindings,
identity=identity,
current=actual,
serialized_snapshot=serialized,
)
def acknowledge_recovery_allocation(
self,
*,
ref: NativeTestContextRef,
expected: "AgentRecoveryRecord",
receipt: "RecoveryAllocationReceipt",
) -> "AgentRecoveryRecord":
"""Protect exact confirmed allocation bytes even after revocation; grants no execution.
Request tenant/Agent/Home are rederived from acknowledged reservation
evidence, including after predecessor/Home lifecycle changes.
Storage/crypto/readback precede the settlement root transaction.
"""
from services.dify_builder.native_testing.agent_recovery import (
RecoveryAllocationReceipt,
settle_recovery_allocation,
validate_allocation_receipt,
)
with _agent_boundary():
receipt = RecoveryAllocationReceipt.model_validate(receipt.model_dump(mode="json"))
with self.loader.session_factory() as session:
owned, _ = self.loader._load(session, ref=ref, lock=False, settlement=True)
evidence_row = session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.id == expected.evidence_object_ref,
NativePayloadObject.kind == "result",
NativePayloadObject.logical_key == "agent-recovery:" + expected.attempt_id + ":evidence",
)
)
if evidence_row is None:
raise BuilderExecutionPolicyError("native_agent_recovery_allocation_mismatch")
evidence_receipt = ObjectReceipt.from_payload_object(evidence_row)
evidence = self._read_object(evidence_receipt)
if not isinstance(evidence, dict) or not isinstance(evidence.get("allocation_owner"), dict):
raise BuilderExecutionPolicyError("native_agent_recovery_allocation_mismatch")
owner = evidence["allocation_owner"]
if (owner.get("tenant_id"), owner.get("home_snapshot_id"), owner.get("home_snapshot_digest")) != (
owned.tenant_id,
expected.home_snapshot_id,
expected.home_snapshot_digest,
):
raise BuilderExecutionPolicyError("native_agent_recovery_allocation_mismatch")
validate_allocation_receipt(
expected=expected,
receipt=receipt,
tenant_id=owned.tenant_id,
agent_id=TypeAdapter(ID).validate_python(owner["agent_id"]),
home_snapshot_ref=TypeAdapter(ID | None).validate_python(owner["home_snapshot_ref"]),
)
protected = self.prepare_object(
ref=ref,
tenant_id=owned.tenant_id,
kind="result",
logical_key="agent-recovery:" + expected.attempt_id + ":allocation",
payload=receipt.model_dump(mode="json"),
)
with self.loader.locked(ref=ref, settlement=True) as (session, current_owner, _scope):
current_evidence = session.scalar(
owner_query(NativePayloadObject, ref, current_owner.tenant_id).where(
NativePayloadObject.id == expected.evidence_object_ref, NativePayloadObject.kind == "result"
)
)
if current_evidence is None or ObjectReceipt.from_payload_object(current_evidence) != evidence_receipt:
raise BuilderExecutionPolicyError("native_agent_recovery_allocation_mismatch")
selected = self.acknowledge_object(session, protected)
return settle_recovery_allocation(
session=session,
tenant_id=current_owner.tenant_id,
ref=ref,
expected=expected,
allocation_receipt_ref=selected.id,
)
def mark_recovery_allocation_unknown(
self,
*,
ref: NativeTestContextRef,
expected: "AgentRecoveryRecord",
) -> "AgentRecoveryRecord":
"""Fence unproven create outcomes permanently; retain an already committed ACK.
This is settlement only, including after revocation. Never reserves or
retries physical creation. A conflicting attempt refuses without changes.
"""
from services.dify_builder.native_testing.agent_recovery import recovery_record
from services.dify_builder.native_testing.agent_recovery_state import (
RECOVERY_RECORD_KEY,
AgentRecoveryRecord,
advance_recovery_record,
)
with self.loader.locked(ref=ref, settlement=True) as (session, owned, _scope):
invocation = session.scalar(owner_query(NativeTestInvocation, ref, owned.tenant_id).with_for_update())
if invocation is None:
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
current = recovery_record(invocation.bindings)
if (
current is not None
and current.attempt_id == expected.attempt_id
and current.allocation_receipt_ref is not None
):
return current
if current != expected or current is None or current.phase != "creating":
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
unknown = advance_recovery_record(
current, AgentRecoveryRecord.model_validate({**current.model_dump(), "phase": "unknown"})
)
invocation.bindings = {**invocation.bindings, RECOVERY_RECORD_KEY: unknown.model_dump(mode="json")}
return unknown
def prepare_object(
self,
*,
ref: NativeTestContextRef,
tenant_id: str,
kind: PayloadKind,
logical_key: str,
payload: JsonValue,
max_bytes: int | None = None,
) -> ObjectReceipt:
"""Internal producer port: caller must subsequently acknowledge under the root lock."""
raw, digest = encode_payload(
tenant_id=tenant_id,
request_id=ref.request_id,
invocation_id=ref.invocation_id,
kind=kind,
payload=payload,
)
identity = str(uuid4())
key = f"builder-native/{identity}"
encrypted = encrypter.encrypt_token(tenant_id, raw).encode("ascii")
if max_bytes is not None and len(encrypted) > max_bytes:
raise BuilderExecutionPolicyError("native_agent_config_unavailable")
storage.save(key, encrypted)
readback = storage.load(key)
if max_bytes is not None and len(readback) > max_bytes:
raise BuilderExecutionPolicyError("native_agent_config_unavailable")
roundtrip = encrypter.decrypt_token(tenant_id, readback.decode("ascii"))
if max_bytes is not None and len(roundtrip.encode("utf-8")) > max_bytes:
raise BuilderExecutionPolicyError("native_agent_config_unavailable")
value = decode_payload(
roundtrip,
tenant_id=tenant_id,
request_id=ref.request_id,
invocation_id=ref.invocation_id,
kind=kind,
digest=digest,
)
return ObjectReceipt(
identity,
tenant_id,
ref.request_id,
ref.invocation_id,
kind,
logical_key,
key,
digest,
canonical_digest(value),
)
@staticmethod
def acknowledge_object(session, receipt: ObjectReceipt) -> NativePayloadObject:
existing = session.scalar(
select(NativePayloadObject).where(
NativePayloadObject.tenant_id == receipt.tenant_id,
NativePayloadObject.request_id == receipt.request_id,
NativePayloadObject.invocation_id == receipt.invocation_id,
NativePayloadObject.kind == receipt.kind,
NativePayloadObject.logical_key == receipt.logical_key,
)
)
if existing is not None:
if (existing.payload_digest, existing.value_digest) != (receipt.payload_digest, receipt.value_digest):
raise BuilderExecutionPolicyError("native_payload_conflict")
return existing
row = NativePayloadObject(**vars(receipt))
session.add(row)
session.flush()
return row
@staticmethod
def _read_object(row: NativePayloadObject | ObjectReceipt, *, max_bytes: int | None = None) -> JsonValue:
started, error_type = perf_counter(), "none"
try:
receipt = row if isinstance(row, ObjectReceipt) else ObjectReceipt.from_payload_object(row)
reads = _payload_reads.get()
if reads is not None and not reads.active:
reads = None
key = receipt, max_bytes
stored = reads.encrypted.get(key) if reads is not None else None
if stored is None:
stored = storage.load(row.storage_key)
if max_bytes is not None and len(stored) > max_bytes:
raise BuilderExecutionPolicyError("native_agent_config_unavailable")
raw = encrypter.decrypt_token(row.tenant_id, stored.decode("ascii"))
if max_bytes is not None and len(raw.encode("utf-8")) > max_bytes:
raise BuilderExecutionPolicyError("native_agent_config_unavailable")
value = decode_payload(
raw,
tenant_id=row.tenant_id,
request_id=row.request_id,
invocation_id=row.invocation_id,
kind=row.kind,
digest=row.payload_digest,
)
if canonical_digest(value) != row.value_digest:
raise BuilderExecutionPolicyError("native_payload_invalid")
if (
reads is not None
and reads.active
and key not in reads.encrypted
and len(reads.encrypted) < _PAYLOAD_READ_MAX_ENTRIES
and len(stored) <= _PAYLOAD_READ_MAX_BYTES - reads.byte_count
):
reads.encrypted[key] = stored
reads.byte_count += len(stored)
return value
except BuilderExecutionPolicyError as error:
error_type = type(error).__name__
raise
except Exception as error:
error_type = type(error).__name__
raise BuilderExecutionPolicyError("native_payload_unavailable") from None
finally:
# This leaf duration includes storage, key-provider I/O and decoding.
# Never log the protected value, storage key or exception message.
with suppress(Exception): # Diagnostics cannot change protected reads.
logger.info(
"Native read timing request_id=%s invocation_id=%s phase=protected_object_read "
"duration_seconds=%.6f error_type=%s",
row.request_id,
row.invocation_id,
perf_counter() - started,
error_type,
)
def initialize_context(self, *, context: ResolvedNativeTestContext) -> None:
"""Trusted admission only, after C02 request/scope creation; child creation has its own port."""
context = ResolvedNativeTestContext.model_validate(context.model_dump(mode="json"))
owned = self.loader.load(ref=context.ref)
if (
context.tenant_id,
context.actor_id,
context.app_id,
context.workflow_id,
context.parent_invocation_id,
context.generation,
context.call_depth,
context.budget_ref,
context.cancellation_ref,
) != (
owned.tenant_id,
owned.actor_id,
owned.app_id,
owned.workflow_id,
None,
owned.generation,
0,
owned.ref.request_id,
owned.ref.request_id,
):
raise BuilderExecutionPolicyError("native_context_mismatch")
with self.loader.session_factory() as session:
workflow = session.get(Workflow, owned.workflow_id)
test_input = session.get(DifyBuilderTestInput, owned.test_input_id)
revision_context = owned.revision_context
if (
workflow is None
or test_input is None
or not admitted_graph_matches(workflow, context.graph, revision_context)
or context.graph_revision
!= (
admitted_workflow_revision(workflow, revision_context)
if revision_context
else native_workflow_revision(workflow)
)
or canonical_digest(context.inputs) != canonical_digest(test_input.inputs)
or context.input_revision != canonical_digest(context.inputs)
):
raise BuilderExecutionPolicyError("native_snapshot_mismatch")
receipt = self.prepare_object(
ref=context.ref,
tenant_id=owned.tenant_id,
kind="context",
logical_key="context",
payload=context.model_dump(mode="json"),
)
with self.loader.locked(ref=context.ref) as (session, current, _):
if current.generation != context.generation:
raise BuilderExecutionPolicyError("native_generation_conflict")
row = self.acknowledge_object(session, receipt)
existing = session.scalar(owner_query(NativeInvocationState, context.ref, owned.tenant_id))
if existing is None:
session.add(
NativeInvocationState(
tenant_id=owned.tenant_id,
request_id=context.ref.request_id,
invocation_id=context.ref.invocation_id,
context_object_id=row.id,
app_id=context.app_id,
workflow_id=context.workflow_id,
parent_invocation_id=None,
parent_operation_digest=None,
child_identity_digest=None,
generation=context.generation,
state="running",
checkpoint_id=None,
pending_ids=[],
wait_reason=None,
outcome=None,
resolution=[],
cancellation_key=None,
)
)
elif existing.context_object_id != row.id:
raise BuilderExecutionPolicyError("native_snapshot_mismatch")
def load_context(self, *, ref: NativeTestContextRef) -> ResolvedNativeTestContext:
return self._load_context(ref=ref, recovery=False)
def load_projection_context(self, *, ref: NativeTestContextRef) -> ResolvedNativeTestContext:
"""Current owned read visibility survives expiry; this grants no execution."""
return self._load_context(ref=ref, recovery=False, allow_expired=True)
def load_recovery_context(self, *, ref: NativeTestContextRef) -> ResolvedNativeTestContext:
"""Read-only recovery material; an inactive invocation gains no authority."""
return self._load_context(ref=ref, recovery=True)
def _load_context(
self, *, ref: NativeTestContextRef, recovery: bool, allow_expired: bool = False
) -> ResolvedNativeTestContext:
owned = validate_private_read(self.loader, ref)
if owned.cancelled:
raise BuilderExecutionPolicyError("native_cancelled")
if not recovery and (not owned.active or owned.request_state not in {"running", "waiting"}):
raise BuilderExecutionPolicyError("native_invocation_mismatch")
if not allow_expired and owned.expires_at <= utc_now():
raise BuilderExecutionPolicyError("native_expired")
with self.loader.session_factory() as session:
row = session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.id == owned.context_object_id, NativePayloadObject.kind == "context"
)
)
if row is None:
raise BuilderExecutionPolicyError("native_state_unavailable")
context = ResolvedNativeTestContext.model_validate(self._read_object(row))
if (
context.ref,
context.tenant_id,
context.actor_id,
context.app_id,
context.workflow_id,
context.parent_invocation_id,
) != (ref, owned.tenant_id, owned.actor_id, owned.app_id, owned.workflow_id, owned.parent_invocation_id):
raise BuilderExecutionPolicyError("native_context_mismatch")
if not owned.root_revision_matches or context.graph_revision != owned.graph_revision:
raise BuilderExecutionPolicyError("native_revision_drift")
if owned.input_revision is not None and owned.input_revision != context.input_revision:
raise BuilderExecutionPolicyError("native_input_drift")
return context.model_copy(update={"generation": owned.generation})
def load_terminal_context(self, *, ref: NativeTestContextRef) -> ResolvedNativeTestContext:
"""Observe an already terminal owned Run after expiry, never a continuation."""
from models.enums import CreatorUserRole
from models.workflow import WorkflowRun
context = self._load_context(ref=ref, recovery=False, allow_expired=True)
with self.loader.session_factory() as session:
run = session.scalar(
select(WorkflowRun).where(
WorkflowRun.id == context.native_run_id,
WorkflowRun.tenant_id == context.tenant_id,
WorkflowRun.app_id == context.app_id,
WorkflowRun.workflow_id == context.workflow_id,
WorkflowRun.created_by == context.actor_id,
WorkflowRun.created_by_role == CreatorUserRole.ACCOUNT,
)
)
if run is None or run.status not in {"succeeded", "failed", "partial-succeeded", "stopped"}:
raise BuilderExecutionPolicyError("native_expired")
return context
def commit_checkpoint(
self, *, ref: NativeTestContextRef, generation: int, payload: NativeCheckpointPayload
) -> CheckpointRef:
try:
payload = NativeCheckpointPayload.model_validate(payload.model_dump(mode="json"))
context = self.load_context(ref=ref)
if type(generation) is not int or generation != context.generation or payload.generation != generation:
raise BuilderExecutionPolicyError("native_generation_conflict")
if context.native_run_id != payload.native_run_id:
raise BuilderExecutionPolicyError("native_run_mismatch")
agent_preparations = self._agent_checkpoint_preparations(ref, payload)
# Validate every referenced private input/result and child owner before storing.
for result_ref in (
*payload.input_result_refs,
*(r for j in payload.node_journals for r in (j.input_result_ref, *j.result_refs)),
):
self.load_result(ref=ref, result_ref=result_ref)
for journal in payload.node_journals:
if journal.invocation_id != ref.invocation_id or any(
k.invocation_id != ref.invocation_id for k in journal.operation_keys
):
raise BuilderExecutionPolicyError("native_journal_owner_mismatch")
if (
canonical_digest(self.load_result(ref=ref, result_ref=journal.input_result_ref))
!= journal.input_digest
):
raise BuilderExecutionPolicyError("native_journal_input_mismatch")
with self.loader.session_factory() as session:
for child in payload.child_refs:
state = invocation_state(session, child, context.tenant_id)
if (
child.request_id != ref.request_id
or child.context_digest != ref.context_digest
or state.parent_invocation_id != ref.invocation_id
):
raise BuilderExecutionPolicyError("native_child_owner_mismatch")
receipt = self.prepare_object(
ref=ref,
tenant_id=context.tenant_id,
kind="checkpoint",
logical_key=str(generation),
payload=payload.model_dump(
mode="json", exclude={"agent_associations"} if not payload.agent_associations else set()
),
)
with self.loader.locked(ref=ref) as (session, owned, scope):
state = invocation_state(session, ref, owned.tenant_id)
if state.generation != generation or state.state not in {"running", "waiting"}:
raise BuilderExecutionPolicyError("native_generation_conflict")
for prepared in agent_preparations:
self._agent_recheck(session, owned, scope, prepared, writing=False)
obj = self.acknowledge_object(session, receipt)
prior = session.scalar(
owner_query(NativeContinuation, ref, owned.tenant_id).where(
NativeContinuation.generation == generation
)
)
if prior is None:
prior = NativeContinuation(
tenant_id=owned.tenant_id,
request_id=ref.request_id,
invocation_id=ref.invocation_id,
generation=generation,
checkpoint_id=obj.id,
payload_digest=obj.payload_digest,
payload_ref=obj.storage_key,
)
session.add(prior)
result = CheckpointRef(checkpoint_id=prior.checkpoint_id, payload_digest=prior.payload_digest)
return result
except BuilderExecutionPolicyError:
raise
except Exception:
raise BuilderExecutionPolicyError("checkpoint_unavailable") from None
def read_checkpoint_for_cancellation(
self, *, ref: NativeTestContextRef, checkpoint: CheckpointRef
) -> NativeCheckpointPayload:
"""Active cancellation metadata only; expiry grants no new execution.
Current actor, revision, invocation and private Agent resource checks
remain mandatory. Revoked or inactive requests cannot enter this path.
"""
self.loader.load(ref=ref)
return self.read_checkpoint(ref=ref, checkpoint=checkpoint, for_projection=True)
def read_checkpoint(
self, *, ref: NativeTestContextRef, checkpoint: CheckpointRef, for_projection: bool = False
) -> NativeCheckpointPayload:
"""Read current protected facts; only server-selected projection ignores TTL.
Projection retains active ownership, revision, input and Agent resource
checks and cannot enter a writing Agent preparation or execution path.
"""
owned = validate_private_read(self.loader, ref)
with self.loader.session_factory() as session:
continuation = session.scalar(
owner_query(NativeContinuation, ref, owned.tenant_id).where(
NativeContinuation.checkpoint_id == checkpoint.checkpoint_id,
NativeContinuation.payload_digest == checkpoint.payload_digest,
)
)
row = session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.id == checkpoint.checkpoint_id, NativePayloadObject.kind == "checkpoint"
)
)
if (
continuation is None
or row is None
or row.payload_digest != checkpoint.payload_digest
or row.storage_key != continuation.payload_ref
):
raise BuilderExecutionPolicyError("checkpoint_unavailable")
payload = NativeCheckpointPayload.model_validate(self._read_object(row))
if payload.agent_associations:
with _agent_boundary():
if payload.generation != continuation.generation:
raise BuilderExecutionPolicyError("native_generation_conflict")
prepared = self._agent_checkpoint_preparations(ref, payload, for_projection=for_projection)
outer = ObjectReceipt.from_payload_object(row)
with self.loader.locked(ref=ref) as (session, current, scope):
selected_continuation = session.scalar(
owner_query(NativeContinuation, ref, current.tenant_id).where(
NativeContinuation.checkpoint_id == checkpoint.checkpoint_id,
NativeContinuation.payload_digest == checkpoint.payload_digest,
)
)
selected_object = session.scalar(
owner_query(NativePayloadObject, ref, current.tenant_id).where(
NativePayloadObject.id == checkpoint.checkpoint_id, NativePayloadObject.kind == "checkpoint"
)
)
if selected_continuation is None or selected_continuation.generation != payload.generation:
raise BuilderExecutionPolicyError("native_generation_conflict")
if (
selected_object is None
or selected_continuation.payload_ref != outer.storage_key
or ObjectReceipt.from_payload_object(selected_object) != outer
):
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
for selected in prepared:
self._agent_recheck(session, current, scope, selected, writing=False)
return payload
def store_result(self, *, ref: NativeTestContextRef, key: OperationKey, result: JsonValue) -> str:
if key.invocation_id != ref.invocation_id:
raise BuilderExecutionPolicyError("native_result_owner_mismatch")
logical_key = operation_digest(key)
# A cancelled/revoked invocation may still save the result of an already
# spent operation, matching C02's settlement path. This creates no I/O.
with self.loader.session_factory() as session:
owned, _ = self.loader._load(session, ref=ref, lock=False, settlement=True)
spent = (
session.scalar(
owner_query(NativeEffectOperation, ref, owned.tenant_id).where(
NativeEffectOperation.operation_digest == logical_key,
NativeEffectOperation.state.in_(("claimed", "completed")),
)
)
is not None
)
if not spent:
self.loader.load(ref=ref)
validate_private_read(self.loader, ref)
receipt = self.prepare_object(
ref=ref, tenant_id=owned.tenant_id, kind="result", logical_key=logical_key, payload=result
)
with self.loader.locked(ref=ref, settlement=spent) as (session, _, _):
row = self.acknowledge_object(session, receipt)
result_ref = row.id
return result_ref
def store_node_inputs(
self, *, ref: NativeTestContextRef, generation: int, native_execution_id: str, inputs: JsonValue
) -> str:
"""Persist typed inputs independently of effect result slots, without issuing authority.
The trusted producer must obtain the UUID from its attached, bound native
node. UUID syntax alone establishes no native execution provenance.
"""
try:
ref = NativeTestContextRef.model_validate(ref.model_dump(warnings=False), strict=True)
if type(native_execution_id) is not str or str(UUID(native_execution_id)) != native_execution_id:
raise BuilderExecutionPolicyError("native_run_mismatch")
logical_key = "node-input:" + native_execution_id
if len(logical_key) > 128:
raise BuilderExecutionPolicyError("native_payload_invalid")
inputs = _JSON.validate_python(inputs)
self.loader.load(ref=ref)
authority = validate_private_read(self.loader, ref)
context = self.load_context(ref=ref)
if type(generation) is not int or generation != context.generation:
raise BuilderExecutionPolicyError("native_generation_conflict")
with self.loader.session_factory() as session:
owned, scope = self.loader._load(session, ref=ref, lock=False)
self._validate_node_inputs_owner(session, owned, scope, context, authority, generation)
receipt = self.prepare_object(
ref=ref, tenant_id=context.tenant_id, kind="result", logical_key=logical_key, payload=inputs
)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._validate_node_inputs_owner(session, owned, scope, context, authority, generation)
result_ref = self.acknowledge_object(session, receipt).id
return result_ref
except BuilderExecutionPolicyError:
raise
except Exception:
raise BuilderExecutionPolicyError("native_payload_unavailable") from None
@staticmethod
def _validate_node_inputs_owner(
session,
owned,
scope,
context: ResolvedNativeTestContext,
authority: PrivateReadOwner,
generation: int,
*,
allowed_states: tuple[str, ...] = ("running",),
allow_expired: bool = False,
) -> None:
"""SQL-only preflight/recheck; caller owns the transaction and any root lock."""
state = invocation_state(session, context.ref, owned.tenant_id)
# The scope counter advances on resume; unrelated running branches keep
# their own invocation generation and remain eligible to persist inputs.
if (context.generation, state.generation, authority.generation) != (generation,) * 3:
raise BuilderExecutionPolicyError("native_generation_conflict")
if not allow_expired and scope.expires_at <= utc_now():
raise BuilderExecutionPolicyError("native_expired")
if state.state not in allowed_states:
raise BuilderExecutionPolicyError("native_invocation_mismatch")
if (
context.ref,
context.tenant_id,
context.actor_id,
context.app_id,
context.workflow_id,
context.parent_invocation_id,
authority.context_object_id,
) != (
owned.ref,
owned.tenant_id,
owned.actor_id,
state.app_id,
state.workflow_id,
state.parent_invocation_id,
state.context_object_id,
):
raise BuilderExecutionPolicyError("native_context_mismatch")
require_ancestor_authority(session, context.ref, owned.tenant_id)
request = session.get(DifyBuilderExecutionRequest, context.ref.request_id)
if (
context.execution_mode != "native"
or not context.native_run_id
or not context.root_native_run_id
or request is None
or request.context.get("mode") != "native"
or request.context.get("native_run_id") != context.root_native_run_id
or (context.parent_invocation_id is None and context.native_run_id != context.root_native_run_id)
):
raise BuilderExecutionPolicyError("native_run_mismatch")
workflow = validate_invocation_workflow(
session, tenant_id=owned.tenant_id, app_id=context.app_id, workflow_id=context.workflow_id
)
revision_context = owned.revision_context
revision = (
admitted_workflow_revision(workflow, revision_context)
if revision_context
else native_workflow_revision(workflow)
)
if revision != context.graph_revision or not admitted_graph_matches(workflow, context.graph, revision_context):
raise BuilderExecutionPolicyError("native_revision_drift")
def load_result(self, *, ref: NativeTestContextRef, result_ref: str) -> JsonValue:
owned = validate_private_read(self.loader, ref)
with self.loader.session_factory() as session:
row = session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.id == result_ref, NativePayloadObject.kind == "result"
)
)
if row is None:
raise BuilderExecutionPolicyError("native_result_owner_mismatch")
return self._read_object(row)
@staticmethod
def _agent_key(namespace: str, parent: AgentParentNodeRef, generation: int | None = None) -> str:
key = namespace + ":" + canonical_digest(parent.model_dump(mode="json"))
if generation is not None:
key += ":" + str(TypeAdapter(Counter).validate_python(generation))
if len(key) > 128:
raise BuilderExecutionPolicyError("native_agent_state_unavailable")
return key
@staticmethod
def _agent_resources(
session,
owned,
context: ResolvedNativeTestContext,
*,
definition_receipt: AgentDefinitionReceipt | None = None,
recovery: "SelectedRecovery | None" = None,
) -> tuple[AgentTestWorkspaceRef, str]:
"""Current SQL allocation/pin/workspace chain only; no physical/backend IO."""
invocation = session.scalar(owner_query(NativeTestInvocation, context.ref, owned.tenant_id))
if (
invocation is None
or not invocation.active
or (invocation.tenant_id, invocation.request_id, invocation.invocation_id)
!= (context.tenant_id, context.ref.request_id, context.ref.invocation_id)
):
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
confirmed = invocation.bindings.get("_agent_confirmed_workspace_v1")
if confirmed is not None:
from services.dify_builder.native_testing.agent_confirmed_workspace import current_confirmed_workspace
return current_confirmed_workspace(
session=session,
owned=owned,
context=context,
allocation=confirmed,
definition_receipt=definition_receipt,
recovery=recovery,
)
allocation = invocation.bindings.get("_agent_workspace_v1")
if not isinstance(allocation, dict):
raise BuilderExecutionPolicyError("native_workspace_unavailable")
# Local import avoids the existing workspace service -> state owner dependency.
# This is its pure parser/ref conversion, not workspace/backend IO.
from services.dify_builder.native_testing.agent_workspace import (
AgentWorkspaceAllocation,
_ref,
)
parsed_allocation = AgentWorkspaceAllocation.model_validate(allocation)
if any(
allocation.get(key) != value
for key, value in {
"tenant_id": context.tenant_id,
"app_id": context.app_id,
"request_id": context.ref.request_id,
"invocation_id": context.ref.invocation_id,
"owner_run_id": context.native_run_id,
"phase": "ready",
}.items()
):
raise BuilderExecutionPolicyError("native_workspace_owner_mismatch")
scope = allocation.get("native_scope")
if not isinstance(scope, dict):
raise BuilderExecutionPolicyError("native_workspace_owner_mismatch")
binding_id = TypeAdapter(ID).validate_python(allocation.get("binding_id"))
TypeAdapter(Counter).validate_python(scope.get("generation"))
if scope.get("scope_id") != binding_id or scope.get("binding_ref") != binding_id:
raise BuilderExecutionPolicyError("native_workspace_owner_mismatch")
if parsed_allocation.native_scope.deadline <= utc_now():
raise BuilderExecutionPolicyError("native_workspace_unavailable")
workspace_ref = _ref(parsed_allocation)
snapshot = session.get(AgentConfigSnapshot, workspace_ref.pinned_snapshot_id)
home = session.get(AgentHomeSnapshot, workspace_ref.home_snapshot_id)
workspace = session.get(AgentWorkspace, workspace_ref.workspace_id)
binding = session.get(AgentWorkspaceBinding, workspace_ref.binding_id)
if snapshot is None or home is None or workspace is None or binding is None:
raise BuilderExecutionPolicyError("native_workspace_owner_mismatch")
agent_id = TypeAdapter(ID).validate_python(allocation.get("agent_id"))
from services.agent import access_service
access_service.require_builder_agent_access(
session=session,
tenant_id=context.tenant_id,
actor_id=context.actor_id,
app_id=context.app_id,
)
agent = session.get(Agent, agent_id)
if agent is None or agent.tenant_id != context.tenant_id or agent.active_config_snapshot_id != snapshot.id:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
if (
(snapshot.tenant_id, snapshot.agent_id, snapshot.home_snapshot_id) != (context.tenant_id, agent_id, home.id)
or (home.tenant_id, home.agent_id, home.status, home.snapshot_ref)
!= (
context.tenant_id,
agent_id,
AgentWorkingResourceStatus.ACTIVE,
scope.get("base_snapshot_ref"),
)
or (
workspace.tenant_id,
workspace.app_id,
workspace.owner_type,
workspace.owner_id,
workspace.owner_scope_key,
workspace.status,
workspace.backend_workspace_ref,
)
!= (
context.tenant_id,
context.app_id,
AgentWorkspaceOwnerType.WORKFLOW_RUN,
context.native_run_id,
"native:" + context.ref.invocation_id,
AgentWorkingResourceStatus.ACTIVE,
workspace_ref.backend_workspace_ref,
)
or (
binding.tenant_id,
binding.app_id,
binding.workspace_id,
binding.agent_id,
binding.base_home_snapshot_id,
binding.agent_config_version_id,
binding.agent_config_version_kind,
binding.backend_binding_ref,
binding.status,
)
!= (
context.tenant_id,
context.app_id,
workspace.id,
agent_id,
home.id,
snapshot.id,
AgentConfigVersionKind.SNAPSHOT,
workspace_ref.backend_binding_ref,
AgentWorkingResourceStatus.ACTIVE,
)
):
raise BuilderExecutionPolicyError("native_workspace_owner_mismatch")
nodes = context.graph.get("nodes", [])
if not isinstance(nodes, list) or not any(
isinstance(node, dict)
and isinstance(node.get("data"), dict)
and isinstance(node["data"].get("agent_binding"), dict)
and node["data"]["agent_binding"].get("current_snapshot_id") == snapshot.id
and node["data"]["agent_binding"].get("agent_id") == agent_id
for node in nodes
):
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
from core.agent.publish_visibility import (
workflow_callable_active_snapshot_filter,
)
from services.agent.roster_service import AgentRosterService
eligible = session.scalar(
AgentRosterService._build_roster_agents_stmt(tenant_id=context.tenant_id, keyword=None).where(
Agent.id == agent_id,
Agent.active_config_has_model.is_(True),
workflow_callable_active_snapshot_filter(),
)
)
if eligible is None:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
workflow = validate_invocation_workflow(
session,
tenant_id=context.tenant_id,
app_id=context.app_id,
workflow_id=context.workflow_id,
)
definitions = []
matched_definition = False
for node in nodes:
if not isinstance(node, dict) or not isinstance(node.get("data"), dict):
continue
selected = node["data"].get("agent_binding")
if not isinstance(selected, dict) or (
selected.get("agent_id"),
selected.get("current_snapshot_id"),
) != (
agent_id,
snapshot.id,
):
continue
published_binding = session.scalar(
select(WorkflowAgentNodeBinding).where(
WorkflowAgentNodeBinding.tenant_id == context.tenant_id,
WorkflowAgentNodeBinding.app_id == context.app_id,
WorkflowAgentNodeBinding.workflow_id == context.workflow_id,
WorkflowAgentNodeBinding.workflow_version == workflow.version,
WorkflowAgentNodeBinding.node_id == node.get("id"),
WorkflowAgentNodeBinding.agent_id == agent_id,
WorkflowAgentNodeBinding.current_snapshot_id == snapshot.id,
WorkflowAgentNodeBinding.binding_type == WorkflowAgentBindingType.ROSTER_AGENT,
)
)
if published_binding is None:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
definitions.append({"node": node, "node_job": published_binding.node_job_config_dict})
if definition_receipt is not None and node.get("id") == definition_receipt.node_id:
# Compare the producing definition, not a fresh stamp assigned to
# already-composed bytes after the claim's initial async reads.
stamp = canonical_digest(
{
"node": node,
"soul": AgentSoulConfig.model_validate(snapshot.config_snapshot_dict).model_dump(mode="json"),
"job": WorkflowNodeJobConfig.model_validate(published_binding.node_job_config_dict).model_dump(
mode="json"
),
}
)
if stamp != definition_receipt.stamp:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
matched_definition = True
if definition_receipt is not None and not matched_definition:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
if not definitions:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
return workspace_ref, canonical_digest(
{
"allocation": allocation,
"workspace": workspace_ref.model_dump(mode="json"),
"snapshot": snapshot.config_snapshot_dict,
"definitions": definitions,
}
)
def _agent_object(
self,
preparation: _AgentPreparation,
*,
logical_key: str,
object_ref: str | None = None,
required: bool = True,
) -> tuple[ObjectReceipt, JsonValue] | None:
with self.loader.session_factory() as session:
query = owner_query(
NativePayloadObject,
preparation.parent.ref,
preparation.context.tenant_id,
).where(
NativePayloadObject.kind == "result",
NativePayloadObject.logical_key == logical_key,
)
if object_ref is not None:
query = query.where(NativePayloadObject.id == TypeAdapter(ID).validate_python(object_ref))
row = session.scalar(query)
if row is None:
if required:
raise BuilderExecutionPolicyError("native_agent_object_unavailable")
return None
receipt = ObjectReceipt.from_payload_object(row)
if (
receipt.tenant_id,
receipt.request_id,
receipt.invocation_id,
receipt.kind,
receipt.logical_key,
) != (
preparation.context.tenant_id,
preparation.parent.ref.request_id,
preparation.parent.ref.invocation_id,
"result",
logical_key,
) or (object_ref is not None and receipt.id != object_ref):
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
value = self._read_object(
receipt,
max_bytes=1024 * 1024 if logical_key.startswith("agent-config:") else None,
)
preparation.objects.append(receipt)
return receipt, value
def _agent_prepare(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
writing: bool,
expected_generation: int | None = None,
settlement: bool = False,
for_projection: bool = False,
) -> _AgentPreparation:
if for_projection and writing:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
parent = AgentParentNodeRef.model_validate(parent_node.model_dump(mode="json"))
if ref != parent.ref:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
self.loader.load(ref=ref)
authority = validate_private_read(self.loader, ref)
context = (
self.load_projection_context(ref=ref)
if for_projection
else self._load_context(ref=ref, recovery=False, allow_expired=True)
if settlement
else self.load_context(ref=ref)
)
if (
expected_generation is not None
and TypeAdapter(Counter).validate_python(expected_generation) != context.generation
):
raise BuilderExecutionPolicyError("native_generation_conflict")
with self.loader.session_factory() as session:
selected_owner, _selected_scope = self.loader._load(session, ref=ref, lock=False)
selected_recovery = self._selected_recovery_graph(owned=selected_owner, context=context, parent=parent)
with self.loader.session_factory() as session:
owned, scope = self.loader._load(session, ref=ref, lock=False)
self._validate_node_inputs_owner(
session,
owned,
scope,
context,
authority,
context.generation,
allowed_states=("running",) if writing else ("running", "waiting", "queued"),
allow_expired=settlement or for_projection,
)
self._agent_single_parent(session, context, parent)
workspace, stamp = self._agent_resources(session, owned, context, recovery=selected_recovery)
prepared = _AgentPreparation(
parent,
context,
authority,
workspace,
stamp,
[],
settlement=settlement,
for_projection=for_projection,
recovery=selected_recovery,
)
selected = self._agent_object(
prepared, logical_key="node-input:" + parent.execution_id, object_ref=parent.input_result_ref
)
assert selected is not None
if canonical_digest(selected[1]) != parent.input_digest:
raise BuilderExecutionPolicyError("native_journal_input_mismatch")
return prepared
def _agent_recheck(
self,
session,
owned,
scope,
prepared: _AgentPreparation,
*,
writing: bool,
definition_receipt: AgentDefinitionReceipt | None = None,
) -> None:
if (prepared.settlement or prepared.for_projection) and writing:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
if writing and "_agent_confirmed_recovery_v1" in owned.bindings:
from services.dify_builder.native_testing.agent_recovery import require_recovery_mutation
require_recovery_mutation(
bindings=owned.bindings,
binding_id=prepared.workspace.binding_id,
workspace_generation=prepared.workspace.workspace_generation,
)
self._validate_node_inputs_owner(
session,
owned,
scope,
prepared.context,
prepared.authority,
prepared.context.generation,
allowed_states=("running",) if writing else ("running", "waiting", "queued"),
allow_expired=prepared.settlement or prepared.for_projection,
)
self._agent_single_parent(session, prepared.context, prepared.parent)
workspace, stamp = self._agent_resources(
session, owned, prepared.context, definition_receipt=definition_receipt, recovery=prepared.recovery
)
if workspace != prepared.workspace or stamp != prepared.resource_digest:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
for receipt in prepared.objects:
row = session.scalar(
owner_query(NativePayloadObject, prepared.parent.ref, owned.tenant_id).where(
NativePayloadObject.id == receipt.id,
NativePayloadObject.kind == "result",
)
)
if row is None or ObjectReceipt.from_payload_object(row) != receipt:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
def _agent_ack(self, prepared: _AgentPreparation, *, logical_key: str, value: JsonValue) -> str:
existing = self._agent_object(prepared, logical_key=logical_key, required=False)
receipt = None
if existing is not None:
if canonical_digest(existing[1]) != canonical_digest(value):
raise BuilderExecutionPolicyError("native_payload_conflict")
else:
receipt = self.prepare_object(
ref=prepared.parent.ref,
tenant_id=prepared.context.tenant_id,
kind="result",
logical_key=logical_key,
payload=value,
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=prepared.parent.ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=True)
if receipt is not None:
return self.acknowledge_object(session, receipt).id
assert existing is not None
return existing[0].id
def _agent_refresh_access(self, prepared: _AgentPreparation) -> None:
"""Current RBAC outside SQL writer, followed by complete local recheck under lock."""
from services.dify_builder.native_testing.agent_binding import (
AgentBindingService,
)
if validate_private_read(self.loader, prepared.parent.ref) != prepared.authority:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
with self.loader.session_factory() as session:
snapshot = session.get(AgentConfigSnapshot, prepared.workspace.pinned_snapshot_id)
if snapshot is None or snapshot.tenant_id != prepared.context.tenant_id:
raise BuilderExecutionPolicyError("native_agent_pin_mismatch")
agent_id = snapshot.agent_id
AgentBindingService.authorize(
actor=self.loader.actor,
app_id=prepared.context.app_id,
agent_ids=frozenset({agent_id}),
)
@staticmethod
def _agent_executable_config(config: JsonValue, parent: AgentParentNodeRef) -> JsonValue:
"""No metadata, secrets, assigned config ref, or caller ownership fields persist."""
from dify_agent.protocol.schemas import CreateRunRequest
if not isinstance(config, dict) or set(config) != {
"composition",
"session_snapshot",
}:
raise BuilderExecutionPolicyError("native_agent_config_mismatch")
request = CreateRunRequest.model_validate(
{
**config,
"execution_mode": "builder_test",
"native_test_ref": parent.ref.model_dump(mode="json"),
"native_test_protocol_version": 1,
"native_parent_node": parent.model_dump(mode="json"),
}
)
# Layer transport fields can contain only opaque references. Empty credential
# maps are required by existing DTOs; populated maps are never durable config.
def credential_free(value):
if isinstance(value, dict):
for key, item in value.items():
if (
key
in {
"credentials",
"api_key",
"access_token",
"refresh_token",
"inner_api_key",
"authorization",
}
and item
):
raise BuilderExecutionPolicyError("native_agent_config_credentials")
credential_free(item)
elif isinstance(value, list):
for item in value:
credential_free(item)
normalized = request.model_dump(mode="json", include={"composition", "session_snapshot"})
credential_free(normalized)
if len(json.dumps(normalized, ensure_ascii=False).encode()) > 1024 * 1024 - 4096:
raise BuilderExecutionPolicyError("native_agent_config_unavailable")
return _JSON.validate_python(normalized)
def claim_agent_create(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
expected_invocation_generation: int,
config: JsonValue,
definition_receipt: AgentDefinitionReceipt,
) -> AgentCreateClaim:
"""First local SQL insertion only; lost ACK and identical rows own no POST branch."""
with _agent_boundary():
prepared = self._agent_prepare(
ref=ref,
parent_node=parent_node,
writing=True,
expected_generation=expected_invocation_generation,
)
config = self._agent_executable_config(config, prepared.parent)
value = {
"schema": "agent-config-v2",
"parent_node": prepared.parent.model_dump(mode="json"),
"workflow_native_run_id": prepared.context.native_run_id,
"workspace": prepared.workspace.model_dump(mode="json"),
"resource_digest": prepared.resource_digest,
"config": config,
}
key = self._agent_key("agent-config", prepared.parent)
# Object crypto/readback is deliberately outside the writer, even if another
# caller wins while it is in progress. That caller's ACK cannot grant us POST.
receipt = self.prepare_object(
ref=ref,
tenant_id=prepared.context.tenant_id,
kind="result",
logical_key=key,
payload=value,
max_bytes=1024 * 1024,
)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(
session,
owned,
scope,
prepared,
writing=True,
definition_receipt=definition_receipt,
)
existing = session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.kind == "result",
NativePayloadObject.logical_key == key,
)
)
if existing is not None:
if (existing.payload_digest, existing.value_digest) != (
receipt.payload_digest,
receipt.value_digest,
):
raise BuilderExecutionPolicyError("native_payload_conflict")
return AgentCreateExisting(existing.id, canonical_digest(config))
selected = self.acknowledge_object(session, receipt)
return AgentCreateInserted(selected.id, canonical_digest(config))
def _agent_prepared_config(self, prepared: _AgentPreparation, config_ref: str | None) -> AgentPreparedConfig:
if prepared.recovery is not None:
config = prepared.recovery.config
if config_ref is not None and config_ref != config.config_ref:
raise BuilderExecutionPolicyError("native_agent_config_mismatch")
return config
selected = self._agent_object(
prepared,
logical_key=self._agent_key("agent-config", prepared.parent),
object_ref=config_ref,
)
assert selected is not None
return self._agent_validate_prepared_config(prepared, *selected)
def _agent_validate_prepared_config(
self, prepared: _AgentPreparation, receipt: ObjectReceipt, value: JsonValue
) -> AgentPreparedConfig:
"""Validate the already selected immutable object; its receipt stays in the SQL recheck."""
if (
not isinstance(value, dict)
or set(value)
!= {
"schema",
"parent_node",
"workflow_native_run_id",
"workspace",
"resource_digest",
"config",
}
or value
!= {
"schema": "agent-config-v2",
"parent_node": prepared.parent.model_dump(mode="json"),
"workflow_native_run_id": prepared.context.native_run_id,
"workspace": prepared.workspace.model_dump(mode="json"),
"resource_digest": prepared.resource_digest,
"config": value.get("config"),
}
):
raise BuilderExecutionPolicyError("native_agent_config_mismatch")
config = self._agent_executable_config(value["config"], prepared.parent)
if prepared.context.native_run_id is None:
raise BuilderExecutionPolicyError("native_run_mismatch")
return AgentPreparedConfig(
parent_node=prepared.parent,
workflow_native_run_id=prepared.context.native_run_id,
workspace=prepared.workspace,
config_ref=receipt.id,
config_digest=canonical_digest(config),
config=config,
)
def load_agent_settlement_identity(self, *, ref, parent_node):
"""Read original identity for result/outcome ACK only; grants no execution.
Static runtime Resources select this mode. Every current owner, config,
association and resource check remains; only expiry of new-call authority
does not discard evidence from an operation already dispatched.
"""
with _agent_boundary():
prepared = self._agent_prepare(ref=ref, parent_node=parent_node, writing=False, settlement=True)
config = self._agent_prepared_config(prepared, None)
selected = self._selected_agent_association_object(prepared)
assert selected is not None
association = AgentNodeAssociation.model_validate(selected[1])
self._agent_config_relation(prepared, association)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
return prepared.context, config, association
def load_prepared_config(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
config_ref: str | None,
for_projection: bool = False,
) -> AgentPreparedConfig:
"""Private current-config read with owner/resource recheck after decryption."""
with _agent_boundary():
prepared = self._agent_prepare(
ref=ref, parent_node=parent_node, writing=False, for_projection=for_projection
)
result = self._agent_prepared_config(prepared, config_ref)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
return result
def load_prepared_config_selection(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
config_ref: str | None,
for_projection: bool = False,
) -> tuple[AgentPreparedConfig, "SelectedRecovery | None"]:
"""Read immutable config and its actual selected proof before a SQL writer.
The existing config owner remains unchanged without recovery. With a
record, return the same decoded proof used to select/recheck this config;
all storage, access refresh and decryption finish before return. Callers
must pass the proof to their root resource recheck. Neither the tuple
nor a missing proof grants mutation or handoff authority.
"""
from services.dify_builder.native_testing.agent_recovery import recovery_record
owned = self.loader.load(ref=ref)
if recovery_record(owned.bindings) is None:
return self.load_prepared_config(
ref=ref, parent_node=parent_node, config_ref=config_ref, for_projection=for_projection
), None
with _agent_boundary():
prepared = self._agent_prepare(
ref=ref, parent_node=parent_node, writing=False, for_projection=for_projection
)
result = self._agent_prepared_config(prepared, config_ref)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, current, scope):
self._agent_recheck(session, current, scope, prepared, writing=False)
return result, prepared.recovery
def discover_agent_run(
self, *, ref: NativeTestContextRef, parent_node: AgentParentNodeRef
) -> AgentNodeAssociation | None:
"""Observed unbound namespace is unknown, never authority to create/retry."""
with _agent_boundary():
prepared = self._agent_prepare(ref=ref, parent_node=parent_node, writing=False)
# A binding discovery always proves its config, including an unbound reservation.
self._agent_prepared_config(prepared, None)
selected = self._selected_agent_association_object(prepared, required=False)
association = None if selected is None else AgentNodeAssociation.model_validate(selected[1])
if association is not None:
self._agent_config_relation(prepared, association)
self._agent_refresh_access(prepared)
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
if (
selected is None
and session.scalar(
owner_query(NativePayloadObject, ref, owned.tenant_id).where(
NativePayloadObject.kind == "result",
NativePayloadObject.logical_key == self._agent_key("agent-bind", prepared.parent),
)
)
is not None
):
raise BuilderExecutionPolicyError("native_agent_discovery_unknown")
return association
def store_agent_config(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
expected_invocation_generation: int,
config: JsonValue,
) -> str:
"""Internal immutable config bytes, not backend authorization or preparation proof."""
with _agent_boundary():
prepared = self._agent_prepare(
ref=ref, parent_node=parent_node, writing=True, expected_generation=expected_invocation_generation
)
value = {
"schema": "agent-config-v1",
"parent_node": prepared.parent.model_dump(mode="json"),
"workflow_native_run_id": prepared.context.native_run_id,
"workspace": prepared.workspace.model_dump(mode="json"),
"config": _JSON.validate_python(config),
}
return self._agent_ack(prepared, logical_key=self._agent_key("agent-config", prepared.parent), value=value)
def _selected_agent_association_object(self, prepared, *, required: bool = True):
"""Return the current association object; historical access is separate."""
recovery = prepared.recovery
return self._agent_object(
prepared,
logical_key=self._agent_key("agent-bind", prepared.parent)
if recovery is None
else "agent-recovery:" + recovery.record.attempt_id + ":association",
object_ref=None if recovery is None else recovery.relation.successor.association_object_ref,
required=required,
)
def _agent_binding(self, prepared: _AgentPreparation, association_object_ref: str) -> AgentNodeAssociation:
if prepared.recovery is not None:
if association_object_ref != prepared.recovery.relation.successor.association_object_ref:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
self._agent_config_relation(prepared, prepared.recovery.association)
return prepared.recovery.association
selected = self._agent_object(
prepared, logical_key=self._agent_key("agent-bind", prepared.parent), object_ref=association_object_ref
)
assert selected is not None
association = AgentNodeAssociation.model_validate(selected[1])
self._agent_config_relation(prepared, association)
return association
def _agent_config_relation(self, prepared: _AgentPreparation, association: AgentNodeAssociation) -> None:
if (
association.parent_node,
association.workflow_native_run_id,
association.workspace,
) != (
prepared.parent,
prepared.context.native_run_id,
prepared.workspace,
):
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
if prepared.recovery is not None:
config = self._agent_prepared_config(prepared, association.config_ref)
if association != prepared.recovery.association or config.config_digest != association.config_digest:
raise BuilderExecutionPolicyError("native_agent_config_mismatch")
return
selected = self._agent_object(
prepared,
logical_key=self._agent_key("agent-config", prepared.parent),
object_ref=association.config_ref,
)
assert selected is not None
value = selected[1]
if isinstance(value, dict) and value.get("schema") == "agent-config-v2":
selected_config = self._agent_validate_prepared_config(prepared, *selected)
if selected_config.config_digest != association.config_digest:
raise BuilderExecutionPolicyError("native_agent_config_mismatch")
return
if (
not isinstance(value, dict)
or set(value)
!= {
"schema",
"parent_node",
"workflow_native_run_id",
"workspace",
"config",
}
or value
!= {
"schema": "agent-config-v1",
"parent_node": prepared.parent.model_dump(mode="json"),
"workflow_native_run_id": prepared.context.native_run_id,
"workspace": prepared.workspace.model_dump(mode="json"),
"config": value.get("config"),
}
or canonical_digest(value["config"]) != association.config_digest
):
raise BuilderExecutionPolicyError("native_agent_config_mismatch")
def bind_agent_run(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
expected_invocation_generation: int,
association: AgentNodeAssociation,
) -> str:
"""Internal protected binding; only the actual later backend producer may supply its Run."""
with _agent_boundary():
association = AgentNodeAssociation.model_validate(association.model_dump(mode="json"))
prepared = self._agent_prepare(
ref=ref, parent_node=parent_node, writing=True, expected_generation=expected_invocation_generation
)
self._agent_config_relation(prepared, association)
return self._agent_ack(
prepared,
logical_key=self._agent_key("agent-bind", prepared.parent),
value=association.model_dump(mode="json"),
)
@staticmethod
def _agent_checkpoint_metadata(checkpoint: JsonValue, association: AgentNodeAssociation, generation: int) -> None:
# The actual backend adapter still owns full AgentCheckpoint/event validation.
if (
not isinstance(checkpoint, dict)
or (
checkpoint.get("run_id"),
checkpoint.get("ref"),
checkpoint.get("config_ref"),
checkpoint.get("config_digest"),
)
!= (
association.agent_run_id,
association.parent_node.ref.model_dump(mode="json"),
association.config_ref,
association.config_digest,
)
or TypeAdapter(Counter).validate_python(checkpoint.get("generation")) != generation
):
raise BuilderExecutionPolicyError("native_agent_checkpoint_mismatch")
def store_agent_wait(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
association_object_ref: str,
agent_generation: int,
checkpoint: JsonValue,
backend_wait_event_id: str,
) -> AgentWaitAssociation:
"""Store selected bytes/event reference, NEVER certify an actual Redis ACK.
The uninstalled later trusted backend producer must first read/validate
the actual checkpoint and event. No caller proof flag can replace it.
"""
with _agent_boundary():
generation = TypeAdapter(Counter).validate_python(agent_generation)
event_id = TypeAdapter(ID).validate_python(backend_wait_event_id)
prepared = self._agent_prepare(ref=ref, parent_node=parent_node, writing=True)
association = self._agent_binding(prepared, association_object_ref)
checkpoint = _JSON.validate_python(checkpoint)
self._agent_checkpoint_metadata(checkpoint, association, generation)
value = {
"schema": "agent-checkpoint-v1",
"parent_node": prepared.parent.model_dump(mode="json"),
"association_object_ref": association_object_ref,
"agent_generation": generation,
"backend_wait_event_id": event_id,
"checkpoint": checkpoint,
}
object_ref = self._agent_ack(
prepared, logical_key=self._agent_key("agent-cp", prepared.parent, generation), value=value
)
return AgentWaitAssociation(
parent_node=prepared.parent,
association_object_ref=association_object_ref,
agent_generation=generation,
checkpoint_object_ref=object_ref,
checkpoint_digest=canonical_digest(checkpoint),
backend_wait_event_id=event_id,
)
def _agent_wait_relation(
self, prepared: _AgentPreparation, association: AgentNodeAssociation, wait: AgentWaitAssociation
) -> JsonValue:
if wait.parent_node != prepared.parent:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
selected = self._agent_object(
prepared,
logical_key=self._agent_key("agent-cp", prepared.parent, wait.agent_generation),
object_ref=wait.checkpoint_object_ref,
)
assert selected is not None
value = selected[1]
if (
not isinstance(value, dict)
or set(value)
!= {
"schema",
"parent_node",
"association_object_ref",
"agent_generation",
"backend_wait_event_id",
"checkpoint",
}
or value
!= {
"schema": "agent-checkpoint-v1",
"parent_node": prepared.parent.model_dump(mode="json"),
"association_object_ref": wait.association_object_ref,
"agent_generation": wait.agent_generation,
"backend_wait_event_id": wait.backend_wait_event_id,
"checkpoint": value.get("checkpoint"),
}
or canonical_digest(value["checkpoint"]) != wait.checkpoint_digest
):
raise BuilderExecutionPolicyError("native_agent_checkpoint_mismatch")
self._agent_checkpoint_metadata(value["checkpoint"], association, wait.agent_generation)
return value["checkpoint"]
def load_agent_association(
self, *, ref: NativeTestContextRef, parent_node: AgentParentNodeRef, wait: AgentWaitAssociation | None = None
) -> AgentProtectedState:
"""Read owned material only; absence refuses and never implies resume/terminal proof."""
return self._read_agent_association(ref=ref, parent_node=parent_node, wait=wait, for_projection=False)
def read_agent_association_for_cancellation(
self, *, ref: NativeTestContextRef, parent_node: AgentParentNodeRef, wait: AgentWaitAssociation | None = None
) -> AgentProtectedState:
"""Locate the original child for active cancellation, never resume it."""
return self._read_agent_association(ref=ref, parent_node=parent_node, wait=wait, for_projection=True)
def _read_agent_association(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
wait: AgentWaitAssociation | None,
for_projection: bool,
) -> AgentProtectedState:
with _agent_boundary():
prepared = self._agent_prepare(
ref=ref, parent_node=parent_node, writing=False, for_projection=for_projection
)
if wait is None:
selected = self._selected_agent_association_object(prepared)
assert selected is not None
association_ref = selected[0].id
else:
wait = AgentWaitAssociation.model_validate(wait.model_dump(mode="json"))
association_ref = wait.association_object_ref
association = self._agent_binding(prepared, association_ref)
recovery_state = None
if (
wait is not None
and prepared.recovery is not None
and wait.checkpoint_object_ref == prepared.recovery.relation.successor_checkpoint.checkpoint_id
):
evidence = self.load_result(ref=ref, result_ref=prepared.recovery.record.evidence_object_ref)
if not isinstance(evidence, dict):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
recovery_state = self.load_recovery_pending_chain(
ref=ref,
original_checkpoint=CheckpointRef.model_validate(evidence["parent_checkpoint"]),
for_projection=for_projection,
)
if (
recovery_state is None
or recovery_state.graph != prepared.recovery
or recovery_state.proof.selection.wait != wait
):
raise BuilderExecutionPolicyError("native_agent_recovery_conflict")
checkpoint = recovery_state.graph.checkpoint.model_dump(mode="json")
else:
checkpoint = None if wait is None else self._agent_wait_relation(prepared, association, wait)
final = self._agent_object(
prepared, logical_key=self._agent_key("agent-final", prepared.parent), required=False
)
terminal_ref, terminal = None, None
if final is not None:
final_value = final[1]
if (
not isinstance(final_value, dict)
or set(final_value)
!= {
"schema",
"parent_node",
"association_object_ref",
"agent_generation",
"backend_terminal_event_id",
"terminal_result",
}
or final_value.get("schema") != "agent-final-v1"
or final_value.get("parent_node") != prepared.parent.model_dump(mode="json")
or final_value.get("association_object_ref") != association_ref
):
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
TypeAdapter(Counter).validate_python(final_value["agent_generation"])
TypeAdapter(ID).validate_python(final_value["backend_terminal_event_id"])
terminal_ref, terminal = final[0].id, final_value["terminal_result"]
with self.loader.locked(ref=ref) as (session, owned, scope):
self._agent_recheck(session, owned, scope, prepared, writing=False)
if recovery_state is not None:
self.recheck_recovery_pending_chain(
session=session,
owned=owned,
state=recovery_state,
original_checkpoint=recovery_state.proof.selection.original_checkpoint,
)
return AgentProtectedState(
association_object_ref=association_ref,
association=association,
wait=wait,
checkpoint=checkpoint,
terminal_result_ref=terminal_ref,
terminal_result=terminal,
)
def store_agent_final(
self,
*,
ref: NativeTestContextRef,
parent_node: AgentParentNodeRef,
association_object_ref: str,
agent_generation: int,
terminal_result: JsonValue,
backend_terminal_event_id: str,
) -> str:
"""Private terminal bytes reference, not verified terminal/physical cleanup proof.
The later actual backend producer must verify terminal state first. This
record never settles C02 or releases capacity and has no installed caller.
"""
with _agent_boundary():
generation = TypeAdapter(Counter).validate_python(agent_generation)
event_id = TypeAdapter(ID).validate_python(backend_terminal_event_id)
prepared = self._agent_prepare(ref=ref, parent_node=parent_node, writing=True)
self._agent_binding(prepared, association_object_ref)
value = {
"schema": "agent-final-v1",
"parent_node": prepared.parent.model_dump(mode="json"),
"association_object_ref": association_object_ref,
"agent_generation": generation,
"backend_terminal_event_id": event_id,
"terminal_result": _JSON.validate_python(terminal_result),
}
return self._agent_ack(prepared, logical_key=self._agent_key("agent-final", prepared.parent), value=value)
def _agent_checkpoint_preparations(
self, ref: NativeTestContextRef, payload: NativeCheckpointPayload, *, for_projection: bool = False
) -> list[_AgentPreparation]:
"""Resolve all selected private references outside the later SQL-only recheck.
This cannot certify backend ACK or authorize generic restoration. The
existing claimed-C03/current-run handoff remains the actual resume gate.
"""
preparations = []
for wait in payload.agent_associations:
prepared = self._agent_prepare(
ref=ref, parent_node=wait.parent_node, writing=False, for_projection=for_projection
)
if (
payload.native_run_id != prepared.context.native_run_id
or payload.generation > prepared.context.generation
):
raise BuilderExecutionPolicyError("native_generation_conflict")
if prepared.recovery is not None and wait == prepared.recovery.predecessor_wait:
if payload.generation != prepared.recovery.record.parent_generation:
raise BuilderExecutionPolicyError("native_generation_conflict")
else:
association = self._agent_binding(prepared, wait.association_object_ref)
self._agent_wait_relation(prepared, association, wait)
preparations.append(prepared)
return preparations
def _agent_single_parent(self, session, context: ResolvedNativeTestContext, parent: AgentParentNodeRef) -> None:
"""Temporary one-workspace limitation: never share it with a distinct occurrence."""
expected = self._agent_key("agent-config", parent)
keys = session.scalars(
owner_query(NativePayloadObject, context.ref, context.tenant_id)
.with_only_columns(NativePayloadObject.logical_key)
.where(
NativePayloadObject.kind == "result",
NativePayloadObject.logical_key.like("agent-config:%"),
NativePayloadObject.logical_key != expected,
)
.limit(1)
)
for key in keys:
if type(key) is not str:
raise BuilderExecutionPolicyError("native_agent_owner_mismatch")
if key.startswith("agent-config:") and key != expected:
raise BuilderExecutionPolicyError("native_agent_single_workspace_limit")