Integrations

AutoGen

An AutoGen FunctionTool for POST /v1/judge. The file does not construct a model client.

All integrations

"""AutoGen FunctionTool that POSTs /v1/judge to Screwtape.ai."""
import json
import os
import urllib.request
from autogen_core.tools import FunctionTool

API = "https://screwtape.ai"


def screwtape_post(path: str, payload: dict) -> dict:
    """POST JSON to Screwtape.ai. Requires SCREWTAPE_API_KEY."""
    key = os.environ["SCREWTAPE_API_KEY"]
    request = urllib.request.Request(
        API + path,
        data=json.dumps(payload).encode(),
        headers={
            "Authorization": "Bearer " + key,
            "Content-Type": "application/json",
        },
        method="POST",
    )
    with urllib.request.urlopen(request) as response:
        body = json.load(response)
        if not isinstance(body, dict):
            raise RuntimeError("expected a JSON object")
        return body


JUDGE = {
    "task": "State the crew size and launch site.",
    "agent_output": "The crew size was 4. The mission launched from Cape Canaveral.",
    "evidence": [
        "The mission launched on 2024-03-01 from Cape Canaveral.",
        "The crew size was 4."
    ],
    "criteria": [
        "crew size",
        "Cape Canaveral"
    ]
}

CONSENSUS = {
    "task": "State the crew size.",
    "answers": [
        {
            "agent_id": "alpha",
            "output": "The crew size was 4."
        },
        {
            "agent_id": "beta",
            "output": "The crew size was 4."
        },
        {
            "agent_id": "gamma",
            "output": "The crew size was 4."
        }
    ]
}

def screwtape_judge(task: str, agent_output: str) -> str:
    """Judge an agent output. Returns JSON with verdict, score, and reasons."""
    return json.dumps(screwtape_post("/v1/judge", {"task": task, "agent_output": agent_output}))


judge_tool = FunctionTool(screwtape_judge, description=screwtape_judge.__doc__ or "Judge an agent output.")

# Pass judge_tool to AssistantAgent(tools=[judge_tool]). This file does not construct a model client.


if __name__ == "__main__":
    print(screwtape_judge(JUDGE["task"], JUDGE["agent_output"]))
    print(json.dumps(screwtape_post("/v1/judge", JUDGE)))
    print(json.dumps(screwtape_post("/v1/consensus", CONSENSUS)))