Integrations

OpenAI Agents SDK

An OpenAI Agents SDK tool for POST /v1/judge. __main__ calls the wrapped function and the consensus example.

All integrations

"""OpenAI Agents SDK tool that POSTs /v1/judge to Screwtape.ai."""
import json
import os
import urllib.request
from agents import Agent
from agents.decorators import tool

API = "https://screwtape.ai"


def screwtape_post(path: str, payload: dict) -> dict:
    """POST JSON to Screwtape.ai. Requires SCREWTAPE_API_KEY."""
    key = os.environ["SCREWTAPE_API_KEY"]
    request = urllib.request.Request(
        API + path,
        data=json.dumps(payload).encode(),
        headers={
            "Authorization": "Bearer " + key,
            "Content-Type": "application/json",
        },
        method="POST",
    )
    with urllib.request.urlopen(request) as response:
        body = json.load(response)
        if not isinstance(body, dict):
            raise RuntimeError("expected a JSON object")
        return body


JUDGE = {
    "task": "State the crew size and launch site.",
    "agent_output": "The crew size was 4. The mission launched from Cape Canaveral.",
    "evidence": [
        "The mission launched on 2024-03-01 from Cape Canaveral.",
        "The crew size was 4."
    ],
    "criteria": [
        "crew size",
        "Cape Canaveral"
    ]
}

CONSENSUS = {
    "task": "State the crew size.",
    "answers": [
        {
            "agent_id": "alpha",
            "output": "The crew size was 4."
        },
        {
            "agent_id": "beta",
            "output": "The crew size was 4."
        },
        {
            "agent_id": "gamma",
            "output": "The crew size was 4."
        }
    ]
}

@tool
def screwtape_judge(task: str, agent_output: str) -> str:
    """Judge an agent output.

    Args:
        task: The task the other agent was given.
        agent_output: The output to judge.
    """
    return json.dumps(screwtape_post("/v1/judge", {"task": task, "agent_output": agent_output}))


def reviewer() -> Agent:
    """Build an Agents SDK agent that can call the judge tool. Runner.run needs your OpenAI key."""
    return Agent(
        name="Reviewer",
        instructions="Call screwtape_judge before you accept an answer. Report the verdict, score, and reasons.",
        tools=[screwtape_judge],
    )


if __name__ == "__main__":
    print(screwtape_judge.__wrapped__(JUDGE["task"], JUDGE["agent_output"]))
    print(json.dumps(screwtape_post("/v1/consensus", CONSENSUS)))