-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathasync_scorer.py
More file actions
66 lines (50 loc) · 2.18 KB
/
Copy pathasync_scorer.py
File metadata and controls
66 lines (50 loc) · 2.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
"""Scorer resource class for asynchronous operations."""
from __future__ import annotations
from typing_extensions import Unpack, override
from ._types import (
BaseRequestOptions,
SDKScorerUpdateParams,
)
from .._client import AsyncRunloop
from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse
class AsyncScorer:
"""A custom scorer for evaluating scenario outputs (async).
Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs.
Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``.
Example:
>>> runloop = AsyncRunloopSDK()
>>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'")
"""
def __init__(self, client: AsyncRunloop, scorer_id: str) -> None:
"""Create an AsyncScorer instance.
:param client: AsyncRunloop client instance
:type client: AsyncRunloop
:param scorer_id: ID of the scorer
:type scorer_id: str
"""
self._client = client
self._id = scorer_id
@override
def __repr__(self) -> str:
return f"<AsyncScorer id={self._id!r}>"
@property
def id(self) -> str:
"""The scorer's unique identifier.
:return: Scorer ID
:rtype: str
"""
return self._id
async def get_info(self, **options: Unpack[BaseRequestOptions]) -> ScorerRetrieveResponse:
"""Fetch current scorer details from the API.
:param options: See :typeddict:`~runloop_api_client.sdk._types.BaseRequestOptions` for available options
:return: Current scorer details
:rtype: ScorerRetrieveResponse
"""
return await self._client.scenarios.scorers.retrieve(self._id, **options)
async def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateResponse:
"""Update the scorer's type or bash script.
:param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerUpdateParams` for available parameters
:return: Updated scorer details
:rtype: ScorerUpdateResponse
"""
return await self._client.scenarios.scorers.update(self._id, **params)