score.md
Score
A Score is a System One question type for rating content against ordered, descriptive levels. The answer includes a score, a probability for each level, and confidence.
export function TypesafeExample({example, display, title}) {
const keyStrUriSafe = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-$";
function compressToEncodedURIComponent(input) {
if (input == null) return "";
return _compress(input, 6, function (a) {
return keyStrUriSafe.charAt(a);
});
}
function _compress(uncompressed, bitsPerChar, getCharFromInt) {
if (uncompressed == null) return "";
var i, value, context_dictionary = {}, context_dictionaryToCreate = {}, context_c = "", context_wc = "", context_w = "", context_enlargeIn = 2, context_dictSize = 3, context_numBits = 2, context_data = [], context_data_val = 0, context_data_position = 0, ii;
for (ii = 0; ii < uncompressed.length; ii += 1) {
context_c = uncompressed.charAt(ii);
if (!Object.prototype.hasOwnProperty.call(context_dictionary, context_c)) {
context_dictionary[context_c] = context_dictSize++;
context_dictionaryToCreate[context_c] = true;
}
context_wc = context_w + context_c;
if (Object.prototype.hasOwnProperty.call(context_dictionary, context_wc)) {
context_w = context_wc;
} else {
if (Object.prototype.hasOwnProperty.call(context_dictionaryToCreate, context_w)) {
if (context_w.charCodeAt(0) < 256) {
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
}
value = context_w.charCodeAt(0);
for (i = 0; i < 8; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
} else {
value = 1;
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1 | value;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = 0;
}
value = context_w.charCodeAt(0);
for (i = 0; i < 16; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = Math.pow(2, context_numBits);
context_numBits++;
}
delete context_dictionaryToCreate[context_w];
} else {
value = context_dictionary[context_w];
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = Math.pow(2, context_numBits);
context_numBits++;
}
context_dictionary[context_wc] = context_dictSize++;
context_w = String(context_c);
}
}
if (context_w !== "") {
if (Object.prototype.hasOwnProperty.call(context_dictionaryToCreate, context_w)) {
if (context_w.charCodeAt(0) < 256) {
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
}
value = context_w.charCodeAt(0);
for (i = 0; i < 8; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
} else {
value = 1;
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1 | value;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = 0;
}
value = context_w.charCodeAt(0);
for (i = 0; i < 16; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = Math.pow(2, context_numBits);
context_numBits++;
}
delete context_dictionaryToCreate[context_w];
} else {
value = context_dictionary[context_w];
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
}
context_enlargeIn--;
if (context_enlargeIn == 0) {
context_enlargeIn = Math.pow(2, context_numBits);
context_numBits++;
}
}
value = 2;
for (i = 0; i < context_numBits; i++) {
context_data_val = context_data_val << 1 | value & 1;
if (context_data_position == bitsPerChar - 1) {
context_data_position = 0;
context_data.push(getCharFromInt(context_data_val));
context_data_val = 0;
} else {
context_data_position++;
}
value = value >> 1;
}
while (true) {
context_data_val = context_data_val << 1;
if (context_data_position == bitsPerChar - 1) {
context_data.push(getCharFromInt(context_data_val));
break;
} else context_data_position++;
}
return context_data.join("");
}
function buildHref(ex) {
const documentText = ex.state === undefined ? "" : typeof ex.state === "string" ? ex.state : JSON.stringify(ex.state, null, 2);
return "https://console.typesafe.ai/decode#share/" + compressToEncodedURIComponent(JSON.stringify({
apiVersion: "v1",
documentText,
promptsText: JSON.stringify(ex.questions, null, 2),
selectedModels: ex.selectedModels
}));
}
const displayedExample = display === "questions" ? example.questions : example.state === undefined ? {
questions: example.questions
} : {
state: example.state,
questions: example.questions
};
const code = JSON.stringify(displayedExample, null, 2);
const href = buildHref(example);
return <div style={{
margin: "1.25rem 0"
}}>
<CodeBlock language="json" filename={title ?? "request"}>
{code}
</CodeBlock>
<div className="pb-8">
<a href={href} target="_blank" rel="noreferrer" className="text-primary">
Try it in the Playground →
</a>
</div>
</div>;
}
Use a Score when the answer is a position on a spectrum you can describe in steps. For example, how severe a bug is, how happy a customer is, or how much Python experience a candidate has. If the answer is one of a fixed set of options with no order between them, use a Choice. If it's a yes or no, use a Noul. Choose a question type compares all three.
A Score answer is a position along your levels in score, which can fall between two levels. The model also returns a probability for every level in probabilities, and a confidence value for the answer.
Example Score questions:
"How severe is the bug being reported?"
→ 0: Cosmetic; no impact to functionality
→ 1: Broken or degraded feature, but workaround exists
→ 2: Blocking issue; no workaround exists
"How formal is this outfit based on the description"
→ 0: gym clothes
→ 1: casual
→ 2: business casual
→ 3: formal
→ 4: black tie
"How relevant is this candidate's experience to the job posting"
→ 0: completely unrelated
→ 1: adjacent field
→ 2: some direct experience
→ 3: deep, direct experience
The numbers in front of each step are positions, explained under Levels.
Request structure
The POST request body to the TypeSafe API has the same three top-level fields as any other question type: state, which is the content to evaluate; model; and questions. Each Score question has the following fields:
type: Always"score".instructions: The question the model answers. What it's rating.criteria: An ordered array of level descriptions, from the low end of the scale to the high end. Needs at least two levels and takes up to 10.
Below is a request where the state is a bug report and the question is how severe the bug is:
<TypesafeExample
display="request"
example={{
state: 'The export button crashes the settings page in Safari. It works in Chrome, but a few of our customers only use Safari.',
selectedModels: ['jev-latest'],
questions: {
bug_severity: {
type: 'score',
instructions: 'How severe is the reported issue?',
criteria: [
'Cosmetic; no impact to functionality',
'Broken or degraded feature, but workaround exists',
'Blocking issue; no workaround exists',
],
},
},
}}
/>
You choose the question id, bug_severity in this case. This id is not sent to the model. The answer is returned under the same id.
Levels
Each entry in criteria is a level: one point on the spectrum of possible answers, described in words. A level's number is its position in the criteria array, starting at 0, so the three entries above are levels 0, 1 and 2. The order of the array is the numbering.
The model gets the descriptions and nothing else, and each level is judged on its own against the state.
The score in the response is a position on the levels spectrum. For a three-level scale it runs from 0 to 2, and it can land between two levels.
Our client SDKs provide typed questions. In Python, the same question is a Score:
from typesafe_sdk import Score, TypeSafeClient
with TypeSafeClient() as client:
response = client.system_one(
state="The export button crashes the settings page in Safari. It works in Chrome, but a few of our customers only use Safari.",
questions={
"bug_severity": Score(
instructions="How severe is the reported issue?",
criteria=[
"Cosmetic; no impact to functionality",
"Broken or degraded feature, but workaround exists",
"Blocking issue; no workaround exists",
],
),
},
)
print(response.answers["bug_severity"].score)
Use the system_one method or the https://api.typesafe.ai/v1/systemone endpoint to call a System One model. The model field selects which model handles the request. How to build with TypeSafe covers where in your code to call it.
Use one of our client SDKs or call the TypeSafe API directly. If a coding agent is writing the integration for you, install the TypeSafe agent skill first so it knows the request and response shapes.