{"id":499109,"date":"2026-10-18T13:56:56","date_gmt":"2026-10-18T10:56:56","guid":{"rendered":"https:\/\/savepearlharbor.com\/?p=499109"},"modified":"2026-10-18T13:56:56","modified_gmt":"2026-10-18T10:56:56","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=499109","title":{"rendered":"\u041a\u0430\u043a \u0443\u0441\u0442\u0440\u043e\u0435\u043d\u0430 decision-\u043c\u043e\u0434\u0435\u043b\u044c Clef: \u0440\u0430\u0437\u0431\u043e\u0440 \u043c\u043e\u0434\u0435\u043b\u0438 \u043f\u0440\u0438\u043d\u044f\u0442\u0438\u044f \u0440\u0435\u0448\u0435\u043d\u0438\u0439"},"content":{"rendered":"
\n

\u041f\u0440\u0438\u0432\u0435\u0442, \u0425\u0430\u0431\u0440! \u041d\u0430 \u0444\u043e\u043d\u0435 \u043f\u043e\u043f\u0443\u043b\u044f\u0440\u043d\u043e\u0441\u0442\u0438 decision-\u043c\u043e\u0434\u0435\u043b\u0435\u0439 \u043c\u043d\u0435 \u0437\u0430\u0445\u043e\u0442\u0435\u043b\u043e\u0441\u044c \u043f\u043e\u0437\u043d\u0430\u043a\u043e\u043c\u0438\u0442\u044c\u0441\u044f \u0441 \u0438\u0445 \u043c\u0435\u0445\u0430\u043d\u0438\u0437\u043c\u043e\u043c \u0440\u0430\u0431\u043e\u0442\u044b. \u041e\u0431\u044a\u0435\u043a\u0442\u043e\u043c \u0438\u0441\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u043d\u0438\u044f \u044f \u0432\u044b\u0431\u0440\u0430\u043b \u043c\u043e\u0434\u0435\u043b\u044c Clef \u043e\u0442 Cloudflare \u043f\u0440\u043e\u0441\u0442\u043e \u043f\u043e\u0442\u043e\u043c\u0443, \u0447\u0442\u043e \u043e\u043d\u0430 \u0431\u043e\u043b\u044c\u0448\u0435 \u0432\u0441\u0435\u0433\u043e \u043d\u0430 \u0441\u043b\u0443\u0445\u0443.<\/p>\n

\u0421\u0442\u0430\u0442\u044c\u044f \u043d\u043e\u0441\u0438\u0442 \u043f\u043e\u0437\u043d\u0430\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u0445\u0430\u0440\u0430\u043a\u0442\u0435\u0440 \u0438 \u0431\u0443\u0434\u0435\u0442 \u0438\u043d\u0442\u0435\u0440\u0435\u0441\u043d\u0430 \u0442\u0435\u043c, \u043a\u0442\u043e, \u043a\u0430\u043a \u0438 \u044f, \u0437\u0430\u0445\u043e\u0447\u0435\u0442 \u0440\u0430\u0437\u043e\u0431\u0440\u0430\u0442\u044c\u0441\u044f \u0432 \u0440\u0430\u0431\u043e\u0442\u0435 \u043c\u043e\u0434\u0435\u043b\u0438. \u0424\u0440\u0430\u0433\u043c\u0435\u043d\u0442\u044b \u043a\u043e\u0434\u0430 \u0438 \u043a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u0438 \u0432\u0437\u044f\u0442\u044b \u0438\u0437 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u044f<\/a> \u0430\u0432\u0442\u043e\u0440\u043e\u0432.<\/p>\n

\u041f\u0440\u0438\u0441\u0442\u0443\u043f\u0438\u043c!<\/p>\n

\u041e\u0431\u0449\u0430\u044f \u0441\u0445\u0435\u043c\u0430 \u0438 \u043e\u0431\u043e\u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f<\/h3>\n
<\/figure>\n
\n
\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
\n

\u0421\u0438\u043c\u0432\u043e\u043b<\/p>\n<\/th>\n

\n

\u0417\u043d\u0430\u0447\u0435\u043d\u0438\u0435<\/p>\n<\/th>\n<\/tr>\n

\n

\"N\"<\/p>\n<\/td>\n

\n

\u0447\u0438\u0441\u043b\u043e \u0442\u043e\u043a\u0435\u043d\u043e\u0432 \u0432 \u043f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438<\/p>\n<\/td>\n<\/tr>\n

\n

\"d\"<\/p>\n<\/td>\n

\n

\u0440\u0430\u0437\u043c\u0435\u0440 \u0441\u043a\u0440\u044b\u0442\u043e\u0433\u043e \u0441\u043e\u0441\u0442\u043e\u044f\u043d\u0438\u044f Qwen<\/p>\n<\/td>\n<\/tr>\n

\n

\"w\"<\/p>\n<\/td>\n

\n

\u0448\u0438\u0440\u0438\u043d\u0430 \u0433\u043e\u043b\u043e\u0432\u044b<\/p>\n<\/td>\n<\/tr>\n

\n

\"K\"<\/p>\n<\/td>\n

\n

\u0447\u0438\u0441\u043b\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u043e\u0432<\/p>\n<\/td>\n<\/tr>\n

\n

\"n_k\"<\/p>\n<\/td>\n

\n

\u0447\u0438\u0441\u043b\u043e \u043e\u043f\u0446\u0438\u0439 \u0443 \u0432\u043e\u043f\u0440\u043e\u0441\u0430 \"k\"<\/p>\n<\/td>\n<\/tr>\n

\n

\"M=\\sum_k<\/p>\n<\/td>\n

\n

\u0432\u0441\u0435\u0433\u043e \u043e\u043f\u0446\u0438\u0439<\/p>\n<\/td>\n<\/tr>\n

\n

\"S_k=[s_k,e_k)\"<\/p>\n<\/td>\n

\n

\u0441\u043f\u0430\u043d \u0438\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u0438 \u0432\u043e\u043f\u0440\u043e\u0441\u0430 \"k\"<\/p>\n<\/td>\n<\/tr>\n

\n

\"O_{kj}=[a_{kj},b_{kj})\"<\/p>\n<\/td>\n

\n

\u0441\u043f\u0430\u043d \u043e\u043f\u0446\u0438\u0438 \"j\" \u0432\u043e\u043f\u0440\u043e\u0441\u0430 \"k\"<\/p>\n<\/td>\n<\/tr>\n

\n

\"\\tilde<\/p>\n<\/td>\n

\n

\u0432\u044b\u0445\u043e\u0434 Qwen \u043f\u043e\u0441\u043b\u0435 LayerNorm \u0433\u043e\u043b\u043e\u0432\u044b, \"[N\\times<\/p>\n<\/td>\n<\/tr>\n

\n

\"E\\in\\mathbb{R}^{V\\times<\/p>\n<\/td>\n

\n

\u043c\u0430\u0442\u0440\u0438\u0446\u0430 lm_head<\/code> Qwen<\/p>\n<\/td>\n<\/tr>\n

\n

\"x_t\"<\/p>\n<\/td>\n

\n

\u043d\u043e\u043c\u0435\u0440 \u0442\u043e\u043a\u0435\u043d\u0430 \u0432 \u043f\u043e\u0437\u0438\u0446\u0438\u0438 \"t\" (input_ids[t]<\/code>)<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n

\u0412\u0445\u043e\u0434: \u0437\u0430\u043f\u0440\u043e\u0441 API<\/h3>\n

\u0417\u0430\u043f\u0440\u043e\u0441 \u0441\u043e\u0441\u0442\u043e\u0438\u0442 \u0438\u0437 state<\/strong> (\u043e \u0447\u0451\u043c \u0440\u0435\u0448\u0430\u0435\u043c) \u0438 questions<\/strong> (\u0447\u0442\u043e \u043d\u0443\u0436\u043d\u043e \u0440\u0435\u0448\u0438\u0442\u044c). \u041f\u0440\u0438\u043c\u0435\u0440 \u043e\u0442 Cloudflare:<\/p>\n

{  \"model\": \"clef\",  \"state\": \"Checkout has been failing for every customer for the last hour.\",  \"questions\": {    \"urgent\": { \"type\": \"noul\", \"instructions\": \"Is this support request urgent?\" },    \"team\": {      \"type\": \"choice\",      \"instructions\": \"Which team should handle this request?\",      \"criteria\": {        \"billing\": \"Payments, invoices, and refunds\",        \"technical\": \"Outages, errors, and configuration\",        \"sales\": \"Plans and upgrades\"      }    },    \"severity\": {      \"type\": \"score\",      \"instructions\": \"How severe is the customer impact?\",      \"criteria\": [\"No impact\", \"Minor\", \"Major\", \"Critical\"]    }  }}<\/code>
<\/span><\/a><\/div><\/pre>\n
\u0422\u0438\u043f\u044b \u0432\u043e\u043f\u0440\u043e\u0441\u043e\u0432<\/h4>\n
\n
\n\n\n\n\n\n\n
\n
\u0422\u0438\u043f<\/p>\n<\/th>\n
\n
\u0421\u043c\u044b\u0441\u043b<\/p>\n<\/th>\n
\n
\u041e\u0442\u043a\u0443\u0434\u0430 \u0431\u0435\u0440\u0443\u0442\u0441\u044f \u043e\u043f\u0446\u0438\u0438<\/p>\n<\/th>\n
\n
\u041e\u043f\u0446\u0438\u0439 \u0432 \u043f\u0440\u0438\u043c\u0435\u0440\u0435<\/p>\n<\/th>\n<\/tr>\n
\n
noul<\/code><\/p>\n<\/td>\n
\n
\u0434\u0430 \/ \u043d\u0435\u0442<\/p>\n<\/td>\n
\n
\u0441\u043e\u0437\u0434\u0430\u044e\u0442\u0441\u044f \u0430\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0447\u0435\u0441\u043a\u0438: true<\/code>, false<\/code><\/p>\n<\/td>\n
\n
2<\/p>\n<\/td>\n<\/tr>\n
\n
choice<\/code><\/p>\n<\/td>\n
\n
\u043e\u0434\u0438\u043d \u0432\u0430\u0440\u0438\u0430\u043d\u0442 \u0438\u0437 \u0441\u043f\u0438\u0441\u043a\u0430<\/p>\n<\/td>\n
\n
\u043a\u043b\u044e\u0447\u0438 criteria<\/code>, \u043e\u0442\u0441\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0435 \u043f\u043e \u0430\u043b\u0444\u0430\u0432\u0438\u0442\u0443<\/p>\n<\/td>\n
\n
3<\/p>\n<\/td>\n<\/tr>\n
\n
score<\/code><\/p>\n<\/td>\n
\n
\u0443\u0440\u043e\u0432\u0435\u043d\u044c \u043d\u0430 \u0448\u043a\u0430\u043b\u0435<\/p>\n<\/td>\n
\n
\u0438\u043d\u0434\u0435\u043a\u0441\u044b \u0441\u043f\u0438\u0441\u043a\u0430 criteria<\/code>: \"0\"<\/code>, \"1\"<\/code>, \u2026<\/p>\n<\/td>\n
\n
4<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n
\u041a\u0430\u043a \u0432\u043e\u043f\u0440\u043e\u0441 \u043f\u0440\u0435\u0432\u0440\u0430\u0449\u0430\u0435\u0442\u0441\u044f \u0432 \u0441\u043f\u0438\u0441\u043e\u043a \u043e\u043f\u0446\u0438\u0439<\/h4>\n
\u041a\u043e\u0434 \u0438\u0437 joint_schema_model.py<\/code>:<\/p>\n
QUESTION_TYPES = {\"noul\": 0, \"choice\": 1, \"score\": 2}def question_options(question: dict[str, Any]) -> list[tuple[str, Any]]:    question_type = str(question[\"type\"])    if question_type == \"noul\":        criteria = {            \"true\": \"The proposition is true or the answer is yes.\",            \"false\": \"The proposition is false or the answer is no.\",        }        criteria.update(question.get(\"criteria\") or {})        return [(key, criteria[key]) for key in (\"true\", \"false\")]    if question_type == \"choice\":        return sorted((str(key), value) for key, value in question[\"criteria\"].items())    return [(str(index), value) for index, value in enumerate(question[\"criteria\"])]<\/code>
<\/span><\/a><\/div><\/pre>\n
\u0414\u043b\u044f team<\/code> \u043f\u043e\u043b\u0443\u0447\u0430\u0435\u0442\u0441\u044f \u043e\u0442\u0441\u043e\u0440\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0439 \u043f\u043e ID \u0441\u043f\u0438\u0441\u043e\u043a:<\/p>\n
[(\"billing\",   \"Payments, invoices, and refunds\"), (\"sales\",     \"Plans and upgrades\"), (\"technical\", \"Outages, errors, and configuration\")]<\/code>
<\/span><\/a><\/div><\/pre>\n
\u042d\u0442\u0430\u043f 1. encode_record: \u0437\u0430\u043f\u0440\u043e\u0441 \u0441\u0442\u0430\u043d\u043e\u0432\u0438\u0442\u0441\u044f \u0442\u0435\u043a\u0441\u0442\u043e\u043c<\/h3>\n
<\/figure>\n
encode_record<\/code> \u0441\u043a\u043b\u0435\u0438\u0432\u0430\u0435\u0442 \u0432\u0441\u0435 \u043f\u043e\u043b\u044f JSON \u0432 \u043e\u0434\u0438\u043d \u043f\u0440\u043e\u043c\u043f\u0442 \u0438 \u043f\u043e\u043f\u0443\u0442\u043d\u043e \u0437\u0430\u043f\u043e\u043c\u0438\u043d\u0430\u0435\u0442, \u0433\u0434\u0435 \u0432 \u043d\u0451\u043c \u043b\u0435\u0436\u0438\u0442 \u043a\u0430\u0436\u0434\u044b\u0439 \u0432\u043e\u043f\u0440\u043e\u0441 \u0438 \u043a\u0430\u0436\u0434\u0430\u044f \u043e\u043f\u0446\u0438\u044f.<\/p>\n
\u042d\u0442\u043e \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0432 \u0442\u0440\u0438 \u0448\u0430\u0433\u0430.<\/p>\n
\u0428\u0430\u0433 1.<\/strong> question_options<\/code> \u043f\u0440\u0435\u0432\u0440\u0430\u0449\u0430\u0435\u0442 criteria<\/code> \u0432 \u0441\u043f\u0438\u0441\u043e\u043a \u043f\u0430\u0440 (\u0440\u0430\u0437\u0434\u0435\u043b 2).<\/p>\n
\u0428\u0430\u0433 2.<\/strong> \u041a\u0430\u0436\u0434\u0430\u044f \u043f\u0430\u0440\u0430 \u0437\u0430\u0432\u043e\u0440\u0430\u0447\u0438\u0432\u0430\u0435\u0442\u0441\u044f \u0432 \u043c\u0430\u043b\u0435\u043d\u044c\u043a\u0438\u0439 \u0441\u043b\u043e\u0432\u0430\u0440\u044c:<\/p>\n
semantics = {\"option_id\": option_id}if description is not None:    semantics[\"description\"] = description<\/code>
<\/span><\/a><\/div><\/pre>\n
\u0428\u0430\u0433 3.<\/strong> \u0421\u043b\u043e\u0432\u0430\u0440\u044c \u043f\u0440\u0435\u0432\u0440\u0430\u0449\u0430\u0435\u0442\u0441\u044f \u0432 \u0441\u0442\u0440\u043e\u043a\u0443 \u0444\u0443\u043d\u043a\u0446\u0438\u0435\u0439 render<\/code>:<\/p>\n
def render(value: Any) -> str:    if isinstance(value, str):        return value    return json.dumps(        value,        ensure_ascii=False,        separators=(\",\", \":\"),        sort_keys=True,    )<\/code>
<\/span><\/a><\/div><\/pre>\n
\u041f\u043e\u043b\u0443\u0447\u0430\u0435\u0442\u0441\u044f:<\/p>\n
{\"description\":\"Payments, invoices, and refunds\",\"option_id\":\"billing\"}<\/code>
<\/span><\/a><\/div><\/pre>\n
\u0424\u0438\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u044b\u0439 \u043f\u043e\u0440\u044f\u0434\u043e\u043a \u043a\u043b\u044e\u0447\u0435\u0439 \u0438 \u043e\u0442\u0441\u0443\u0442\u0441\u0442\u0432\u0438\u0435 \u043f\u0440\u043e\u0431\u0435\u043b\u043e\u0432 \u043d\u0443\u0436\u043d\u044b, \u0447\u0442\u043e\u0431\u044b \u043e\u0434\u043d\u0430 \u0438 \u0442\u0430 \u0436\u0435 \u043e\u043f\u0446\u0438\u044f \u0432\u0441\u0435\u0433\u0434\u0430 \u0434\u0430\u0432\u0430\u043b\u0430 \u043e\u0434\u043d\u0438 \u0438 \u0442\u0435 \u0436\u0435 \u0442\u043e\u043a\u0435\u043d\u044b.<\/p>\n
\u041f\u0440\u043e\u043c\u043f\u0442 \u0446\u0435\u043b\u0438\u043a\u043e\u043c ([\u043f\u0440\u0435\u0444\u0438\u043a\u0441], [state] \u0438 \u0442.\u0434 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0434\u043b\u044f \u043d\u0430\u0433\u043b\u044f\u0434\u043d\u043e\u0441\u0442\u0438):<\/strong><\/p>\n
[\u043f\u0440\u0435\u0444\u0438\u043a\u0441]  <|im_start|>system           Read the complete state and schema. Decide every field jointly. ...<|im_end|>           <|im_start|>user           STATE:[state]    Checkout has been failing for every customer for the last hour.[\u0441\u0445\u0435\u043c\u0430]           SCHEMA FIELDS:           FIELD 1           ID: urgent           TYPE: noul           INSTRUCTION: Is this support request urgent?                    \u2190 question_span           ALLOWED OPTIONS:           OPTION 1: {\"description\":\"The proposition is true or the answer is yes.\",\"option_id\":\"true\"}   \u2190 option_span           OPTION 2: {\"description\":\"The proposition is false or the answer is no.\",\"option_id\":\"false\"} \u2190 option_span           END FIELD           FIELD 2           ID: team           TYPE: choice           INSTRUCTION: Which team should handle this request?             \u2190 question_span           ALLOWED OPTIONS:           OPTION 1: {\"description\":\"Payments, invoices, and refunds\",\"option_id\":\"billing\"}           OPTION 2: {\"description\":\"Plans and upgrades\",\"option_id\":\"sales\"}           OPTION 3: {\"description\":\"Outages, errors, and configuration\",\"option_id\":\"technical\"}           END FIELD           FIELD 3           ID: severity           TYPE: score           INSTRUCTION: How severe is the customer impact?                 \u2190 question_span           ALLOWED OPTIONS:           OPTION 1: {\"description\":\"No impact\",\"option_id\":\"0\"}           OPTION 2: {\"description\":\"Minor\",\"option_id\":\"1\"}           OPTION 3: {\"description\":\"Major\",\"option_id\":\"2\"}           OPTION 4: {\"description\":\"Critical\",\"option_id\":\"3\"}           END FIELD[\u0441\u0443\u0444\u0444\u0438\u043a\u0441]  <|im_end|>           <|im_start|>assistant           <think>           <\/think>           JOINT SCHEMA DECISIONS:<\/code>
<\/span><\/a><\/div><\/pre>\n
\n\u041a\u043e\u0434<\/summary>\n
\n
\u041f\u043e\u0441\u043b\u0435 \u0442\u043e\u043a\u0435\u043d\u0438\u0437\u0430\u0446\u0438\u0438 \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c \u0437\u0430\u043f\u043e\u043c\u0438\u043d\u0430\u0435\u0442 \u043f\u043e\u0437\u0438\u0446\u0438\u0438 \u0432\u00a0\u043f\u0440\u043e\u043c\u043f\u0442\u0435 \u0432\u043e\u043f\u0440\u043e\u0441\u043e\u0432 \u0438 \u043e\u043f\u0446\u0438\u0439:<\/p>\n
def encode_record(    tokenizer: Any,    record: dict[str, Any],    max_length: int = 16384,    max_state_tokens: int | None = None,    processor: Any | None = None,) -> EncodedRecord:    schema_ids = _tokens(tokenizer, \"\\n\\nSCHEMA FIELDS:\\n\")    questions: list[EncodedQuestion] = []    for question_index, (question_id, question) in enumerate(record[\"questions\"].items()):        schema_ids.extend(            _tokens(                tokenizer,                f\"\\nFIELD {question_index + 1}\\nID: {question_id}\\nTYPE: {question['type']}\\nINSTRUCTION: \",            )        )        question_start = len(schema_ids)        instructions = question.get(\"instructions\")        if instructions is None or instructions == \"\":            instructions = str(question_id)        schema_ids.extend(_tokens(tokenizer, render(instructions)))        question_end = len(schema_ids)        schema_ids.extend(_tokens(tokenizer, \"\\nALLOWED OPTIONS:\\n\"))        option_spans: list[tuple[int, int]] = []        option_ids: list[str] = []        for option_index, (option_id, description) in enumerate(question_options(question)):            schema_ids.extend(_tokens(tokenizer, f\"OPTION {option_index + 1}: \"))            option_start = len(schema_ids)            semantics = {\"option_id\": option_id}            if description is not None:                semantics[\"description\"] = description            schema_ids.extend(_tokens(tokenizer, render(semantics)))            option_spans.append((option_start, len(schema_ids)))            option_ids.append(option_id)            schema_ids.extend(_tokens(tokenizer, \"\\n\"))        schema_ids.extend(_tokens(tokenizer, \"END FIELD\\n\"))<\/code>
<\/span><\/a><\/div><\/pre>\n
\u041a\u0430\u0436\u0434\u044b\u0439 \u043a\u0443\u0441\u043e\u043a \u0442\u043e\u043a\u0435\u043d\u0438\u0437\u0438\u0440\u0443\u0435\u0442\u0441\u044f \u043e\u0442\u0434\u0435\u043b\u044c\u043d\u043e \u0434\u043b\u044f \u0442\u043e\u0433\u043e, \u0447\u0442\u043e\u0431\u044b \u043d\u0435 \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u043b\u043e \u0441\u043a\u043b\u0435\u0438\u0432\u0430\u043d\u0438\u0435 (\u0435\u0441\u043b\u0438 \u0442\u043e\u043a\u0435\u043d\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0432\u0435\u0441\u044c \u043f\u0440\u043e\u043c\u043f\u0442 \u0446\u0435\u043b\u0438\u043a\u043e\u043c, BPE \u043c\u043e\u0436\u0435\u0442 \u0441\u043a\u043b\u0435\u0438\u0442\u044c \u043a\u043e\u043d\u0435\u0446 \"OPTION 1: \"<\/code> \u0441 \u043d\u0430\u0447\u0430\u043b\u043e\u043c JSON \u0432 \u043e10. \u0434\u0438\u043d \u0442\u043e\u043a\u0435\u043d)<\/p>\n
\u041f\u043e\u0441\u043b\u0435 \u044d\u0442\u043e\u0433\u043e \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0441\u0434\u0432\u0438\u0433 \u0432\u0441\u0435\u0439 \u0441\u0445\u0435\u043c\u044b \u043d\u0430 \u0434\u043b\u0438\u043d\u0443 \u043f\u0440\u0435\u0444\u0438\u043a\u0441\u0430 \u0438 state:<\/p>\n
    schema_offset = len(prefix_ids) + len(state_ids)    shifted_questions = tuple(        EncodedQuestion(            question_id=question.question_id,            question_type=question.question_type,            question_span=(                question.question_span[0] + schema_offset,                question.question_span[1] + schema_offset,            ),            option_spans=tuple(                (start + schema_offset, end + schema_offset)                for start, end in question.option_spans            ),            option_ids=question.option_ids,        )        for question in questions    )    input_ids = tuple(prefix_ids + state_ids + schema_ids + suffix_ids)<\/code>
<\/span><\/a><\/div><\/pre>\n<\/div>\n<\/details>\n
\u0412\u044b\u0445\u043e\u0434 \u044d\u0442\u0430\u043f\u0430<\/h4>\n
\n
\n\n\n\n\n\n\n\n
\n
\u0412\u044b\u0445\u043e\u0434<\/p>\n<\/th>\n
\n
\u0420\u0430\u0437\u043c\u0435\u0440<\/p>\n<\/th>\n<\/tr>\n
\n
input_ids<\/code><\/p>\n<\/td>\n
\n
\"N\"<\/p>\n<\/td>\n<\/tr>\n
\n
question_span<\/code><\/p>\n<\/td>\n
\n
\"K \u043f\u0430\u0440\u044b \u0447\u0438\u0441\u0435\u043b<\/p>\n<\/td>\n<\/tr>\n
\n
option_spans<\/code><\/p>\n<\/td>\n
\n
\"M \u043f\u0430\u0440 \u0447\u0438\u0441\u0435\u043b, \u0441\u0433\u0440\u0443\u043f\u043f\u0438\u0440\u043e\u0432\u0430\u043d\u044b \u043f\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430\u043c<\/p>\n<\/td>\n<\/tr>\n
\n
option_ids<\/code>, question_type<\/code><\/p>\n<\/td>\n
\n
\u043c\u0435\u0442\u0430\u0434\u0430\u043d\u043d\u044b\u0435 \u0434\u043b\u044f \u0441\u0431\u043e\u0440\u043a\u0438 \u043e\u0442\u0432\u0435\u0442\u0430<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n
\u042d\u0442\u0430\u043f 2. Qwen: \u043e\u0434\u0438\u043d \u043f\u0440\u043e\u0445\u043e\u0434<\/h3>\n
<\/figure>\n
\u041d\u0430 \u044d\u0442\u043e\u043c \u044d\u0442\u0430\u043f\u0435 \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0440\u0430\u0441\u0447\u0435\u0442 \u043f\u043e \u0432\u0441\u0435\u043c \u0441\u043b\u043e\u044f\u043c \u0431\u0435\u0437 \u0433\u043e\u043b\u043e\u0432\u044b \u0438 \u043f\u0440\u0438\u043c\u0435\u043d\u0435\u043d\u0438\u0435 LayerNorm \u043a \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0435\u043c\u0443 \u0432\u044b\u0445\u043e\u0434\u043d\u043e\u043c\u0443 \u0441\u043b\u043e\u044e:<\/p>\n
\"H=\\mathrm{Qwen}(x_1,\\dots,x_N)\\in\\mathbb{R}^{N\\times<\/p>\n
\u0412\u044b\u0445\u043e\u0434 \u044d\u0442\u0430\u043f\u0430<\/h4>\n
last_hidden_state<\/code> \"[B\\times\u2014 \u043f\u043e \u0432\u0435\u043a\u0442\u043e\u0440\u0443 \u043d\u0430 \u043a\u0430\u0436\u0434\u044b\u0439 \u0442\u043e\u043a\u0435\u043d \u043a\u0430\u0436\u0434\u043e\u0439 \u0437\u0430\u043f\u0438\u0441\u0438 \u0431\u0430\u0442\u0447\u0430.<\/p>\n
\u042d\u0442\u0430\u043f A. \u0418\u0437\u0432\u043b\u0435\u0447\u0435\u043d\u0438\u0435 \u0432\u0435\u043a\u0442\u043e\u0440\u043e\u0432<\/h3>\n
<\/figure>\n
\u041f\u043e\u0441\u043b\u0435 \u044d\u0442\u0430\u043f\u0430 2 \u0435\u0441\u0442\u044c \u0442\u0430\u0431\u043b\u0438\u0446\u0430 \"\\tilde. \u041d\u043e\u043c\u0435\u0440 \u0441\u0442\u0440\u043e\u043a\u0438 \u0441\u043e\u0432\u043f\u0430\u0434\u0430\u0435\u0442 \u0441 \u043d\u043e\u043c\u0435\u0440\u043e\u043c \u0442\u043e\u043a\u0435\u043d\u0430 \u0432 input_ids<\/code>. \u042d\u0442\u0430\u043f A \u0432\u044b\u0440\u0435\u0437\u0430\u0435\u0442 \u0438\u0437 \u044d\u0442\u043e\u0439 \u0442\u0430\u0431\u043b\u0438\u0446\u044b \u043d\u0443\u0436\u043d\u044b\u0435 \u0441\u0442\u0440\u043e\u043a\u0438 (\u0441\u043f\u0430\u043d\u044b, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0431\u044b\u043b\u0438 \u0432\u044b\u0434\u0435\u043b\u0435\u043d\u044b \u0440\u0430\u043d\u0435\u0435).<\/p>\n
\u0441\u0442\u0440\u043e\u043a\u0430   \u0442\u043e\u043a\u0435\u043d            \u0432\u0435\u043a\u0442\u043e\u0440  0      <|im_start|>     [ ... ] ... 152     Which            [ ... ]  \u2510 ...                               \u251c question_span team = (152, 160) 159     ?                [ ... ]  \u2518 ... 164     {\"               [ ... ]  \u2510 165     description      [ ... ]  \u2502 ...                               \u251c option_span billing = (164, 184) 183     \"}               [ ... ]  \u2518 ... 319     :                [ ... ]  \u2190 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u0439 \u0442\u043e\u043a\u0435\u043d<\/code>
<\/span><\/a><\/div><\/pre>\n
\n\u041a\u043e\u0434<\/summary>\n
\n
    def forward(        self,        hidden_states: torch.Tensor,        input_ids: torch.Tensor,        attention_mask: torch.Tensor,        records: list[EncodedRecord],        output_embedding_weight: torch.Tensor,    ) -> list[list[torch.Tensor]]:        results: list[list[torch.Tensor]] = []        normalized_hidden = self.hidden_norm(hidden_states)        for batch_index, record in enumerate(records):            sequence_length = int(attention_mask[batch_index].sum().item())            sequence_hidden = normalized_hidden[batch_index, :sequence_length]            memory = self.memory_projection(sequence_hidden).unsqueeze(0)            global_vector = sequence_hidden[-1]            question_vectors = torch.stack(                [                    self._mean_span(sequence_hidden, question.question_span)                    for question in record.questions                ]            )            type_ids = torch.tensor(                [question.question_type for question in record.questions],                device=hidden_states.device,            )            option_contexts: list[torch.Tensor] = []            lexical_options: list[torch.Tensor] = []            option_counts = []            for question in record.questions:                context_vectors = torch.stack(                    [                        self._mean_span(sequence_hidden, span)                        for span in question.option_spans                    ]                )                lexical_vectors = []                for start, end in question.option_spans:                    token_ids = input_ids[batch_index, start:end]                    lexical_vectors.append(output_embedding_weight[token_ids].mean(dim=0))                lexical = torch.stack(lexical_vectors)                option_contexts.append(context_vectors)                lexical_options.append(lexical)                option_counts.append(len(question.option_spans))```<\/code>
<\/span><\/a><\/div><\/pre>\n<\/div>\n<\/details>\n
\u0421\u0430\u043c\u0430 \u043e\u043f\u0435\u0440\u0430\u0446\u0438\u044f \u0441\u0440\u0435\u0434\u043d\u0435\u0433\u043e \u043f\u043e \u0441\u043f\u0430\u043d\u0443:<\/p>\n
    @staticmethod    def _mean_span(values: torch.Tensor, span: tuple[int, int]) -> torch.Tensor:        start, end = span        return values[start:end].mean(dim=0)<\/code>
<\/span><\/a><\/div><\/pre>\n
\u0412 \u0438\u0442\u043e\u0433\u0435 \u0438\u043c\u0435\u0435\u043c \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0435 \u0441\u0443\u0449\u043d\u043e\u0441\u0442\u0438:<\/p>\n
\n
\n\n\n\n\n\n\n\n\n
\n
\u0412\u044b\u0445\u043e\u0434<\/p>\n<\/th>\n
\n
\u041a\u0430\u043a\u0438\u0435 \u0441\u0442\u0440\u043e\u043a\u0438 \u0431\u0435\u0440\u0443\u0442\u0441\u044f<\/p>\n<\/th>\n
\n
\u0420\u0430\u0437\u043c\u0435\u0440<\/p>\n<\/th>\n
\n
\u0421\u043c\u044b\u0441\u043b<\/p>\n<\/th>\n<\/tr>\n
\n
memory \"\\mathcal<\/p>\n<\/td>\n
\n
 \"\\tilde\u0443\u043c\u043d\u043e\u0436\u0435\u043d\u043d\u0430\u044f \u043d\u0430 \"W_{\\text{mem}}\"<\/p>\n<\/td>\n
\n
\"[N\\times1024]\"<\/p>\n<\/td>\n
\n
\u00ab\u0431\u0430\u0437\u0430 \u0443\u043b\u0438\u043a\u00bb, \u0432\u0435\u0441\u044c \u0442\u0435\u043a\u0441\u0442<\/p>\n<\/td>\n<\/tr>\n
\n
\"g\"<\/p>\n<\/td>\n
\n
\u043e\u0434\u043d\u0430 \u0441\u0442\u0440\u043e\u043a\u0430\"<\/p>\n<\/td>\n
\n
\"[5120]\"<\/p>\n<\/td>\n
\n
\u0441\u0436\u0430\u0442\u043e\u0435 \u00ab\u0432\u0441\u0451 \u043f\u0440\u043e\u0447\u0438\u0442\u0430\u043d\u043d\u043e\u0435\u00bb<\/p>\n<\/td>\n<\/tr>\n
\n
\"q_k\"<\/p>\n<\/td>\n
\n
\u0441\u0442\u0440\u043e\u043a\u0438 \u0438\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u0438 \u0432\u043e\u043f\u0440\u043e\u0441\u0430<\/p>\n<\/td>\n
\n
\"K\\times[5120]\"<\/p>\n<\/td>\n
\n
\u0441\u043c\u044b\u0441\u043b \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430<\/p>\n<\/td>\n<\/tr>\n
\n
\"c_{kj}<\/p>\n<\/td>\n
\n
\u0441\u0442\u0440\u043e\u043a\u0438 \u043e\u043f\u0446\u0438\u0438 \u0432 \"\\tilde<\/p>\n<\/td>\n
\n
\"M\\times[5120]\"<\/p>\n<\/td>\n
\n
\u043e\u043f\u0446\u0438\u044f \u0432 \u043a\u043e\u043d\u0442\u0435\u043a\u0441\u0442\u0435<\/p>\n<\/td>\n<\/tr>\n
\n
\"\\ell_{kj}<\/p>\n<\/td>\n
\n
\u0442\u0435 \u0436\u0435 \u043f\u043e\u0437\u0438\u0446\u0438\u0438, \u043d\u043e \u0447\u0435\u0440\u0435\u0437 input_ids<\/code> \u0432 \u0442\u0430\u0431\u043b\u0438\u0446\u0435 lm_head<\/code><\/p>\n<\/td>\n
\n
\"M\\times[5120]\"<\/p>\n<\/td>\n
\n
\u0441\u043b\u043e\u0432\u0430\u0440\u043d\u044b\u0439 \u0441\u043c\u044b\u0441\u043b \u0441\u043b\u043e\u0432 \u043e\u043f\u0446\u0438\u0438<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n
\u041d\u0435\u043c\u043d\u043e\u0433\u043e \u0444\u043e\u0440\u043c\u0443\u043b:<\/p>\n
\"\\mathcal<\/p>\n
\"q_k<\/p>\n
\n\u041f\u0440\u0438\u043c\u0435\u0440:<\/summary>\n
\n
\u0412\u0435\u043a\u0442\u043e\u0440 \u0432\u043e\u043f\u0440\u043e\u0441\u0430.<\/strong> \u041f\u0443\u0441\u0442\u044c question_span<\/code> team = (5, 8), \u0442\u043e \u0435\u0441\u0442\u044c \u0441\u0442\u0440\u043e\u043a\u0438 5, 6, 7:<\/p>\n
\u0441\u0442\u0440\u043e\u043a\u0430 5  \"Which\"   [ 0.2,  1.0, -0.4,  0.6 ]\u0441\u0442\u0440\u043e\u043a\u0430 6  \"team\"    [ 0.8,  0.6,  0.0,  0.2 ]\u0441\u0442\u0440\u043e\u043a\u0430 7  \"?\"       [-0.1,  0.2,  0.4,  1.0 ]                    \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 \u0441\u0440\u0435\u0434\u043d\u0435\u0435 \u043f\u043e \u0441\u0442\u043e\u043b\u0431\u0446\u0430\u043cq_team            = [ 0.3,  0.6,  0.0,  0.6 ]<\/code>
<\/span><\/a><\/div><\/pre>\n
\u041a\u0430\u0436\u0434\u0430\u044f \u043a\u043e\u043e\u0440\u0434\u0438\u043d\u0430\u0442\u0430 — \u0441\u0440\u0435\u0434\u043d\u0435\u0435 \u0442\u0440\u0451\u0445 \u0447\u0438\u0441\u0435\u043b \u0432 \u0441\u0442\u043e\u043b\u0431\u0446\u0435: \u043d\u0430\u043f\u0440\u0438\u043c\u0435\u0440, (0.2+0.8-0.1)\/3=0.3.<\/p>\n
\u041a\u043e\u043d\u0442\u0435\u043a\u0441\u0442\u043d\u044b\u0439 \u0432\u0435\u043a\u0442\u043e\u0440 \u043e\u043f\u0446\u0438\u0438. \u041f\u0443\u0441\u0442\u044c option_span<\/code> billing = (9, 11), \u0442\u043e \u0435\u0441\u0442\u044c \u0441\u0442\u0440\u043e\u043a\u0438 9 \u0438 10:<\/p>\n
\u0441\u0442\u0440\u043e\u043a\u0430  9  \"billing\"    [ 1.2, -0.2,  0.4,  0.0 ]\u0441\u0442\u0440\u043e\u043a\u0430 10  \"payments\"   [ 0.6,  0.4,  0.2,  0.8 ]                        \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500ctx_billing           = [ 0.9,  0.1,  0.3,  0.4 ]<\/code>
<\/span><\/a><\/div><\/pre>\n
\u041b\u0435\u043a\u0441\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0432\u0435\u043a\u0442\u043e\u0440 \u043e\u043f\u0446\u0438\u0438.<\/strong> \u041f\u043e\u0437\u0438\u0446\u0438\u0438 \u0442\u0435 \u0436\u0435 (9, 11), \u043d\u043e \u0441\u0442\u0440\u043e\u043a\u0438 \u0431\u0435\u0440\u0443\u0442\u0441\u044f \u043d\u0435 \u0438\u0437 \"\\tilde. \u0421\u043d\u0430\u0447\u0430\u043b\u0430 \u043f\u043e \u0441\u043f\u0430\u043d\u0443 \u0447\u0438\u0442\u0430\u044e\u0442\u0441\u044f \u043d\u043e\u043c\u0435\u0440\u0430 \u0442\u043e\u043a\u0435\u043d\u043e\u0432, \u043f\u043e\u0442\u043e\u043c \u043f\u043e \u044d\u0442\u0438\u043c \u043d\u043e\u043c\u0435\u0440\u0430\u043c \u0431\u0435\u0440\u0443\u0442\u0441\u044f \u0441\u0442\u0440\u043e\u043a\u0438 lm_head<\/code> (\u043d\u043e\u043c\u0435\u0440\u0430 \u0443\u0441\u043b\u043e\u0432\u043d\u044b\u0435):<\/p>\n
input_ids[9]  = 3729    \u2192  E[3729]  = [ 0.05, -0.02,  0.10,  0.01 ]input_ids[10] = 14310   \u2192  E[14310] = [ 0.03,  0.04,  0.02, -0.01 ]                                       \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500lex_billing                          = [ 0.04,  0.01,  0.06,  0.00 ]<\/code>
<\/span><\/a><\/div><\/pre>\n
\u0420\u0430\u0437\u043d\u0438\u0446\u0430 \u043c\u0435\u0436\u0434\u0443 ctx \u0438 lex: \u0441\u0442\u0440\u043e\u043a\u0430 9 \u0432 \"\\tilde— \u044d\u0442\u043e \u0441\u043b\u043e\u0432\u043e \u00abbilling\u00bb, \u043f\u0440\u043e\u0447\u0438\u0442\u0430\u043d\u043d\u043e\u0435 Qwen \u043f\u043e\u0441\u043b\u0435 \u0442\u0438\u043a\u0435\u0442\u0430, \u0438 \u043e\u043d\u0430 \u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f \u043e\u0442 \u0437\u0430\u043f\u0440\u043e\u0441\u0430 \u043a \u0437\u0430\u043f\u0440\u043e\u0441\u0443. \u0421\u0442\u0440\u043e\u043a\u0430 \"E_{3729}\" — \u0441\u043b\u043e\u0432\u043e \u00abbilling\u00bb \u0441\u0430\u043c\u043e \u043f\u043e \u0441\u0435\u0431\u0435, \u0438 \u043e\u043d\u0430 \u0432\u0441\u0435\u0433\u0434\u0430 \u043e\u0434\u043d\u0430 \u0438 \u0442\u0430 \u0436\u0435.<\/p>\n
g \u0438 memory.<\/strong> g — \u043f\u0440\u043e\u0441\u0442\u043e \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u044f\u044f \u0441\u0442\u0440\u043e\u043a\u0430 \u0442\u0430\u0431\u043b\u0438\u0446\u044b. memory — \u0432\u0441\u0435 \u0441\u0442\u0440\u043e\u043a\u0438, \u043a\u0430\u0436\u0434\u0430\u044f \u0443\u043c\u043d\u043e\u0436\u0435\u043d\u043d\u0430\u044f \u043d\u0430 \"W_{\\text{mem}}\". \u0421\u043f\u0430\u043d\u044b \u0442\u0443\u0442 \u043d\u0435 \u043d\u0443\u0436\u043d\u044b.<\/p>\n<\/div>\n<\/details>\n
\u0412\u044b\u0445\u043e\u0434 \u044d\u0442\u0430\u043f\u0430<\/h4>\n
\n
\n\n\n\n\n\n\n\n\n\n\n
\n
\u0412\u044b\u0445\u043e\u0434<\/p>\n<\/th>\n
\n
\u0420\u0430\u0437\u043c\u0435\u0440<\/p>\n<\/th>\n
\n
\u041a\u0430\u043a \u0445\u0440\u0430\u043d\u0438\u0442\u0441\u044f<\/p>\n<\/th>\n<\/tr>\n
\n
memory<\/p>\n<\/td>\n
\n
\"[N\\times1024]\"<\/p>\n<\/td>\n
\n
\u043e\u0434\u043d\u0430 \u043c\u0430\u0442\u0440\u0438\u0446\u0430<\/p>\n<\/td>\n<\/tr>\n
\n
\"g\"<\/p>\n<\/td>\n
\n
\"[5120]\"<\/p>\n<\/td>\n
\n
\u043e\u0434\u0438\u043d \u0432\u0435\u043a\u0442\u043e\u0440<\/p>\n<\/td>\n<\/tr>\n
\n
\"q\"<\/p>\n<\/td>\n
\n
\"K\\times[5120]\"<\/p>\n<\/td>\n
\n
\u043f\u043e \u0432\u0435\u043a\u0442\u043e\u0440\u0443 \u043d\u0430 \u0432\u043e\u043f\u0440\u043e\u0441, \u0432 \u043f\u043e\u0440\u044f\u0434\u043a\u0435 \u0432\u043e\u043f\u0440\u043e\u0441\u043e\u0432<\/p>\n<\/td>\n<\/tr>\n
\n
\"ctx\"<\/p>\n<\/td>\n
\n
\"M\\times[5120]\"<\/p>\n<\/td>\n
\n
\u0441\u043f\u0438\u0441\u043a\u0438 \u043f\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430\u043c: 2, 3, 4<\/p>\n<\/td>\n<\/tr>\n
\n
\"lex\"<\/p>\n<\/td>\n
\n
\"M\\times[5120]\"<\/p>\n<\/td>\n
\n
\u0441\u043f\u0438\u0441\u043a\u0438 \u043f\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430\u043c: 2, 3, 4<\/p>\n<\/td>\n<\/tr>\n
\n
option_counts<\/code><\/p>\n<\/td>\n
\n
[2, 3, 4]<\/code><\/p>\n<\/td>\n
\n
\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043e\u043f\u0446\u0438\u0439 \u0443 \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430<\/p>\n<\/td>\n<\/tr>\n
\n
type_ids<\/code><\/p>\n<\/td>\n
\n
[0, 1, 2]<\/code><\/p>\n<\/td>\n
\n
\u0442\u0438\u043f \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n
\u041f\u0440\u0435\u0436\u0434\u0435 \u0447\u0435\u043c \u043f\u0435\u0440\u0435\u0445\u043e\u0434\u0438\u0442\u044c \u043a \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u043c\u0443 \u044d\u0442\u0430\u043f\u0443 \u0440\u0430\u0441\u043f\u0438\u0448\u0443 \u043a\u0430\u043a\u0438\u0435 \u043f\u0440\u043e\u0435\u043a\u0446\u0438\u0438 \u043c\u044b \u0438\u043c\u0435\u0435\u043c:<\/p>\n
self.hidden_norm = torch.nn.LayerNorm(hidden_size)self.memory_projection = torch.nn.Linear(hidden_size, width, bias=False)self.question_projection = torch.nn.Linear(hidden_size, width, bias=False)self.option_question_projection = torch.nn.Linear(hidden_size, width, bias=False)self.global_projection = torch.nn.Linear(hidden_size, width, bias=False)self.option_context_projection = torch.nn.Linear(hidden_size, width, bias=False)self.option_lexical_projection = torch.nn.Linear(hidden_size, width, bias=False)self.type_embedding = torch.nn.Embedding(3, width)<\/code>
<\/span><\/a><\/div><\/pre>\n
\n
\n\n\n\n\n\n\n\n\n\n\n
\n
\u041c\u0430\u0442\u0440\u0438\u0446\u0430<\/p>\n<\/th>\n
\n
\u0421\u043b\u043e\u0439 \u0432 \u043a\u043e\u0434\u0435<\/p>\n<\/th>\n
\n
\u0412\u0445\u043e\u0434<\/p>\n<\/th>\n<\/tr>\n
\n
\"W_{\\text{mem}}\"<\/p>\n<\/td>\n
\n
memory_projection<\/code><\/p>\n<\/td>\n
\n
\u0432\u0441\u0435 \u0442\u043e\u043a\u0435\u043d\u044b \"\\tilde<\/p>\n<\/td>\n<\/tr>\n
\n
\"W_c\"<\/p>\n<\/td>\n
\n
option_context_projection<\/code><\/p>\n<\/td>\n
\n
\"ctx\" \u043a\u0430\u0436\u0434\u043e\u0439 \u043e\u043f\u0446\u0438\u0438 \"[M\\times5120]\"<\/p>\n<\/td>\n<\/tr>\n
\n
\"W_\\ell\"<\/p>\n<\/td>\n
\n
option_lexical_projection<\/code><\/p>\n<\/td>\n
\n
\"lex\" \u043a\u0430\u0436\u0434\u043e\u0439 \u043e\u043f\u0446\u0438\u0438 \"[M\\times5120]\"<\/p>\n<\/td>\n<\/tr>\n
\n
\"W_{oq}\"<\/p>\n<\/td>\n
\n
option_question_projection<\/code><\/p>\n<\/td>\n
\n
\"q\" \u0432\u043e\u043f\u0440\u043e\u0441\u0430, \u043a\u043e\u0442\u043e\u0440\u043e\u043c\u0443 \u043f\u0440\u0438\u043d\u0430\u0434\u043b\u0435\u0436\u0438\u0442 \u043e\u043f\u0446\u0438\u044f<\/p>\n<\/td>\n<\/tr>\n
\n
\"W_q\"<\/p>\n<\/td>\n
\n
question_projection<\/code><\/p>\n<\/td>\n
\n
\"q\" \u043a\u0430\u0436\u0434\u043e\u0433\u043e \u0432\u043e\u043f\u0440\u043e\u0441\u0430 \"[K\\times5120]\"<\/p>\n<\/td>\n<\/tr>\n
\n
\"W_g\"<\/p>\n<\/td>\n
\n
global_projection<\/code><\/p>\n<\/td>\n
\n
\"g<\/p>\n<\/td>\n<\/tr>\n
\n
\"T\"<\/p>\n<\/td>\n
\n
type_embedding<\/code><\/p>\n<\/td>\n
\n
\u0442\u0438\u043f \u0432\u043e\u043f\u0440\u043e\u0441\u0430 (0, 1, 2)<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n
\u042d\u0442\u0430\u043f B. Evidence routing \u00d72<\/h3>\n
<\/figure>\n
Evidence routing — \u044d\u0442\u0430\u043f, \u0432 \u043a\u043e\u0442\u043e\u0440\u043e\u043c \u043a\u0430\u0436\u0434\u0430\u044f \u043e\u043f\u0446\u0438\u044f \u0441\u0430\u043c\u0430 \u0438\u0449\u0435\u0442 \u0432 \u0442\u0435\u043a\u0441\u0442\u0435 \u043f\u043e\u0434\u0442\u0432\u0435\u0440\u0436\u0434\u0435\u043d\u0438\u044f \u0434\u043b\u044f \u0441\u0435\u0431\u044f. \u042d\u0442\u043e \u0434\u0432\u0430 \u043e\u0434\u0438\u043d\u0430\u043a\u043e\u0432\u044b\u0445 \u043f\u043e \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0443 \u0441\u043b\u043e\u044f EvidenceRoutingLayer<\/code>, \u043a\u0430\u0436\u0434\u044b\u0439 \u0441\u043e \u0441\u0432\u043e\u0438\u043c\u0438 \u0432\u0435\u0441\u0430\u043c\u0438.<\/p>\n
\n\u0421\u0430\u043c \u0421\u043b\u043e\u0439<\/summary>\n
\n
class EvidenceRoutingLayer(torch.nn.Module):    def __init__(        self,        width: int,        heads: int,        feedforward: int,        dropout: float = 0.0,    ) -> None:        super().__init__()        self.query_norm = torch.nn.LayerNorm(width)        self.memory_norm = torch.nn.LayerNorm(width)        self.attention = torch.nn.MultiheadAttention(            width,            heads,            dropout=dropout,            batch_first=True,        )        self.attention_dropout = torch.nn.Dropout(dropout)        self.feedforward_norm = torch.nn.LayerNorm(width)        self.feedforward = torch.nn.Sequential(            torch.nn.Linear(width, feedforward),            torch.nn.GELU(),            torch.nn.Dropout(dropout),            torch.nn.Linear(feedforward, width),            torch.nn.Dropout(dropout),        )    def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:        normalized_queries = self.query_norm(queries)        routed, _ = self.attention(            normalized_queries,            self.memory_norm(memory),            self.memory_norm(memory),            need_weights=False,        )        queries = queries + self.attention_dropout(routed)        return queries + self.feedforward(self.feedforward_norm(queries))<\/code>
<\/span><\/a><\/div><\/pre>\n
\u041a\u0430\u0436\u0434\u0430\u044f \u043e\u043f\u0446\u0438\u044f \u043e\u0431\u0440\u0430\u0431\u0430\u0442\u044b\u0432\u0430\u0435\u0442\u0441\u044f \u043d\u0435\u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e \u0438 \u043d\u0430 \u0432\u044b\u0445\u043e\u0434\u0435 \u043f\u043e\u043b\u0443\u0447\u0430\u0435\u043c \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u0438\u0435 \u00ab\u0432\u0430\u0436\u043d\u043e\u0441\u0442\u0438\u00bb \u0442\u043e\u043a\u0435\u043d\u0430 \u0432\u0441\u0435\u0433\u043e \u0437\u0430\u043f\u0440\u043e\u0441\u0430 \u043a \u043a\u043e\u043d\u043a\u0440\u0435\u0442\u043d\u043e\u0439 \u043e\u043f\u0446\u0438\u0438.<\/p>\n<\/div>\n<\/details>\n
\u0412\u0445\u043e\u0434<\/h4>\n
\"u^{(0)}_{kj}=W_c<\/p>\n
\n\u0414\u043b\u044f \u0432\u043e\u043f\u0440\u043e\u0441\u0430 team:<\/summary>\n
\n
u_billing   = W_c\u00b7ctx_billing   + W_\u2113\u00b7lex_billing   + W_oq\u00b7q_teamu_sales     = W_c\u00b7ctx_sales     + W_\u2113\u00b7lex_sales     + W_oq\u00b7q_teamu_technical = W_c\u00b7ctx_technical + W_\u2113\u00b7lex_technical + W_oq\u00b7q_team<\/code>
<\/span><\/a><\/div><\/pre>\n<\/div>\n<\/details>\n
\u0427\u0442\u043e \u0434\u0430\u0451\u0442 \u043a\u0430\u0436\u0434\u043e\u0435 \u0441\u043b\u0430\u0433\u0430\u0435\u043c\u043e\u0435:<\/p>\n