{"id":308444,"date":"2020-08-13T21:00:26","date_gmt":"2020-08-13T21:00:26","guid":{"rendered":"http:\/\/savepearlharbor.com\/?p=308444"},"modified":"-0001-11-30T00:00:00","modified_gmt":"2020-08-13T21:00:26","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=308444","title":{"rendered":"\u041e\u0431\u0437\u043e\u0440 \u043c\u0435\u0442\u043e\u0434\u043e\u0432 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u043e\u0432 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0439, \u0427\u0430\u0441\u0442\u044c2"},"content":{"rendered":"\n
\n

\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435, \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0435\u043d\u0438\u0435 \u0441\u0442\u0430\u0442\u044c\u0438 \u043f\u0440\u043e \u043c\u0435\u0442\u043e\u0434\u044b \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u043e\u0432 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0439. \u0412 \u044d\u0442\u043e\u043c \u0433\u0430\u0439\u0434\u0435 \u043c\u0430\u043b\u043e \u0441\u043b\u043e\u0432 \u0438 \u043c\u043d\u043e\u0433\u043e \u043a\u043e\u0434\u0430, \u0433\u043e\u0442\u043e\u0432\u043e \u0434\u043b\u044f Ctrl+\u0441, Ctrl+v \u0434\u043b\u044f \u0443\u0434\u0443\u0447\u0448\u0435\u043d\u0438\u0439 \u0438 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0438\u0445 \u0442\u0435\u0441\u0442\u043e\u0432.
\u0427\u0430\u0441\u0442\u044c1<\/a> \u043e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c\u043d\u0430 \u0434\u043b\u044f \u043e\u0437\u043d\u0430\u043a\u043e\u043c\u043b\u0435\u043d\u0438\u044f<\/p>\n

<\/p>\n

4. BERT<\/h1>\n

<\/p>\n

from deeppavlov.core.common.file import read_json from deeppavlov import build_model, configs from deeppavlov.models.embedders.elmo_embedder import ELMoEmbedder # \u0441\u0441\u044b\u043b\u043a\u0430 \u0434\u043b\u044f \u0441\u043a\u0430\u0447\u0438\u0432\u0430\u043d\u0438\u044f \u043c\u043e\u0434\u0435\u043b\u0435\u0439 http:\/\/docs.deeppavlov.ai\/en\/master\/features\/pretrained_vectors.html<\/code><\/pre>\n
  <\/p>\n
4.1 rubert_cased_L-12_H-768_A-12_pt<\/h3>\n
  <\/p>\n
class RU_BERT_CLASS:     def __init__(self, name):         bert_config = read_json(configs.embedder.bert_embedder)         bert_config['metadata']['variables']['BERT_PATH'] = os.path.join('.\/.', name)         self.m = build_model(bert_config)      def vectorizer(self, sentences):         return [sentence.split() for sentence in sentences]      def predict(self, tokens):         _, _, _, _, sent_max_embs, sent_mean_embs, _ = self.m(tokens)         return sent_mean_embs  bert = RU_BERT_CLASS('rubert_cased_L-12_H-768_A-12_pt') get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'rubert')<\/code><\/pre>\n
  <\/p>\n
‘rubert: 2895.7’<\/p>\n
<\/a>  <\/p>\n
4.2 ru_conversational_cased_L-12_H-768_A-12_pt<\/h3>\n
  <\/p>\n
bert = RU_BERT_CLASS('ru_conversational_cased_L-12_H-768_A-12_pt') get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'ru_conversational')<\/code><\/pre>\n
  <\/p>\n
‘ru_conversational: 3559.1’<\/p>\n
  <\/p>\n
4.3 sentence_ru_cased_L-12_H-768_A-12_pt<\/h3>\n
  <\/p>\n
bert = RU_BERT_CLASS('sentence_ru_cased_L-12_H-768_A-12_pt') get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'sentence_ru')<\/code><\/pre>\n
  <\/p>\n
‘sentence_ru: 2660.2’<\/p>\n
  <\/p>\n
4.4 elmo_ru-news_wmt11-16_1.5M_steps<\/h3>\n
  <\/p>\n
class ELMO_CLASS(RU_BERT_CLASS):     def __init__(self, name):         self.m = ELMoEmbedder(f"http:\/\/files.deeppavlov.ai\/deeppavlov_data\/{name}")      def predict(self, tokens):         return self.m(tokens)<\/code><\/pre>\n
  <\/p>\n
elmo = ELMO_CLASS('elmo_ru-news_wmt11-16_1.5M_steps.tar.gz') get_similarity_values = similarity_values_wrapper(elmo.predict, elmo.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'elmo_ru-news')<\/code><\/pre>\n
  <\/p>\n
‘elmo_ru-news: 4631.3’<\/p>\n
  <\/p>\n
4.5 elmo_ru-wiki_600k_steps<\/h3>\n
  <\/p>\n
elmo = ELMO_CLASS('elmo_ru-wiki_600k_steps.tar.gz') get_similarity_values = similarity_values_wrapper(elmo.predict, elmo.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'elmo_ru-wiki')<\/code><\/pre>\n
  <\/p>\n
‘elmo_ru-wiki: 4507.6’<\/p>\n
  <\/p>\n
4.6 elmo_ru-twitter_2013-01_2018-04_600k_steps<\/h3>\n
  <\/p>\n
elmo = ELMO_CLASS('elmo_ru-twitter_2013-01_2018-04_600k_steps.tar.gz') get_similarity_values = similarity_values_wrapper(elmo.predict, elmo.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'elmo_ru-twitter')<\/code><\/pre>\n
  <\/p>\n
‘elmo_ru-twitter: 2962.2’<\/p>\n
  <\/p>\n
plot_results()<\/code><\/pre>\n
  <\/p>\n
\"png\"\/<\/p>\n
  <\/p>\n
5. \u0410\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440\u044b<\/h1>\n
  <\/p>\n
\u0410\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440\u044b \u0441\u043e\u0437\u0434\u0430\u043d\u044b \u0434\u043b\u044f \u0441\u0436\u0430\u0442\u0438\u044f \u043c\u043d\u043e\u0433\u043e\u043c\u0435\u0440\u043d\u043e\u0433\u043e \u0432\u0435\u0442\u043e\u0440\u0430 \u0434\u043e \u043e\u0434\u043d\u043e\u043c\u0435\u0440\u043d\u043e\u0433\u043e \u0438, \u0442\u0435\u043e\u0440\u0435\u0442\u0438\u0447\u0435\u0441\u043a\u0438, \u0434\u043e\u043b\u0436\u043d\u044b \u0438\u0434\u0435\u0430\u043b\u044c\u043d\u043e \u043f\u043e\u0434\u043e\u0439\u0442\u0438 \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u043e\u0432 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f. <\/p>\n
  <\/p>\n
5.1 \u0410\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 embedings -> embedings<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     def cosine_loss(y_true, y_pred):         return K.mean(cosine_similarity(y_true, y_pred, axis=-1))      complexity = 300     inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))     X = inp     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Bidirectional(LSTM(int(complexity\/10), return_sequences=True))(X)     X = Flatten()(X)     X = Dense(complexity, activation='elu')(X)     X = Dense(complexity, activation='elu')(X)     X = Dense(complexity, activation='linear', name='embeding_output')(X)     X = Dense(complexity, activation='elu')(X)     X = Dense(data_generator.max_len*complexity, activation='elu')(X)     X = Reshape((data_generator.max_len, complexity))(X)     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Dense(data_generator.embedding_size, activation='elu')(X)     autoencoder = Model(inputs=inp, outputs=X)     autoencoder.compile(loss=cosine_loss, optimizer='adam')     autoencoder.summary()      embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)     return autoencoder, embedder  data_generator = EmbedingsDataGenerator(use_fasttext=False) autoencoder, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize, distance_function=cosine_distances)<\/code><\/pre>\n
  <\/p>\n
new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 embedings -> embedings')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x, y in data_generator:         autoencoder.train_on_batch(x, x)<\/code><\/pre>\n
  <\/p>\n
0 1770.2
  3 212.6
  6 138.8
  9 84.8
  12 78.1
  15 106.4
  18 112.7
  21 79.7<\/p>\n
  <\/p>\n
5.2 \u0410\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 embedings -> indexes<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     complexity = 300     inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))     X = inp     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Bidirectional(LSTM(int(complexity\/10), return_sequences=True))(X)     X = Flatten()(X)     X = Dense(complexity, activation='elu')(X)     X = Dense(complexity, activation='elu')(X)     X = Dense(complexity, activation='linear', name='embeding_output')(X)     X = Dense(complexity, activation='elu')(X)     X = Dense(data_generator.max_len*complexity, activation='elu')(X)     X = Reshape((data_generator.max_len, complexity))(X)     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Dense(len(data_generator.token2index), activation='softmax')(X)     autoencoder = Model(inputs=inp, outputs=X)     autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])     autoencoder.summary()      embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)     return autoencoder, embedder  data_generator = IndexesDataGenerator() autoencoder, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 embedings -> indexes')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x_e, x_i, y_i in data_generator:         autoencoder.train_on_batch(x_e, x_i)<\/code><\/pre>\n
  <\/p>\n
0 1352.9
  3 43.6
  6 41.7
  9 8.1
  12 -5.6
  15 43.1
  18 36.1
  21 -3.7<\/p>\n
  <\/p>\n
5.3 \u0410\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430 LSTM -> LSTM<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     def cosine_loss(y_true, y_pred):         return K.mean(cosine_similarity(y_true, y_pred, axis=-1))      complexity = 300     inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))     X = inp     X, state_h, state_c = LSTM(complexity, return_state=True)(X)     X = Concatenate()([state_h, state_c])     X = Dense(complexity, activation='linear', name='embeding_output')(X)      state_c = Dense(complexity, activation='linear')(X)     state_h = Dense(complexity, activation='linear')(X)     inp_zeros = Input(shape=(data_generator.max_len, data_generator.embedding_size))      X = LSTM(complexity, return_sequences=True)(inp_zeros, [state_c, state_h])     X = Dense(data_generator.embedding_size, activation='linear')(X)      autoencoder = Model(inputs=[inp, inp_zeros], outputs=X)     autoencoder.compile(loss=cosine_loss, optimizer='adam')     autoencoder.summary()      embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)     return autoencoder, embedder  data_generator = EmbedingsDataGenerator(use_fasttext=False) autoencoder, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
zeros = np.zeros((data_generator.batch_size, data_generator.max_len, data_generator.embedding_size)) new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 embedings -> indexes')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x, y in data_generator:         autoencoder.train_on_batch([x, zeros], x)<\/code><\/pre>\n
  <\/p>\n
0 1903.6
  3 1299.3
  6 313.5
  9 445.3
  12 454.9
  15 447.7
  18 454.5
  21 448.1<\/p>\n
  <\/p>\n
5.4 \u0410\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430 LSTM -> LSTM -> indexes<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     complexity = 300     inp = Input(shape=(data_generator.max_len, data_generator.embedding_size))     X = inp     X, state_h, state_c = LSTM(complexity, return_state=True)(X)     X = Concatenate()([state_h, state_c])     X = Dense(complexity, activation='linear', name='embeding_output')(X)     state_c = Dense(complexity, activation='linear')(X)     state_h = Dense(complexity, activation='linear')(X)     inp_zeros = Input(shape=(data_generator.max_len, data_generator.embedding_size))      X = LSTM(complexity, return_sequences=True)(inp_zeros, [state_c, state_h])     X = Dense(len(data_generator.token2index), activation='softmax')(X)      autoencoder = Model(inputs=[inp, inp_zeros], outputs=X)     autoencoder.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])     autoencoder.summary()      embedder = Model(inputs=inp, outputs=autoencoder.get_layer('embeding_output').output)     return autoencoder, embedder  data_generator = IndexesDataGenerator() autoencoder, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
zeros = np.zeros((data_generator.batch_size, data_generator.max_len, data_generator.embedding_size)) new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430 LSTM -> LSTM -> indexes')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x_e, x_i, y_i in data_generator:         autoencoder.train_on_batch([x_e, zeros], x_i)<\/code><\/pre>\n
  <\/p>\n
0 1903.6
  3 1483.3
  6 1249.3
  9 566.3
  12 789.2
  15 702.3
  18 480.5
  21 552.3
  24 533.0<\/p>\n
  <\/p>\n
\u041c\u0435\u0442\u043e\u0434\u044b \u0441 \u0443\u0447\u0438\u0442\u0435\u043b\u0435\u043c<\/h1>\n
  <\/p>\n
6. \u042d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u0438 \u043d\u0430 Transfer Learning<\/h1>\n
  <\/p>\n
TEXTS_CORPUS_WITH_LABEL = [(sentence, topic) for topic in texts_for_training for sentence in texts_for_training[topic]]  class BowDataGenerator(EmbedingsDataGenerator):     def __init__(self, texts_topics=TEXTS_CORPUS_WITH_LABEL, batch_size=128, batches_per_epoch=100):         self.texts_topics = texts_topics         self.topic2index = {topic: index for index, topic in enumerate({topic for text, topic in self.texts_topics})}         self.batch_size = batch_size         self.batches_per_epoch = batches_per_epoch         self.count_vectorizer = CountVectorizer().fit([text_topic[0] for text_topic in self.texts_topics])         counts = Counter([text_topic[1] for text_topic in self.texts_topics])         self.class_weight = {self.topic2index[intent_id]:1\/counts[intent_id] for intent_id in counts}      def vectorize(self, sentences):         return self.count_vectorizer.transform(sentences).toarray()      def __iter__(self):         for _ in tqdm(range(self.batches_per_epoch), leave=False):             X_batch = []             y_batch = []             finished_batch = False             while not finished_batch:                 text, topic = random.choice(self.texts_topics)                 X_batch.append(text)                 y_batch.append(self.topic2index[topic])                  if len(X_batch) >= self.batch_size:                     X_batch = self.count_vectorizer.transform(X_batch).toarray()                     y_batch = to_categorical(y_batch, num_classes=len(self.topic2index))                     yield np.array(X_batch), np.array(y_batch)                     finished_batch = True  data_generator = BowDataGenerator()<\/code><\/pre>\n
  <\/p>\n
6.1 \u042d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u0438 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 BOW<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     complexity = 500     inp = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))     X = inp     X = Dense(complexity)(X)     X = Activation('elu')(X)     X = Dense(complexity)(X)     X = Activation('elu')(X)     X = Dense(complexity, name='embeding_output')(X)     X = Activation('elu')(X)     X = Dense(len(data_generator.topic2index), activation='softmax')(X)      model = Model(inputs=inp, outputs=X)     model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])     model.summary()      embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)     return model, embedder  data_generator = BowDataGenerator() model, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u0435\u043c\u0431\u0435\u0434\u0438\u043d\u0433 \u043d\u0430 BOW')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x, y in data_generator:         model.train_on_batch(x, y, class_weight=data_generator.class_weight)<\/code><\/pre>\n
  <\/p>\n
0 601.4
  3 1175.4
  6 1187.0
  9 1175.9
  12 1097.9
  15 1083.4
  18 1083.8
  21 1060.5<\/p>\n
  <\/p>\n
6.2 \u042d\u043c\u0431\u0435\u0434\u0438\u043d\u0433 \u043d\u0430 LSTM + MaxPooling (InferSent)<\/h3>\n
  <\/p>\n
\u0421\u044b\u044b\u043b\u043a\u0438 \u043d\u0430 \u0441\u0442\u0430\u0442\u044c:
  Arxiv \u0441 \u0442\u0435\u043e\u0440\u0438\u0435\u0439<\/a>
  \u041e\u0431\u044a\u044f\u0441\u043d\u0435\u043d\u043e \u043f\u043e-\u0447\u0435\u043b\u043e\u0432\u0435\u0447\u0435\u0441\u043a\u0438<\/a><\/p>\n
  <\/p>\n
class LabelsDataGenerator(EmbedingsDataGenerator):     def __init__(self, texts_topics=TEXTS_CORPUS_WITH_LABEL, target_len=20, batch_size=128, batches_per_epoch=100, use_word2vec=True, use_fasttext=True):         self.texts_topics = texts_topics         self.topic2index = {topic: index for index, topic in enumerate({topic for text, topic in self.texts_topics})}         self.target_len = target_len         self.batch_size = batch_size         self.batches_per_epoch = batches_per_epoch         self.use_word2vec = use_word2vec         self.use_fasttext = use_fasttext         self.embedding_size = len(vectorize('token', use_word2vec=self.use_word2vec, use_fasttext=self.use_fasttext))         counts = Counter([text_topic[1] for text_topic in self.texts_topics])         self.class_weight = {self.topic2index[intent_id]:1\/counts[intent_id] for intent_id in counts}             def vectorize(self, sentences):         vectorized = []         for text in sentences:             tokens = str(text).split()             x_vec = []             for token in tokens:                 token_vec = vectorize(token, use_word2vec=self.use_word2vec, use_fasttext=self.use_fasttext)                                        x_vec.append(token_vec)             vectorized.append(x_vec)          vectorized = pad_sequences(vectorized, maxlen=self.target_len)         return vectorized      def __iter__(self):         for _ in tqdm(range(self.batches_per_epoch), leave=False):             X_batch = []             y_batch = []             finished_batch = False             while not finished_batch:                 text, topic = random.choice(self.texts_topics)                 tokens = text.split()                 x_vec = []                 for token in tokens:                     token_vec = vectorize(token, use_word2vec=self.use_word2vec, use_fasttext=self.use_fasttext)                     if len(x_vec) >= self.target_len:                         X_batch.append(x_vec)                         y_batch.append(self.topic2index[topic])                         if len(X_batch) >= self.batch_size:                             break                     x_vec.append(token_vec)                 else:                     X_batch.append(x_vec)                     y_batch.append(self.topic2index[topic])                  if len(X_batch) >= self.batch_size:                     X_batch = pad_sequences(X_batch, maxlen=self.target_len)                     y_batch = to_categorical(y_batch, num_classes=len(self.topic2index))                     yield np.array(X_batch), np.array(y_batch)                     finished_batch = True<\/code><\/pre>\n
  <\/p>\n
def models_builder(data_generator):     complexity = 768     inp = Input(shape=(data_generator.target_len, data_generator.embedding_size))     X = inp     X = Bidirectional(LSTM(complexity, return_sequences=True))(X)     X = Permute((2,1))(X)     X = MaxPooling1D(pool_size=600)(X)     X = Flatten()(X)     X = Dense(complexity)(X)     X = Activation('elu')(X)     X = Dense(complexity, name='embeding_output')(X)     X = Activation('sigmoid')(X)     X = Dense(len(data_generator.topic2index), activation='softmax')(X)      model = Model(inputs=inp, outputs=X)     model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])     model.summary()      embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)     return model, embedder  data_generator = LabelsDataGenerator() model, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433 \u043d\u0430 LSTM + MaxPooling')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x, y in data_generator:         model.train_on_batch(x, y, class_weight=data_generator.class_weight)<\/code><\/pre>\n
  <\/p>\n
0 87.0
  3 152.1
  6 110.5
  9 146.7
  12 166.2
  15 79.8
  18 47.2
  21 84.0
  24 144.8
  27 83.8<\/p>\n
  <\/p>\n
6.3 \u042d\u043c\u0431\u0435\u0434\u0438\u043d\u0433 \u043d\u0430 LSTM + Conv1D + AveragePooling<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     complexity = 600     inp = Input(shape=(data_generator.target_len, data_generator.embedding_size))     X_R = inp     X_R = Bidirectional(LSTM(complexity, return_sequences=True))(X_R)     X_R = Bidirectional(LSTM(complexity, return_sequences=True))(X_R)      X_C = inp     X_C = Conv1D(complexity, 3, strides=1, padding='same')(X_C)     X_C = Conv1D(complexity, 3, strides=1, padding='same')(X_C)      X = Concatenate()([X_R, X_C])     X = AveragePooling1D(pool_size=2)(X)      X = Conv1D(complexity, 3, strides=1, padding='same')(X)     X = AveragePooling1D(pool_size=2)(X)      X = Conv1D(complexity, 3, strides=1, padding='same')(X)     X = AveragePooling1D(pool_size=2)(X)     X = Flatten()(X)     X = Dense(complexity)(X)     X = Activation('sigmoid')(X)     X = Dense(complexity, name = 'embeding_output')(X)     X = Activation('elu')(X)     X = Dense(len(data_generator.topic2index), activation='softmax')(X)      model = Model(inputs=inp, outputs=X)     model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])     model.summary()      embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)     return model, embedder  data_generator = LabelsDataGenerator() model, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
0 353.8
  3 -147.8
  6 7.6
  9 5.5
  12 -133.6
  15 -133.6
  18 9.0
  21 9.0
  24 -133.6<\/p>\n
  <\/p>\n
6.4 \u042d\u043c\u0431\u0435\u0434\u0438\u043d\u0433 \u043d\u0430 LSTM + Inception + Attention<\/h3>\n
  <\/p>\n
def models_builder(data_generator):     rate = 0.20     complexity = 500      def inception_convolutional_layer(X, complexity, rate=0.2, regularizer=0):         X_7 = Conv1D(int(complexity\/7), kernel_size=7, strides=1, padding='same')(X)         X_6 = Conv1D(int(complexity\/6), kernel_size=6, strides=1, padding='same')(X)         X_5 = Conv1D(int(complexity\/5), kernel_size=5, strides=1, padding='same')(X)         X_4 = Conv1D(int(complexity\/4), kernel_size=4, strides=1, padding='same')(X)         X_3 = Conv1D(int(complexity\/3), kernel_size=3, strides=1, padding='same')(X)         X_2 = Conv1D(int(complexity\/2), kernel_size=2, strides=1, padding='same')(X)         X_1 = Conv1D(int(complexity\/1), kernel_size=1, strides=1, padding='same')(X)         X = Concatenate()([X_7, X_6, X_5, X_4, X_3, X_2, X_1])         X = Activation('elu')(X)         X = BatchNormalization()(X)         X = Dropout(rate)(X)         return X      def bi_LSTM(X, complexity, rate=0.2, regularizer=0):         X = Bidirectional(LSTM(int(complexity\/2), return_sequences=True))(X)         X = BatchNormalization()(X)         X = Dropout(rate)(X)         return X      def dense_layer(X, complexity, activation='elu', rate=0.2, regularizer=0, name=None):         X = Dense(int(complexity), name=name)(X)         X = Activation(activation)(X)         X = BatchNormalization()(X)         X = Dropout(rate)(X)         return X      inp = Input(shape=(data_generator.target_len, data_generator.embedding_size))     X = inp     X = inception_convolutional_layer(X, complexity)     X = inception_convolutional_layer(X, complexity)     X = inception_convolutional_layer(X, complexity)     X = MaxPooling1D(pool_size=2)(X)     X = inception_convolutional_layer(X, complexity)     X = MaxPooling1D(pool_size=2)(X)     X = inception_convolutional_layer(X, complexity)     X = MaxPooling1D(pool_size=2)(X)      R = inp     R = bi_LSTM(R, complexity)     R = bi_LSTM(R, complexity\/2)     attention_probs = Dense(int(complexity\/2), activation='sigmoid', name='attention_probs')(R)     R = multiply([R, attention_probs], name='attention_mul')     R = Dropout(rate)(R)     R = MaxPooling1D(pool_size=2)(R)     R = inception_convolutional_layer(R, complexity)     R = MaxPooling1D(pool_size=2)(R)     R = inception_convolutional_layer(R, complexity)     R = MaxPooling1D(pool_size=2)(R)      X = Concatenate(axis=-1)([X, R])     X = Flatten()(X)     X = BatchNormalization()(X)     X = Dropout(rate)(X)      X = dense_layer(X, complexity)     X = dense_layer(X, complexity, activation='sigmoid')     X = dense_layer(X, complexity, name='embeding_output')      X = Dense(len(data_generator.topic2index), activation='softmax')(X)      model = Model(inputs=inp, outputs=X)     model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])     model.summary()      embedder = Model(inputs=inp, outputs=model.get_layer('embeding_output').output)     return model, embedder  data_generator = LabelsDataGenerator() model, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, '\u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433 \u043d\u0430 LSTM + Inception + Attention')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for x, y in data_generator:         model.train_on_batch(x, y, class_weight=data_generator.class_weight)<\/code><\/pre>\n
  <\/p>\n
0 275.0
  3 126.8
  6 173.9
  9 155.5
  12 168.4
  15 287.2
  18 382.8
  21 303.4<\/p>\n
  <\/p>\n
plot_results()<\/code><\/pre>\n
  <\/p>\n
\"png\"\/<\/p>\n
  <\/p>\n
7 Triplet loss<\/h1>\n
  <\/p>\n
\u041e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 \u0431\u0443\u0434\u0435\u0442 \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u043d\u0430 \u0442\u043e\u043c, \u0447\u0442\u043e \u043c\u044b \u0432\u0435\u043a\u0442\u043e\u0440\u044b \u0438\u0437 \u043e\u0434\u043d\u043e\u0433\u043e \u0438\u043d\u0442\u0435\u043d\u0442\u0430 \u0434\u043e\u043b\u0436\u043d\u044b \u0440\u0430\u0441\u043f\u043e\u043b\u043e\u0433\u0430\u0442\u044c\u0441\u044f \u0431\u043b\u0438\u0436\u0435 \u0434\u0440\u0443\u0433 \u043a \u0434\u0440\u0443\u0433\u0443, \u0430 \u0438\u0437 \u0440\u0430\u0437\u043d\u044b\u0445 \u0438\u043d\u0442\u0435\u043d\u0442\u043e\u0432, \u0434\u0430\u043b\u044c\u0448\u0435. \u0422\u0435\u043c \u0441\u0430\u043c\u044b\u043c \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f, \u0438\u0435\u043c\u044e\u0449\u0438\u0435 \u043f\u043e\u0445\u043e\u0436\u0438\u0439 \u0441\u043c\u044b\u0441\u043b \u0431\u0443\u0434\u0443\u0442 \u0441\u0442\u043e\u044f\u0442\u044c \u0431\u043b\u0438\u0436 \u0434\u0440\u0443\u0433 \u043a \u0434\u0440\u0443\u0433\u0443, \u0430 \u0440\u0430\u0437\u043d\u044b\u0439, \u0431\u0443\u0434\u0443\u0442 \u043e\u0442\u0441\u0442\u043e\u044f\u0442\u044c \u0434\u0440\u0443\u0433 \u043e\u0442 \u0434\u0440\u0443\u0433\u0430.
  \u041f\u043e\u0434\u0440\u043e\u0431\u043d\u0435\u0435 \u043f\u0440\u043e Triplet loss \u0432\u043e\u0442 \u0442\u0443\u0442<\/a><\/p>\n
  <\/p>\n
7.1 Triplet loss \u043d\u0430 BOW<\/h3>\n
  <\/p>\n
class TripletDataGeneratorIndexes(BowDataGenerator):     def __init__(self, *args, **kwargs):         super().__init__(*args, **kwargs)         self.database = {}         for text, topic in self.texts_topics:             if topic not in self.database:                 self.database[topic] = []             self.database[topic].append(text)         # \u043f\u043e\u0447\u0438\u0441\u0442\u0438\u043c \u0432\u0441\u0435 \u0438\u043d\u0442\u0435\u043d\u0442\u044b \u0441 <5 \u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u044f\u043c\u0438          sh_database = {}         for topic in self.database:             if len(self.database[topic]) > 5:                 sh_database[topic] = self.database[topic]         self.database = sh_database          self.all_topics = [topic for topic in self.database]      def __iter__(self):         for _ in tqdm(range(self.batches_per_epoch), leave=False):             anchor = []             positive = []             negative = []              for _ in range(self.batch_size):                 anchor_topic = random.choice(self.all_topics)                 anchor_index = np.random.randint(len(self.database[anchor_topic]))                 positive_index = np.random.randint(len(self.database[anchor_topic]))                 while positive_index == anchor_index:                     positive_index = np.random.randint(len(self.database[anchor_topic]))                  negative_topic = random.choice(self.all_topics)                 while negative_topic == anchor_topic:                     negative_topic = random.choice(self.all_topics)                  negative_index = np.random.randint(len(self.database[negative_topic]))                  anchor.append(self.database[anchor_topic][anchor_index])                 positive.append(self.database[anchor_topic][positive_index])                 negative.append(self.database[negative_topic][negative_index])              yield self.vectorize(anchor), self.vectorize(positive), self.vectorize(negative)<\/code><\/pre>\n
  <\/p>\n
def models_builder(data_generator):     sentence_embeding_size = 100     def lossless_triplet_loss(y_true, y_pred, N=sentence_embeding_size, beta=100, epsilon=1e-8):         """         Implementation of the triplet loss function          Arguments:         y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.         y_pred -- python list containing three objects:                 anchor -- the encodings for the anchor data                 positive -- the encodings for the positive data (similar to anchor)                 negative -- the encodings for the negative data (different from anchor)         N  --  The number of dimension          beta -- The scaling factor, N is recommended         epsilon -- The Epsilon value to prevent ln(0)          Returns:         loss -- real number, value of the loss         """         anchor = tf.convert_to_tensor(y_pred[:,0:N])         positive = tf.convert_to_tensor(y_pred[:,N:N*2])          negative = tf.convert_to_tensor(y_pred[:,N*2:N*3])          # distance between the anchor and the positive         pos_dist = tf.reduce_sum(tf.square(tf.subtract(anchor,positive)),1)         # distance between the anchor and the negative         neg_dist = tf.reduce_sum(tf.square(tf.subtract(anchor,negative)),1)          #Non Linear Values           pos_dist = -tf.math.log(-tf.math.divide((pos_dist),beta)+1+epsilon)         neg_dist = -tf.math.log(-tf.math.divide((N-neg_dist),beta)+1+epsilon)          # compute loss         loss = neg_dist + pos_dist         return loss      def basic_sentence_vectorizer():         inp = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))         X = inp         X = Dense(complexity)(X)         X = Activation('elu')(X)         X = Dense(complexity)(X)         X = Activation('elu')(X)         X = Dense(complexity, name='embeding_output')(X)         X = Activation('elu')(X)         X = Dense(complexity)(X)         vectorizer = Model(inputs=inp, outputs=X)         return vectorizer      complexity = 300      inp_anchor = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))     inp_positive = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))     inp_negative = Input(shape=(len(data_generator.count_vectorizer.get_feature_names()),))      embedder = basic_sentence_vectorizer()      anchor = embedder(inp_anchor)     positive = embedder(inp_positive)     negative = embedder(inp_negative)      output = Concatenate(axis=1)([anchor, positive, negative])      model = Model(inputs=[inp_anchor, inp_positive, inp_negative], outputs=output)     model.compile(optimizer='adagrad', loss=lossless_triplet_loss)     model.summary()     return model, embedder  data_generator = TripletDataGeneratorIndexes(batch_size=128, batches_per_epoch=10000) model, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
zeros = np.zeros((data_generator.batch_size, 1, 1)) new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, 'triplet loss indexes')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i > 20:             break     for a, p, n in data_generator:         model.train_on_batch([a, p, n], zeros)<\/code><\/pre>\n
  <\/p>\n
0 724.1
  3 -143.5
  6 11.7
  9 36.2
  12 -123.5
  15 150.1
  18 -51.9
  21 5.0
  24 -43.5<\/p>\n
  <\/p>\n
7.2 Triplet loss \u043d\u0430 embedings<\/h3>\n
  <\/p>\n
class TripletDataGeneratorEmbedings(TripletDataGeneratorIndexes):     def __init__(self, *args, **kwargs):         super().__init__()         self.target_len = kwargs['target_len']         self.embedding_size = len(vectorize('any_token'))         self.use_word2vec = True         self.use_fasttext = True         self.batches_per_epoch = kwargs['batches_per_epoch']      def vectorize(self, sentences):         return LabelsDataGenerator.vectorize(self, sentences)<\/code><\/pre>\n
  <\/p>\n
def models_builder(data_generator):     sentence_embeding_size = 300     def lossless_triplet_loss(y_true, y_pred, N=sentence_embeding_size, beta=100, epsilon=1e-8):         """         Implementation of the triplet loss function          Arguments:         y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.         y_pred -- python list containing three objects:                 anchor -- the encodings for the anchor data                 positive -- the encodings for the positive data (similar to anchor)                 negative -- the encodings for the negative data (different from anchor)         N  --  The number of dimension         beta -- The scaling factor, N is recommended         epsilon -- The Epsilon value to prevent ln(0)          Returns:         loss -- real number, value of the loss         """         anchor = tf.convert_to_tensor(y_pred[:,0:N])         positive = tf.convert_to_tensor(y_pred[:,N:N*2])         negative = tf.convert_to_tensor(y_pred[:,N*2:N*3])          # distance between the anchor and the positive         pos_dist = tf.math.reduce_sum(tf.math.square(tf.math.subtract(anchor,positive)),1)         # distance between the anchor and the negative         neg_dist = tf.math.reduce_sum(tf.math.square(tf.math.subtract(anchor,negative)),1)          #Non Linear Values           pos_dist = -tf.math.log(-tf.math.divide((pos_dist),beta)+1+epsilon)         neg_dist = -tf.math.log(-tf.math.divide((N-neg_dist),beta)+1+epsilon)          # compute loss         loss = neg_dist + pos_dist          return loss      def inception_convolutional_layer(X, complexity, rate=0.2, regularizer=0):         X_7 = Conv1D(int(complexity\/7), kernel_size=7, strides=1, padding='same')(X)         X_6 = Conv1D(int(complexity\/6), kernel_size=6, strides=1, padding='same')(X)         X_5 = Conv1D(int(complexity\/5), kernel_size=5, strides=1, padding='same')(X)         X_4 = Conv1D(int(complexity\/4), kernel_size=4, strides=1, padding='same')(X)         X_3 = Conv1D(int(complexity\/3), kernel_size=3, strides=1, padding='same')(X)         X_2 = Conv1D(int(complexity\/2), kernel_size=2, strides=1, padding='same')(X)         X_1 = Conv1D(int(complexity\/1), kernel_size=1, strides=1, padding='same')(X)         X = Concatenate()([X_7, X_6, X_5, X_4, X_3, X_2, X_1])         X = Activation('elu')(X)         X = BatchNormalization()(X)         X = Dropout(rate)(X)         return X      def bi_LSTM(X, complexity, rate=0.2, regularizer=0):         X = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(int(complexity\/2), return_sequences=True))(X)         X = tf.keras.layers.BatchNormalization()(X)         X = tf.keras.layers.Dropout(rate)(X)         return X      def dense_layer(X, complexity, rate=0.2, regularizer=0):         X = tf.keras.layers.Dense(int(complexity))(X)         X = tf.keras.layers.Activation('elu')(X)         X = tf.keras.layers.BatchNormalization()(X)         X = tf.keras.layers.Dropout(rate)(X)         return X      def basic_sentence_vectorizer():         rate = 0.20         complexity = 300         inp = Input(shape = (data_generator.target_len, data_generator.embedding_size))          X = inp         X = inception_convolutional_layer(X, complexity)         X = inception_convolutional_layer(X, complexity)         X = inception_convolutional_layer(X, complexity)         X = tf.keras.layers.MaxPooling1D(pool_size=2)(X)         X = inception_convolutional_layer(X, complexity)         X = tf.keras.layers.MaxPooling1D(pool_size=2)(X)         X = inception_convolutional_layer(X, complexity)         X = tf.keras.layers.MaxPooling1D(pool_size=2)(X)          R = inp         R = bi_LSTM(R, complexity)         R = bi_LSTM(R, complexity\/2)         attention_probs = tf.keras.layers.Dense(int(complexity\/2), activation='sigmoid', name='attention_probs')(R)         R = multiply([R, attention_probs], name='attention_mul')         R = Dropout(rate)(R)         R = MaxPooling1D(pool_size=2)(R)         R = inception_convolutional_layer(R, complexity)         R = MaxPooling1D(pool_size=2)(R)         R = inception_convolutional_layer(R, complexity)         R = MaxPooling1D(pool_size=2)(R)          X = Concatenate(axis=-1)([X, R])         X = Flatten()(X)         X = BatchNormalization()(X)         X = Dropout(rate)(X)          X = dense_layer(X, complexity)         X = dense_layer(X, complexity)         X = dense_layer(X, complexity)          X = Dense(sentence_embeding_size, activation='sigmoid')(X)         vectorizer = Model(inputs=inp, outputs=X)         return vectorizer      inp_anchor = Input(shape = (data_generator.target_len, data_generator.embedding_size))     inp_positive = Input(shape = (data_generator.target_len, data_generator.embedding_size))     inp_negative = Input(shape = (data_generator.target_len, data_generator.embedding_size))      embedder = basic_sentence_vectorizer()      anchor = embedder(inp_anchor)     positive = embedder(inp_positive)     negative = embedder(inp_negative)      output = Concatenate(axis=1)([anchor, positive, negative])      model = Model(inputs=[inp_anchor, inp_positive, inp_negative], outputs=output)     model.compile(optimizer='adagrad', loss=lossless_triplet_loss)     model.summary()     return model, embedder  data_generator = TripletDataGeneratorEmbedings(target_len=20, batch_size=32, batches_per_epoch=10000) model, embedder = models_builder(data_generator) get_similarity_values = similarity_values_wrapper(embedder.predict, data_generator.vectorize)<\/code><\/pre>\n
  <\/p>\n
zeros = np.zeros((data_generator.batch_size, 1, 1)) new_result = -10e5 for i in tqdm(range(1000)):     if i%3==0:         previous_result = new_result         new_result = evaluate(get_similarity_values, 'triplet loss embeding')         new_result = parse_result(new_result)         print(i, new_result)         if new_result < previous_result and i>20:             break     for a, p, n in data_generator:         model.train_on_batch([a, p, n], zeros)<\/code><\/pre>\n
  <\/p>\n
0 283.9
  3 334.2
  6 218.1
  9 219.6
  12 262.8
  15 282.4
  18 289.7
  21 274.9<\/p>\n
  <\/p>\n
plot_results()<\/code><\/pre>\n
  <\/p>\n
\"png\"\/<\/p>\n
  <\/p>\n
\u0418\u0442\u043e\u0433\u0438<\/h1>\n
  <\/p>\n
\u041c\u043e\u0436\u043d\u043e \u0431\u044b\u043b\u043e \u043f\u0440\u0435\u0434\u0441\u043a\u0430\u0437\u0430\u0442\u044c, \u0447\u0442\u043e \u043f\u043e\u0431\u0435\u0434\u0438\u0442\u0435\u043b\u044f\u043c\u0438 \u0431\u0443\u0434\u0443\u0442 \u043c\u043e\u0434\u0435\u043b\u0438 ELMO \u0442.\u043a. \u043e\u043d\u0438 \u0431\u044b\u043b\u0438 \u0441\u043e\u0437\u0434\u0430\u043d\u044b \u0434\u043b\u044f \u0432\u0435\u043a\u0442\u043e\u0440\u0438\u0437\u0430\u0446\u0438\u0438 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0439. \u0418\u0445 \u043c\u043e\u0436\u043d\u043e \u0441\u043c\u0435\u043b\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c, \u043a\u043e\u0433\u0434\u0430 \u0432\u0430\u043c \u043d\u0443\u0436\u043d\u043e \u0431\u044b\u0441\u0442\u0440\u043e \u0438\u0437\u0432\u043b\u0435\u0447\u044c \u0444\u0438\u0447\u0438 \u0438\u0437 \u0442\u0435\u043a\u0441\u0442\u0430.
  \u041b\u0438\u0447\u043d\u043e \u043c\u0435\u043d\u044f \u043f\u0440\u0438\u044f\u0442\u043d\u043e \u0443\u0434\u0438\u0432\u0438\u043b BOW \u0438 \u0441\u0440\u0435\u0434\u043d\u0435\u0435 \u043f\u043e \u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u0430\u043c. \u0414\u0430\u0436\u0435 \u0431\u0435\u0437 \u0443\u0447\u0451\u0442\u0430 \u043f\u043e\u0440\u044f\u0434\u043a\u0430 \u0441\u043b\u043e\u0432, \u043e\u043d\u0438 \u0441\u043c\u043e\u0433\u043b\u0438 \u043f\u043e\u0441\u0442\u0430\u0432\u0438\u0442\u044c \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f \u0438\u0437 \u043e\u0434\u043d\u043e\u0439 \u0442\u0435\u043c\u044b \u0440\u044f\u0434\u043e\u043c.
  \u0411\u044b\u043b \u0440\u0430\u0437\u043e\u0447\u0430\u0440\u043e\u0432\u0430\u043d \u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440\u0430\u043c\u0438. \u0421\u0440\u0430\u0437\u0443 \u043f\u043e\u0441\u043b\u0435 \u0438\u043d\u0438\u0446\u0438\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442 \u043b\u0443\u0447\u0448\u0435, \u0447\u0435\u043c \u043f\u043e\u0441\u043b\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f. \u041d\u0435 \u043c\u043e\u0433\u0443 \u0441\u043a\u0430\u0437\u0430\u0442\u044c \u0432 \u0447\u0451\u043c \u043f\u0440\u043e\u0431\u043b\u0435\u043c\u0430, \u0441\u043a\u043e\u0440\u0435\u0435 \u0432\u0441\u0435\u0433\u043e \u0430\u0432\u0442\u043e\u044d\u043d\u043a\u043e\u0434\u0435\u0440 \u043d\u0435 \u043c\u043e\u0436\u0435\u0442 \u0441\u0436\u0430\u0442\u044c \u0432\u0441\u0451 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u043e \u0438 \u043d\u0430\u0447\u0438\u043d\u0430\u0435\u0442 \u043f\u0440\u0435\u0434\u0441\u043a\u0430\u0437\u044b\u0432\u0430\u0442\u044c \u043d\u0443\u043b\u0438. \u0415\u0441\u043b\u0438 \u0443 \u0432\u0430\u0441 \u0431\u0443\u0434\u0443\u0442 \u0438\u0434\u0435\u0438 \u043f\u043e \u0443\u043b\u0443\u0447\u0448\u0435\u043d\u0438\u044e, \u0442\u043e \u0436\u0434\u0443 \u0432 \u043a\u043e\u043c\u043c\u0435\u043d\u0442\u0430\u0440\u0438\u044f\u0445.
  \u041c\u043e\u0439 \u043b\u0438\u0447\u043d\u044b\u0439 \u0444\u0430\u0432\u043e\u0440\u0438\u0442 Triplet loss \u043d\u0430 embedings \u0442\u043e\u0436\u0435 \u043d\u0435 \u0434\u0430\u043b \u0432\u044b\u0434\u0430\u044e\u0449\u0435\u0433\u043e\u0441\u044f \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430. \u0414\u0443\u043c\u0430\u044e, \u0447\u0442\u043e \u043e\u043d \u0440\u0430\u0441\u043a\u0440\u043e\u0435\u0442 \u0441\u0432\u043e\u0439 \u043f\u043e\u0442\u0435\u043d\u0446\u0438\u0430\u043b \u043d\u0430 \u043c\u043e\u0434\u0435\u043b\u044f\u0445 \u0432 100 \u0440\u0430\u0437 \u0431\u043e\u043b\u044c\u0448\u0435 \u043f\u043e \u0440\u0430\u0437\u043c\u0435\u0440\u0443 \u0438 \u0441 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435\u043c \u0432 \u0442\u0435\u0447\u0435\u043d\u0438\u0438 \u043d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u0438\u0445 \u043c\u0435\u0441\u044f\u0446\u0435\u0432.
  \u0414\u0432\u0430 \u043c\u0435\u0442\u043e\u0434\u0430: BOW \u0441 \u043b\u0435\u043c\u043c\u0430\u043c\u0438 \u0431\u0435\u0437 \u0441\u0442\u043e\u043f \u0441\u043b\u043e\u0432 \u0438 \u0441\u0440\u0435\u0434\u043d\u0435\u0435 \u0441 \u0432\u0435\u0441\u0430\u043c\u0438 tf-idf \u0445\u043e\u0442\u044c \u0438 \u043d\u0435 \u0434\u0430\u044e\u0442 \u0432\u044b\u0434\u0430\u044e\u0449\u0438\u0445\u0441\u044f \u0441\u0440\u0435\u0434\u043d\u0438\u0445 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u0432, \u043d\u043e \u0434\u043b\u044f \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0445 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0439 \u0434\u0430\u044e\u0442 \u043e\u0447\u0435\u043d\u044c \u0438 \u043e\u0447\u0435\u043d\u044c \u0445\u043e\u0440\u043e\u0448\u0438\u0439 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442. \u041f\u043e\u044d\u0442\u043e\u043c\u0443, \u0434\u043b\u044f \u044d\u0442\u0438\u0445 \u043c\u0435\u0442\u043e\u0434\u043e\u0432, \u0432\u0441\u0451 \u0434\u043e\u043b\u0436\u043d\u043e \u0437\u0430\u0432\u0438\u0441\u0435\u0442\u044c \u043e\u0442 \u0434\u0430\u043d\u043d\u044b\u0445.
  \u0412\u0435\u0440\u043e\u044f\u0442\u043d\u043e, \u0447\u0442\u043e \u0441\u043e \u0432\u0440\u0435\u043c\u0435\u043d\u0435\u043c \u0431\u0443\u0434\u0435\u0442 \u0438 \u0427\u0430\u0441\u0442\u044c 3, \u0435\u0441\u043b\u0438 \u043d\u0430\u0431\u0435\u0440\u0443 \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e\u0435 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e \u0438\u0434\u0435\u0439.<\/p>\n<\/div>\n
 \u0441\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 \u043e\u0440\u0438\u0433\u0438\u043d\u0430\u043b \u0441\u0442\u0430\u0442\u044c\u0438  https:\/\/habr.com\/ru\/post\/515084\/<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"\n
\n
\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u0439\u0442\u0435, \u043f\u0440\u043e\u0434\u043e\u043b\u0436\u0435\u043d\u0438\u0435 \u0441\u0442\u0430\u0442\u044c\u0438 \u043f\u0440\u043e \u043c\u0435\u0442\u043e\u0434\u044b \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u044d\u043c\u0431\u0435\u0434\u0438\u043d\u0433\u043e\u0432 \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u0439. \u0412 \u044d\u0442\u043e\u043c \u0433\u0430\u0439\u0434\u0435 \u043c\u0430\u043b\u043e \u0441\u043b\u043e\u0432 \u0438 \u043c\u043d\u043e\u0433\u043e \u043a\u043e\u0434\u0430, \u0433\u043e\u0442\u043e\u0432\u043e \u0434\u043b\u044f Ctrl+\u0441, Ctrl+v \u0434\u043b\u044f \u0443\u0434\u0443\u0447\u0448\u0435\u043d\u0438\u0439 \u0438 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0438\u0445 \u0442\u0435\u0441\u0442\u043e\u0432.
  \u0427\u0430\u0441\u0442\u044c1<\/a> \u043e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c\u043d\u0430 \u0434\u043b\u044f \u043e\u0437\u043d\u0430\u043a\u043e\u043c\u043b\u0435\u043d\u0438\u044f<\/p>\n
  <\/p>\n
4. BERT<\/h1>\n
  <\/p>\n
from deeppavlov.core.common.file import read_json from deeppavlov import build_model, configs from deeppavlov.models.embedders.elmo_embedder import ELMoEmbedder # \u0441\u0441\u044b\u043b\u043a\u0430 \u0434\u043b\u044f \u0441\u043a\u0430\u0447\u0438\u0432\u0430\u043d\u0438\u044f \u043c\u043e\u0434\u0435\u043b\u0435\u0439 http:\/\/docs.deeppavlov.ai\/en\/master\/features\/pretrained_vectors.html<\/code><\/pre>\n
  <\/p>\n
4.1 rubert_cased_L-12_H-768_A-12_pt<\/h3>\n
  <\/p>\n
class RU_BERT_CLASS:     def __init__(self, name):         bert_config = read_json(configs.embedder.bert_embedder)         bert_config['metadata']['variables']['BERT_PATH'] = os.path.join('.\/.', name)         self.m = build_model(bert_config)      def vectorizer(self, sentences):         return [sentence.split() for sentence in sentences]      def predict(self, tokens):         _, _, _, _, sent_max_embs, sent_mean_embs, _ = self.m(tokens)         return sent_mean_embs  bert = RU_BERT_CLASS('rubert_cased_L-12_H-768_A-12_pt') get_similarity_values = similarity_values_wrapper(bert.predict, bert.vectorizer, distance_function=cosine_distances) evaluate(get_similarity_values, 'rubert')<\/code><\/pre>\n
  <\/p>\n
‘rubert: 2895.7’<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"class_list":["post-308444","post","type-post","status-publish","format-standard","hentry"],"_links":{"self":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/308444","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=308444"}],"version-history":[{"count":0,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/308444\/revisions"}],"wp:attachment":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=308444"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=308444"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=308444"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}