Aug-17-2021, 07:54 AM
I have to test word2vector model for text data similarity it generate this kind of error ValueError: dimension mismatch
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
tokenizer=Tokenizer()
tokenizer.fit_on_texts(documents_df.documents_cleaned)
tokenized_documents=tokenizer.texts_to_sequences(documents_df.documents_cleaned)
tokenized_paded_documents=pad_sequences(tokenized_documents,maxlen=64,padding='post')
vocab_size=len(tokenizer.word_index)+1
# reading Glove word embeddings into a dictionary with "word" as key and values as word vectors
embeddings_index = dict()
with open("D:\Clustering\glove.6B.100d.txt", 'r', encoding="utf8") as file:
for line in file:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
# creating embedding matrix, every row is a vector representation from the vocabulary indexed by the tokenizer index.
embedding_matrix=np.zeros((vocab_size,100))
for word,i in tokenizer.word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
# calculating average of word vectors of a document weighted by tf-idf
document_embeddings=np.zeros((len(tokenized_paded_documents),100))
words=tfidfvectoriser.get_feature_names()
# instead of creating document-word embeddings, directly creating document embeddings
for i in range(documents_df.shape[0]):
for j in range(len(words)):
document_embeddings[i]+=embedding_matrix[tokenizer.word_index[words[j]]]*tfidf_vectors[i][j]
pairwise_similarities=cosine_similarity(document_embeddings)
pairwise_differences=euclidean_distances(document_embeddings)Error: Error:Traceback (most recent call last):
File "D:/Clustering/text-cluster-master/Cos_Sim_Eucliden_distance.py", line 156, in <module>
document_embeddings[i] += embedding_matrix[tokenizer.word_index[words[j]]] * tfidf_vectors[i][j]
File "D:\Python3.8.0\Python\lib\site-packages\scipy\sparse\base.py", line 550, in __rmul__
return (self.transpose() * tr).transpose()
File "D:\Python3.8.0\Python\lib\site-packages\scipy\sparse\base.py", line 498, in __mul__
raise ValueError('dimension mismatch')
ValueError: dimension mismatch
