1.Word2Vec简介

Word2Vec也称Word Embeddings，中文比较常见的叫法是“词向量”或者是“词嵌入”。通俗的来说就是把单词进行编码，变成数字的形式让计算机知道那个单词的代号。哈哈， 感觉就像以前的间谍通过莫尔斯电码进行信息传递，只不过那个是通过声音的长短进行编码，我们如果使用one-hot的编码方式，比如I是第一个单词，那么在维度为10的单词向量中，编码就是[1, 0, 0, 0, 0, 0, 0, 0, 0, 0]。通过这种方式我们进行编码，可以发现如果把一片文章进行编码，那么肯定是非常稀疏的，因为一个单词对应的向量只有一个有效的位置。而且还会存在一个问题，通常我们对单词的编码都是随机的，那么如果以“江苏”和“南京”为例，那么那样的编码关系不会存在地理联系，即江苏的省会城市是南京，这种包含关系是没办法体现的。
使用向量表达就可以有效地解决这个问题。它可以从原始语料中学习字词空间向量的预测模型。主要分为CBOW（Continuous Bag of Words）和Skip-Gram两种模式。在本篇博客中，我们主要使用的是Skip-Gram的模式。

2.Word2Vec的代码实现

#coding:utf-8#因为要下载数据，所以导入的依赖库比较多importcollectionsimportmathimportosimportrandomimportzipfileimportnumpyasnpimportsysimporttensorflowastffromsklearn.manifoldimportTSNEimportmatplotlib.pyplotasplt#这边是python版本的一个检查，不同版本对应函数调用的接口是不一样的ifsys.version_info[0]>=3:fromurllib.requestimporturlretrieveelse:fromurllibimporturlretrieve#从网址下载数据并检查数据的准确性url='http://mattmahoney.net/dc/'defmaybe_download(filename,excepted_bytes):ifnotos.path.exists(filename):filename,_=urlretrieve(url+filename,filename)statinfo=os.stat(filename)ifstatinfo.st_size==excepted_bytes:print("Found and verified",filename)else:print(statinfo.st_size)raiseException("Failed to verfy"+filename+"Can you get to it with browser?")returnfilename filename=maybe_download('text8.zip',31344016)#定义读取数据的函数，并把数据转成列表defread_data(filename):withzipfile.ZipFile(filename)asf:data=tf.compat.as_str(f.read(f.namelist()[0])).split()returndata words=read_data(filename)print('Data size',len(words))#创建词汇表，选取前50000频数的单词，其余单词认定为Unknown,编号为0vocabulary_size=50000defbuild_dataset(words):count=[['UNK',-1]]count.extend(collections.Counter(words).most_common(vocabulary_size-1))dictionary=dict()forword,_incount:dictionary[word]=len(dictionary)data=list()unk_count=0forwordinwords:ifwordindictionary:index=dictionary[word]else:index=0unk_count+=1data.append(index)count[0][1]=unk_count reverse_dictionary=dict(zip(dictionary.values(),dictionary.keys()))returndata,count,dictionary,reverse_dictionary data,count,dictionary,reverse_dictionary=build_dataset(words)#为了节约内存删除原始单词列表，打印出最高频出现的词汇及其数量delwordsprint('Most common words (+UNK)',count[:5])print('Sample data',data[:10],[reverse_dictionary[i]foriindata[:10]])#生成训练样本，assert断言：申明其布尔值必须为真的判定，如果发生异常，就表示为假data_index=0defgenerate_batch(batch_size,num_skips,skip_window):globaldata_indexassertbatch_size%num_skips==0assertnum_skips<=2*skip_window batch=np.ndarray(shape=(batch_size),dtype=np.int32)labels=np.ndarray(shape=(batch_size,1),dtype=np.int32)span=2*skip_window+1buffer=collections.deque(maxlen=span)for_inrange(span):buffer.append(data[data_index])data_index=(data_index+1)%len(data)foriinrange(batch_size//num_skips):target=skip_window targets_to_avoid=[skip_window]forjinrange(num_skips):whiletargetintargets_to_avoid:target=random.randint(0,span-1)targets_to_avoid.append(target)batch[i*num_skips+j]=buffer[skip_window]labels[i*num_skips+j,0]=buffer[target]buffer.append(data[data_index])data_index=(data_index+1)%len(data)returnbatch,labels#调用generate_batch函数简单测试一下功能batch,labels=generate_batch(batch_size=8,num_skips=2,skip_window=1)foriinrange(8):print(batch[i],reverse_dictionary[batch[i]],'->',labels[i,0],reverse_dictionary[labels[i,0]])#定义训练是的参数batch_size=128embedding_size=128skip_window=1num_skips=2valid_size=16valid_window=100valid_examples=np.random.choice(valid_window,valid_size,replace=False)num_sampled=64#定义Skip-Gram Word2Vec模型的网络结构graph=tf.Graph()withgraph.as_default():train_inputs=tf.placeholder(tf.int32,shape=[batch_size])train_labels=tf.placeholder(tf.int32,shape=[batch_size,1])valid_dataset=tf.constant(valid_examples,dtype=tf.int32)withtf.device('/gpu:0'):embeddings=tf.Variable(tf.random_uniform([vocabulary_size,embedding_size],-1.0,1.0))embed=tf.nn.embedding_lookup(embeddings,train_inputs)nce_weights=tf.Variable(tf.truncated_normal([vocabulary_size,embedding_size],stddev=1.0/math.sqrt(embedding_size)))nce_biases=tf.Variable(tf.zeros([vocabulary_size]))loss=tf.reduce_mean(tf.nn.nce_loss(weights=nce_weights,biases=nce_biases,labels=train_labels,inputs=embed,num_sampled=num_sampled,num_classes=vocabulary_size))optimizer=tf.train.GradientDescentOptimizer(1.0).minimize(loss)norm=tf.sqrt(tf.reduce_sum(tf.square(embeddings),1,keep_dims=True))normalized_embeddings=embeddings/norm valid_embeddings=tf.nn.embedding_lookup(normalized_embeddings,valid_dataset)similarity=tf.matmul(valid_embeddings,normalized_embeddings,transpose_b=True)init=tf.global_variables_initializer()#定义最大迭代次数，创建并设置默认的sessionnum_steps=100001withtf.Session(graph=graph)assession:init.run()print("Initialized")average_loss=0forstepinrange(num_steps):batch_inputs,batch_labels=generate_batch(batch_size,num_skips,skip_window)feed_dict={train_inputs:batch_inputs,train_labels:batch_labels}_,loss_val=session.run([optimizer,loss],feed_dict=feed_dict)average_loss+=loss_valifstep%2000==0:ifstep>0:average_loss/=2000print("Average loss at step ",step,":",average_loss)average_loss=0ifstep%10000==0:sim=similarity.eval()foriinrange(valid_size):valid_word=reverse_dictionary[valid_examples[i]]top_k=8nearest=(-sim[i,:]).argsort()[1:top_k+1]log_str="Nearest to %s:"%valid_wordforkinrange(top_k):close_woreverse_dictionary[nearest[k]]log_str="%s %s,"%(log_str,close_word)print(log_str)final_embeddings=normalized_embeddings.eval()#定义可视化Word2Vec效果的函数defplot_with_labels(low_dim_embs,labels,filename='tsne.png'):assertlow_dim_embs.shape[0]>=len(labels),"More labels than embeddings"plt.figure(figsize=(18,18))fori,labelinenumerate(labels):x,y=low_dim_embs[i,:]plt.scatter(x,y)plt.annotate(label,xy=(x,y),xy=(5,2),textcoords='offset points',ha='right',va='bottom')plt.savefig(filename)tsne=TSNE(perplexity=30,n_components=2,init='pca',n_iter=5000)plo t_only=100low_dim_embs=tsne.fit_transform(final_embeddings[:plot_only,:])labels=[reverse_dictionary[i]foriinrange(plot_only)]plot_with_labels(low_dim_embs,labels)

LZ其实对自然语言处理也不是很懂，所以NLP的部分也就只能浅尝辄止啦O(∩_∩)O