纯文本
整个文本只有一行,无换行,字之间空格隔开app
方法一:torchtext
任务:构造语言模型数据集,返回的单个数据类型以下,target为inputs的偏移。函数
inputs:[A B C D E F] target:[B C D E F G]
为此咱们会使用LanguageModelingDataset创建数据集,而后使用BPTTIterator建立迭代器。ui
注意:若是文本数太小,且BPTTIterator中设置的batch_size * bptt_len大于文本总长度,则生成的batch的seq_len达不到bptt_len。this
若是处理中文,tokenize函数能够使用jieba进行分词:spa
tokenize = lambda x: jieba.lcut(x)
import torchtext import torch device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') BATCH_SIZE = 32 MAX_VOCAB_SIZE = 50000 tokenize = lambda x: x.split() """ 定义TEXT field用于处理文本的方法 sequential: Whether the datatype represents sequential data. If False, no tokenization is applied. Default: True. use_vocab: Whether to use a Vocab object. If False, the data in this field should already be numerical. Default: True. tokenize: The function used to tokenize strings using this field into sequential examples. Default: string.split. """ TEXT = torchtext.data.Field(sequential=True, use_vocab=True, tokenize=tokenize, lower=True, batch_first=True, init_token=None, eos_token=None) """ LanguageModelingDataset.split() 处理纯文本数据,分词方法直接使用str.split() """ train, val, test = torchtext.datasets.LanguageModelingDataset.splits(path="data", train="text8.train.txt", validation="text8.dev.txt", test="text8.test.txt", text_field=TEXT) # 只有一条数据 result=1 每条数据用一个字典表示 print('total example row = ', len(train)) # 打印第一条数据的keys值 result='result' print(train[0].__dict__.keys()) # 打印第一条数据的values值 result='result' # print(train[0].__dict__.values()) # create vocabulary TEXT.build_vocab(train, max_size=MAX_VOCAB_SIZE) VOCAB_SIZE = len(TEXT.vocab) print("vocabulary size: ", VOCAB_SIZE) print(TEXT.vocab.itos[:10]) print(TEXT.vocab.stoi['apple']) print('<BOS> indx is ', TEXT.vocab.stoi['<BOS>']) print('<EOS> indx is ', TEXT.vocab.stoi['<EOS>']) UNK_STR = TEXT.unk_token PAD_STR = TEXT.pad_token UNK_IDX = TEXT.vocab.stoi[UNK_STR] PAD_IDX = TEXT.vocab.stoi[PAD_STR] print(f'{UNK_STR} index is {UNK_IDX}') print(f'{PAD_STR} index is {PAD_IDX}') """ Defines an iterator for language modeling tasks that use BPTT. bptt: Length of sequences for backpropagation through time repeat: Whether to repeat the iterator for multiple epochs. Default: False. """ train_iter, val_iter, test_iter = torchtext.data.BPTTIterator.splits((train, val, test), batch_size=BATCH_SIZE, device=device, bptt_len=50, repeat=False, shuffle=True) for batch in train_iter: print(batch.text.shape) # (batch=32, seqlen=50) print(batch.target.shape) # (batch=32, seqlen=50) print(" ".join(TEXT.vocab.itos[i] for i in batch.text[-1, :].data.cpu())) print(" ".join(TEXT.vocab.itos[i] for i in batch.target[-1, :].data.cpu())) break
方法二:torch.utils.data
任务:构造一个word2vec的skip-gram数据, 返回的单个数据类型以下:code
[center_word, pos_words, neg_words] center_word.shape: (batchSize) pos_words.shape: (batchSize, 2*C) neg_words.shape: (batchSize, 2*C*K)
能够使用torch.utils.data的Dataset和DataLoader进行数据的处理。blog
import torch from torch.utils.data import Dataset, DataLoader from collections import Counter import numpy as np K = 100 # number of negative samples C = 3 # nearby words threshold MAX_VOCAB_SIZE = 30000 # the vocabulary size BATCH_SIZE = 128 # the batch size train_text = open('data/text8.train.txt', 'r').read() # val_text = open('data/text8.dev.txt', 'r').read() # test_text = open('data/text8.test.txt', 'r').read() text = [word for word in train_text.split()] vocab = dict(Counter(text).most_common(MAX_VOCAB_SIZE-1)) vocab["<unk>"] = len(text) - np.sum(list(vocab.values())) idx_to_word = [word for word in vocab.keys()] word_to_idx = {word: i for i, word in enumerate(idx_to_word)} """ 统计词典中词出现的频率 """ # 获取单词出现的个数 word_counts = np.array([count for count in vocab.values()], dtype=np.float32) # 计算频率 word_freqs = word_counts / np.sum(word_counts) # 0.75 次幂 word_freqs = word_freqs ** (3./4.) # 归一化 word_freqs = word_freqs / np.sum(word_freqs) # 用来作 negative sampling VOCAB_SIZE = len(idx_to_word) class WordEmbeddingDataset(torch.utils.data.Dataset): def __init__(self, text, word_to_idx, idx_to_word, word_freqs, word_counts): super(WordEmbeddingDataset, self).__init__() # 将单词转换成数字索引 self.text_encoded = [word_to_idx.get(t, VOCAB_SIZE - 1) for t in text] self.text_encoded = torch.Tensor(self.text_encoded).long() # dict:word->index self.word_to_idx = word_to_idx # list: index->word self.idx_to_word = idx_to_word # 单词频率 self.word_freqs = torch.Tensor(word_freqs) # 单词次数统计 self.word_counts = torch.Tensor(word_counts) def __len__(self): return len(self.text_encoded) def __getitem__(self, idx): # 中心词 center_word = self.text_encoded[idx] # 周边词 pos_indices = list(range(idx - C, idx)) + list(range(idx + 1, idx + C + 1)) pos_indices = [i % len(self.text_encoded) for i in pos_indices] # 正采样 pos_words = self.text_encoded[pos_indices] # 负采样 neg_words = torch.multinomial(self.word_freqs, K * pos_words.shape[0], True) return center_word, pos_words, neg_words dataset = WordEmbeddingDataset(text, word_to_idx, idx_to_word, word_freqs, word_counts) dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)