!onmt_preprocess -train_src "/content/drive/My Drive/Bachelor/baseline-1M-enfr/baseline-1M_train.en" -train_tgt "/content/drive/My Drive/Bachelor/baseline-1M-enfr/baseline-1M_train.fr" -valid_src "/content/drive/My Drive/Bachelor/baseline-1M-enfr/baseline-1M_test.en" -valid_tgt "/content/drive/My Drive/Bachelor/baseline-1M-enfr/baseline-1M_test.fr" -save_data "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data" -src_vocab_size 10000 -tgt_vocab_size 10000
import torch.nn as nn
import torch
import onmt
import onmt.inputters
import onmt.modules
import onmt.utils
import logging
logging.basicConfig(level=logging.NOTSET)
"""
We begin by loading in the vocabulary for the model of interest.
This will let us check vocab size and to get the special ids for padding
"""
vocab_fields = torch.load("/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.vocab.pt") #vocab field 만들기
src_text_field = vocab_fields["src"].base_field #src field 만들기
src_vocab = src_text_field.vocab
src_padding = src_vocab.stoi[src_text_field.pad_token] #Padding
tgt_text_field = vocab_fields['tgt'].base_field #tgt field 만들기
tgt_vocab = tgt_text_field.vocab
tgt_padding = tgt_vocab.stoi[tgt_text_field.pad_token] #Padding
"""
Next we specify the core model itself.
Here we will build a small model with an encoder and an attention based input feeding decoder.
Both models will be RNNs and the encoder will be bidirectional
"""
emb_size = 100
rnn_size = 500
# Specify the core model.
encoder_embeddings = onmt.modules.Embeddings(emb_size, len(src_vocab),
word_padding_idx=src_padding ) #src 임베딩
encoder = onmt.encoders.RNNEncoder(hidden_size=rnn_size, num_layers=1,
rnn_type="LSTM", bidirectional=True,
embeddings=encoder_embeddings) #인코더
decoder_embeddings = onmt.modules.Embeddings(emb_size, len(tgt_vocab),
word_padding_idx=tgt_padding) #tgt 임베딩
decoder = onmt.decoders.decoder.InputFeedRNNDecoder(
hidden_size=rnn_size, num_layers=1, bidirectional_encoder=True,
rnn_type="LSTM", embeddings=decoder_embeddings) #Decoder
device = "cuda" if torch.cuda.is_available() else "cpu" #Device 설정
model = onmt.models.model.NMTModel(encoder, decoder) #모델 생성
model.to(device)#디바이스 올리기
# modelSaver = onmt.models.ModelSaver()
# Specify the tgt word generator and loss computation module
model.generator = nn.Sequential(
nn.Linear(rnn_size, len(tgt_vocab)),
nn.LogSoftmax(dim=-1)).to(device) #제너레이터 부분
loss = onmt.utils.loss.NMTLossCompute(
criterion=nn.NLLLoss(ignore_index=tgt_padding, reduction="sum"),
generator=model.generator) #Loss 설정
"""
Now we set up the optimizer.
Our wrapper around a core torch optim class handles learning rate updates and gradient normalization automatically.
"""
lr = 1
torch_optimizer = torch.optim.SGD(model.parameters(), lr=lr) #옵티마이져
optim = onmt.utils.optimizers.Optimizer(
torch_optimizer, learning_rate=lr, max_grad_norm=2) #옵티마이저
"""
Now we load the data from disk with the associated vocab fields.
To iterate through the data itself we use a wrapper around a torchtext iterator class.
We specify one for both the training and test data.
"""
# Load some data
from itertools import chain
train_data_file0 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.0.pt" #학습데이터
train_data_file1 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.1.pt" #학습데이터
train_data_file2 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.2.pt" #학습데이터
train_data_file3 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.3.pt" #학습데이터
train_data_file4 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.4.pt" #학습데이터
train_data_file5 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.5.pt" #학습데이터
train_data_file6 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.6.pt" #학습데이터
train_data_file7 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.7.pt" #학습데이터
train_data_file8 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.8.pt" #학습데이터
train_data_file9 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.9.pt" #학습데이터
train_data_file10 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.10.pt" #학습데이터
train_data_file11 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.11.pt" #학습데이터
train_data_file12 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.12.pt" #학습데이터
train_data_file13 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.13.pt" #학습데이터
train_data_file14 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.14.pt" #학습데이터
train_data_file15 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.15.pt" #학습데이터
train_data_file16 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.16.pt" #학습데이터
train_data_file17 = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.train.17.pt" #학습데이터
valid_data_file = "/content/drive/My Drive/Bachelor/baseline-1M-enfr/data.valid.0.pt" #검증데이터
train_iter = onmt.inputters.inputter.DatasetLazyIter(dataset_paths=[train_data_file0,
train_data_file1,
train_data_file2,
train_data_file3,
train_data_file4,
train_data_file5,
train_data_file6,
train_data_file7,
train_data_file8,
train_data_file9,
train_data_file10,
train_data_file11,
train_data_file12,
train_data_file13,
train_data_file14,
train_data_file15,
train_data_file16,
train_data_file17,
],
fields=vocab_fields,
batch_size=50,
batch_size_multiple=1,
batch_size_fn=None,
device=device,
is_train=True,
repeat=True,
pool_factor=True#기존 소스 코드 변경
) #iterator 만들기
valid_iter = onmt.inputters.inputter.DatasetLazyIter(dataset_paths=[valid_data_file],
fields=vocab_fields,
batch_size=10,
batch_size_multiple=1,
batch_size_fn=None,
device=device,
is_train=False,
repeat=False,
pool_factor=True
)
"""
Finally we train. Keeping track of the output requires a report manager.
"""
report_manager = onmt.utils.ReportMgr(
report_every=500, start_time=None, tensorboard_writer=None) #리포트 매니저
model_saver = onmt.models.ModelSaver("/content/drive/My Drive/Bachelor/baseline-1M-enfr/checkpoint",model=model,optim=optim,keep_checkpoint=2, fields=vocab_fields,model_opt="text")
trainer = onmt.Trainer(model=model,
train_loss=loss,
valid_loss=loss,
model_saver=model_saver,
optim=optim,
report_manager=report_manager) #트레이너 생성
trainer.train(train_iter=train_iter,
train_steps=7000,
save_checkpoint_steps=1000,
valid_iter=valid_iter,
valid_steps=250) #실제 학습진행
I need to restore from the latest checkpoint which is 7000 train step to continue to 11000 train step , how can I do it ?