Show Code
import torch
import torch.nn as nn
import random
import re
from collections import CounterMachine Translation using Seq2Seq Models
In this experiment, we will build a Seq2seq archeticture to translate short English sentences into Arabic. We will wire up an encoder and a decoder and watch them learn to translate together.
The idea is simple. We feed an English sentence like “Ahmed loves reading” into the encoder, which reads it and squeezes its meaning into a small context vector. We then hand that summary to the decoder, which uses it to produce the Arabic sentence “أحمد يحب القراءة” one token at a time.
A new concept in training called teacher forcing will be introduced and explained, and we will disscuss how it can make the training stable.
Imports and data preparations
Show Code
pairs = [
# greetings & common phrases
("hello", "مرحبا"),
("goodbye", "مع السلامة"),
("thank you", "شكرا لك"),
("good morning", "صباح الخير"),
("good night", "تصبح على خير"),
("welcome", "أهلا وسهلا"),
("how are you", "كيف حالك"),
("i am fine", "أنا بخير"),
("see you later", "أراك لاحقا"),
("please", "من فضلك"),
("excuse me", "عذرا"),
("congratulations", "مبروك"),
("happy birthday", "عيد ميلاد سعيد"),
("good luck", "حظا سعيدا"),
("no problem", "لا مشكلة"),
# i am ...
("i am happy", "أنا سعيد"),
("i am sad", "أنا حزين"),
("i am tired", "أنا متعب"),
("i am hungry", "أنا جائع"),
("i am thirsty", "أنا عطشان"),
("i am here", "أنا هنا"),
("i am a student", "أنا طالب"),
("i am a teacher", "أنا معلم"),
("i am ready", "أنا مستعد"),
("i am busy", "أنا مشغول"),
# i + verb + object
("i love you", "أحبك"),
("i love reading", "أحب القراءة"),
("i read a book", "أقرأ كتابا"),
("i drink water", "أشرب الماء"),
("i eat bread", "آكل الخبز"),
("i drink coffee", "أشرب القهوة"),
("i drink tea", "أشرب الشاي"),
("i write a letter", "أكتب رسالة"),
("i want water", "أريد ماء"),
("i have a car", "لدي سيارة"),
("i have a book", "لدي كتاب"),
("i see the moon", "أرى القمر"),
("i go to school", "أذهب إلى المدرسة"),
("i live in cairo", "أعيش في القاهرة"),
("i speak arabic", "أتحدث العربية"),
("i play football", "ألعب كرة القدم"),
("i like music", "أحب الموسيقى"),
("i need help", "أحتاج مساعدة"),
("i know the answer", "أعرف الإجابة"),
("i open the door", "أفتح الباب"),
# he / she
("he is a doctor", "هو طبيب"),
("she is a teacher", "هي معلمة"),
("he reads a book", "هو يقرأ كتابا"),
("she writes a letter", "هي تكتب رسالة"),
("he loves coffee", "هو يحب القهوة"),
("she drinks tea", "هي تشرب الشاي"),
("he goes to work", "هو يذهب إلى العمل"),
("she plays piano", "هي تعزف البيانو"),
("he is my friend", "هو صديقي"),
("she is my sister", "هي أختي"),
("he has a dog", "لديه كلب"),
("she has a cat", "لديها قطة"),
("he speaks english", "هو يتحدث الإنجليزية"),
("she lives in dubai", "هي تعيش في دبي"),
("he is tall", "هو طويل"),
# we / they / you
("we are friends", "نحن أصدقاء"),
("we love our country", "نحن نحب بلدنا"),
("we go home", "نذهب إلى البيت"),
("they are students", "هم طلاب"),
("they play together", "هم يلعبون معا"),
("you are welcome", "على الرحب والسعة"),
("you are kind", "أنت لطيف"),
("where are you", "أين أنت"),
("what is your name", "ما اسمك"),
("my name is ahmed", "اسمي أحمد"),
# questions & statements
("what time is it", "كم الساعة"),
("how much is this", "بكم هذا"),
("where is the bathroom", "أين الحمام"),
("i do not understand", "لا أفهم"),
("can you help me", "هل يمكنك مساعدتي"),
("i am sorry", "أنا آسف"),
("it is cold", "الجو بارد"),
("it is hot", "الجو حار"),
("the weather is nice", "الطقس جميل"),
("today is monday", "اليوم الاثنين"),
("the book is on the table", "الكتاب على الطاولة"),
("the food is delicious", "الطعام لذيذ"),
("the house is big", "البيت كبير"),
("the car is fast", "السيارة سريعة"),
("the water is cold", "الماء بارد"),
# simple subject-verb-object
("the boy eats an apple", "الولد يأكل تفاحة"),
("the girl reads a book", "البنت تقرأ كتابا"),
("the cat drinks milk", "القطة تشرب الحليب"),
("the dog runs fast", "الكلب يجري بسرعة"),
("the sun is bright", "الشمس ساطعة"),
("the children play in the garden", "الأطفال يلعبون في الحديقة"),
("the man drives a car", "الرجل يقود سيارة"),
("the woman cooks food", "المرأة تطبخ الطعام"),
("the bird flies", "الطائر يطير"),
("the teacher explains the lesson", "المعلم يشرح الدرس"),
("the student asks a question", "الطالب يسأل سؤالا"),
("mother makes tea", "الأم تصنع الشاي"),
("father reads the newspaper", "الأب يقرأ الجريدة"),
("the train is late", "القطار متأخر"),
("i will travel tomorrow", "سأسافر غدا"),
]
print(len(pairs), "pairs")100 pairsNow the special tokens — every seq2seq model needs these:
<pad>— filler to make sequences in a batch the same length.<sos>— “start of sequence,” the decoder’s first input that says begin translating.<eos>— “end of sequence,” what the decoder learns to emit when it’s done.<unk>— “unknown,” words outside the vocab gets marked as unknown.
Show Code
PAD, SOS, EOS, UNK = "<pad>", "<sos>", "<eos>", "<unk>"
special_tokens = [PAD, SOS, EOS, UNK] # PAD=0, SOS=1, EOS=2, UNK=3Show Code
def tokenize(text):
text = text.lower().strip()
text = re.sub(r"([^\w\s])", r" \1 ", text) # space out punctuation
return text.split() # \w matches Arabic letters tooWe will define a function below to map each word to a dictionary of numbers.
The function recieves a list of
tokenized_sentences:[["i", "love", "you"], ["i", "read", "a", "book"], ...], Then thecounterwill counts how many times each word appear, it returns this:Counter({'hello': 1, 'goodbye': 1, 'thank': 1, 'you': 1})Then only keeps the words that at least appears once
min_freq=1–> keeping all words in this case.prepend the special tokens to the
frequent_wordsdictionary.Finally, map each word to an index:
{'hello': 4,'goodbye': 5,'thank': 6,'you': 7}
Show Code
def build_vocab(tokenized_sentences, min_freq=1):
# count how many times each word appears → every word with its frequency
word_counts = Counter(tok for sent in tokenized_sentences for tok in sent)
# keep a word only if it appears at least min_freq times, dropping rare words
frequent_words = [w for w, c in word_counts.items() if c >= min_freq]
# prepend the special tokens (so PAD=0, SOS=1, EOS=2, UNK=3)
vocab = special_tokens + frequent_words
# map each word to an index, and the reverse
word_to_index = {word: idx for idx, word in enumerate(vocab)}
index_to_word = {idx: word for word, idx in word_to_index.items()}
return word_to_index, index_to_word
english_tokens = [tokenize(en) for en, ar in pairs]
arabic_tokens = [tokenize(ar) for en, ar in pairs]
english_word_to_index, english_index_to_word = build_vocab(english_tokens)
arabic_word_to_index, arabic_index_to_word = build_vocab(arabic_tokens)
print(f"English vocab: {len(english_word_to_index):,} words")
print(f"Arabic vocab: {len(arabic_word_to_index):,} words")English vocab: 163 words
Arabic vocab: 180 wordsThen below we prepare the input and target data for the decoder:
The function
encode_sourcemapp the sentance to indicies[18, 28], and set<unk>to unknown words, then append with the end of sentance token<eos>=2.The function
encode_target_wrappedwrapps the sentance in special chars,<sos>=1at start and<eos>=2at the end.Then
make_decoder_iowill output 2 lists, the first one is the input to the decoder which contains all chars without the first one (<sos>), and the second list contains all chars without the last char (<eos>).
Show Code
def encode_source(english_tokens):
# English words → indices (+ EOS), with <unk> fallback
ids = [english_word_to_index.get(t, english_word_to_index[UNK]) for t in english_tokens]
return ids + [english_word_to_index[EOS]]
def encode_target_wrapped(arabic_tokens):
# Arabic words → indices, wrapped with <sos> at the start and <eos> at the end
ids = [arabic_word_to_index.get(t, arabic_word_to_index[UNK]) for t in arabic_tokens] #this gets the index for each word
return [arabic_word_to_index[SOS]] + ids + [arabic_word_to_index[EOS]] # then wrapp it in special tokens
def make_decoder_io(arabic_tokens):
# [<sos>, w1, w2, ..., wn, <eos>]
wrapped = encode_target_wrapped(arabic_tokens)
# [<sos>, w1, w2, ..., wn] → FED to decoder
decoder_input = wrapped[:-1] #all but the last char <EOS>
# [w1, w2, ..., wn, <eos>] → what it PREDICTS
decoder_target = wrapped[1:] # all but the first char <SOS>
return decoder_input, decoder_target
make_decoder_io(["أنا", "سعيد"])([1, 18, 28], [18, 28, 2])Here we create a padding function that do the following: - First finds the longest sequence in the list –> max_len
make a tensor pre-filled with
pad_index(0) of shape(num_sequences, max_len)copy each real sequence into the start of its row, leaving the rest as padding
Then the build_full_batch function loops over all pairs to build 3 lists : - english_input : the input to the encoder (English text) - decoder_input : the input to the decoder (Arabic text) - decoder_target : the target (true vlues) to the decoder (Arabic text)
Becasue we have 100 pairs, the output of the above 3 lists is: (100, max_english_len), (100, max_arabic_len), (100, max_arabic_len)
Show Code
def pad_sequences(sequences, pad_index=0):
max_len = max(len(s) for s in sequences)
padded = torch.full((len(sequences), max_len), pad_index, dtype=torch.long)
for i, seq in enumerate(sequences):
# seq is a plain Python list of integers, e.g. [1, 15, 23, 2]
padded[i, :len(seq)] = torch.tensor(seq, dtype=torch.long) # :len(seq) fill columns from 0 to len(seq)
return padded
def build_full_batch(pairs):
english_seqs, decoder_in_seqs, decoder_tgt_seqs = [], [], []
for english, arabic in pairs:
english_seqs.append(encode_source(tokenize(english)))
dec_in, dec_tgt = make_decoder_io(tokenize(arabic))
decoder_in_seqs.append(dec_in)
decoder_tgt_seqs.append(dec_tgt)
english_input = pad_sequences(english_seqs) # (100, max_english_len)
decoder_input = pad_sequences(decoder_in_seqs) # (100, max_arabic_len)
decoder_target = pad_sequences(decoder_tgt_seqs) # (100, max_arabic_len)
return english_input, decoder_input, decoder_target
english_input, decoder_input, decoder_target = build_full_batch(pairs)
print("english_input :", english_input.shape)
print("decoder_input :", decoder_input.shape)
print("decoder_target:", decoder_target.shape)english_input : torch.Size([100, 7])
decoder_input : torch.Size([100, 5])
decoder_target: torch.Size([100, 5])Show Code
print(" ".join(english_index_to_word[i.item()] for i in english_input[15]))
print(" ".join(arabic_index_to_word[i.item()] for i in decoder_input[15]))
print(" ".join(arabic_index_to_word[i.item()] for i in decoder_target[15]))i am happy <eos> <pad> <pad> <pad>
<sos> أنا سعيد <pad> <pad>
أنا سعيد <eos> <pad> <pad>Notice the output above for the english sentance, it is appended at the end with the <eos> token, while the arabic text that will be fed to the decoder is started with <sos> for the input and ended with <eos> for the target sentance.
Building the model
The Encoder
The encoder’s only job is to read the English sentence and squeeze its meaning into a context vector. It is a small two-layer module:
nn.Embedding— turns each English word index into a dense vector.padding_idx=PAD_IDXtells it the<pad>token is meaningless, so its embedding stays at zero and gets no gradient.nn.LSTM— reads the embedded sequence left to right. We usebatch_first=Trueso tensors are shaped(batch, seq_len, ...)to match our padded batches.
In the forward() pass, we feed the english_input which has this shape (batch, seq_len) = (100,7). These will go first to the embed layer which will create a 32-dimensional vector for every token, turning (100, 7) into (100, 7, 32) — now each of the 7 positions in each sentence is represented as a 32-number vector instead of a single index.
Then the output (100, 7, 32) of the embedding will go to the LSTM layer, which reads through the 7 positions left to right, updating its (hidden, cell) at each step. After the 7th step the LSTM returns:
outputs— shape(100, 7, 64), the hidden state at every step. We ignore this.hidden— shape(1, 100, 64), the final hidden state after reading the last real word.cell— shape(1, 100, 64), the final cell state (long-term memory) at the last step.
Show Code
PAD_IDX = english_word_to_index[PAD] # 0 — shared by both vocabs
class Encoder(nn.Module):
def __init__(self, input_vocab_size, embed_size, hidden_size):
super().__init__()
self.embed = nn.Embedding(input_vocab_size, embed_size, padding_idx=PAD_IDX)
self.lstm = nn.LSTM(embed_size, hidden_size, batch_first=True)
def forward(self, english_input):
# english_input: (batch, seq_len) of word indices
embedded = self.embed(english_input)
outputs, (hidden, cell) = self.lstm(embedded)
return hidden, cell
# quick shape check
encoder = Encoder(input_vocab_size=len(english_word_to_index), embed_size=32, hidden_size=64)
h, c = encoder(english_input)
print("hidden:", h.shape)
print("cell :", c.shape)hidden: torch.Size([1, 100, 64])
cell : torch.Size([1, 100, 64])The Decoder
The decoder is the generator. It starts from the encoder’s (hidden, cell) context and produces the Arabic sentence one token at a time. It has three layers:
nn.Embedding— turns each Arabic word index into a dense vector (its own vocab, separate from English).nn.LSTM— samehidden_sizeas the encoder’s, so it can pick up the encoder’s(hidden, cell)and keep running from there.nn.Linear— maps each LSTM step(…, hidden_size)to a score for every word in the Arabic vocab(…, arabic_vocab_size). The highest score is the predicted next word.
In the forward() we feed the whole decoder_input ([<sos>, w1, w2, …]) in one shot, along with the encoder’s (hidden, cell):
decoder_input(100, 5)→embed→(100, 5, 32)that goes into the LSTM with
(hidden, cell)as its starting state → outputs(100, 5, 64), the hidden state at every Arabic position.fcmaps each position →(100, 5, 180), a score over the whole Arabic vocab at every step. The180 = Arabic vocab size.
Show Code
class Decoder(nn.Module):
def __init__(self, output_vocab_size, embed_size, hidden_size):
super().__init__()
self.embed = nn.Embedding(output_vocab_size, embed_size, padding_idx=PAD_IDX)
self.lstm = nn.LSTM(embed_size, hidden_size, batch_first=True)
self.fc = nn.Linear(hidden_size, output_vocab_size)
def forward(self, decoder_input, hidden, cell): #hidden and cell comming from encoder
# decoder_input: (batch, seq_len) of Arabic word indices
# (batch, seq_len, embed_size)
embedded = self.embed(decoder_input)
outputs, (hidden, cell) = self.lstm(embedded, (hidden, cell))
# (batch, seq_len, arabic_vocab_size)
logits = self.fc(outputs)
return logits, hidden, cell
# quick shape check — start from the encoder's context
decoder = Decoder(output_vocab_size=len(arabic_word_to_index), embed_size=32, hidden_size=64)
logits, h, c = decoder(decoder_input, h, c)
print("logits:", logits.shape) # (100, 5, 180)logits: torch.Size([100, 5, 180])Full Model
Below the class that wrapps the encoder and decoder together.
Show Code
class Seq2Seq(nn.Module):
def __init__(self, encoder, decoder):
super().__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, english_input, decoder_input):
# 1. encoder reads the English sentence → context (hidden, cell)
hidden, cell = self.encoder(english_input)
# 2. decoder generates Arabic, starting from that context
logits, hidden, cell = self.decoder(decoder_input, hidden, cell)
return logits # (batch, arabic_seq_len, arabic_vocab_size)
# build the full model
encoder = Encoder(input_vocab_size=len(english_word_to_index), embed_size=32, hidden_size=64)
decoder = Decoder(output_vocab_size=len(arabic_word_to_index), embed_size=32, hidden_size=64)
model = Seq2Seq(encoder, decoder)
# quick shape check
logits = model(english_input, decoder_input)
print("logits:", logits.shape) # (100, 5, 180)logits: torch.Size([100, 5, 180])Below we start the training loop, where we pass both english_input and decoder_input to the model class build above, to get the logits which is (100, 5, 180) for each seq 5 vectors each with 180 length (prob score for each arabic word).
Then flatt this to (500, 180) and flatt the target to (500,) to feed both to the CrossEntropyLoss.
Show Code
import torch.optim as optim
loss_fn = nn.CrossEntropyLoss(ignore_index=PAD_IDX) # skip <pad> positions
optimizer = optim.Adam(model.parameters(), lr=0.01)
EPOCHS = 300
for epoch in range(1, EPOCHS + 1):
model.train()
# forward: English + teacher-forced Arabic input → scores over Arabic vocab
logits = model(english_input, decoder_input) # (100, 5, 180)
# flatten for the loss:
# logits: (100, 5, 180) → (500, 180) predictions
# target: (100, 5) → (500,) targets
loss = loss_fn(
logits.reshape(-1, logits.size(-1)),
decoder_target.reshape(-1),
)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch % 20 == 0:
print(f"epoch {epoch:3d} | loss {loss.item():.4f}")epoch 20 | loss 2.8197
epoch 40 | loss 1.2038
epoch 60 | loss 0.2868
epoch 80 | loss 0.0463
epoch 100 | loss 0.0147
epoch 120 | loss 0.0083
epoch 140 | loss 0.0060
epoch 160 | loss 0.0047
epoch 180 | loss 0.0038
epoch 200 | loss 0.0032
epoch 220 | loss 0.0027
epoch 240 | loss 0.0024
epoch 260 | loss 0.0021
epoch 280 | loss 0.0019
epoch 300 | loss 0.0017Show Code
def translate(english_sentence, max_len=20):
model.eval()
with torch.no_grad():
# 1. encode the English sentence → context (hidden, cell)
# list of indices + <eos>
source = encode_source(tokenize(english_sentence))
source = torch.tensor(source).unsqueeze(0) # (1, src_len)
hidden, cell = model.encoder(source)
# 2. start the decoder with just <sos>
current = torch.tensor([[arabic_word_to_index[SOS]]]) # (1, 1)
result = []
for _ in range(max_len):
# feed ONE token, carry (hidden, cell) forward
logits, hidden, cell = model.decoder(
current, hidden, cell) # (1, 1, vocab)
# pick highest-scoring word
next_idx = logits[0, -1].argmax().item()
# decoder says "done"
if next_idx == arabic_word_to_index[EOS]:
break
result.append(arabic_index_to_word[next_idx])
# the prediction becomes the next input
current = torch.tensor([[next_idx]]) # (1, 1)
return " ".join(result)
# try it
for en, _ in pairs[:20]:
print(f"{en:25s} → {translate(en)}")hello → مرحبا
goodbye → مع السلامة
thank you → هي تعزف البيانو
good morning → صباح الخير
good night → تصبح على خير
welcome → هي تشرب الشاي
how are you → كيف حالك
i am fine → أنا بخير
see you later → أراك لاحقا
please → هي تشرب الشاي
excuse me → عذرا
congratulations → نحن أصدقاء
happy birthday → القطار متأخر
good luck → حظا سعيدا
no problem → لا مشكلة
i am happy → أنا سعيد
i am sad → أنا حزين
i am tired → أنا متعب
i am hungry → أنا جائع
i am thirsty → أنا عطشانShow Code
print(translate("sad happy hungry me"))
print(translate("see her please"))الأب يقرأ الجريدة
أعيش في القاهرةExplaining the results
We see that the model when it sees new english sentances that are not in the training data, it fails to mapp even a single word to the correct arabic meaning, it translated this "sad happy hungry me" to this الأب يقرأ الجريدة , this means that the model squeezed the words meanings to a single vector and never capture each word meaning individually.
Hypothesis: the model stores one ID-code per sentence and just retrieves the nearest one. The model dosen’t compose anything new other than replaying a stored sentence.
Validation: we should be able to predict the model output without running the decoder — just by finding the nearest training context vector.
steps:
We build a function that gets the sentance then return both
handc.Then build another function that takes the new sentance, get the context vector using the above step, then pick the nearset vector from the training step using
cosine_similarity.Finally, compare what the model will transalte with what the function that picks the nearset vector will output.
Show Code
def encode_to_context(sentence):
"""Run only the encoder → the (h, c) context as one flat vector."""
model.eval()
with torch.no_grad():
ids = encode_source(tokenize(sentence))
src = torch.tensor(ids).unsqueeze(0) # (1, src_len)
hidden, cell = model.encoder(src)
return torch.cat([hidden.squeeze(), cell.squeeze()]) # (128,) = 64 + 64
# encode all 100 training sentences → their context vectors
train_contexts = torch.stack([encode_to_context(en) for en, ar in pairs])
print(train_contexts.shape) # (100, 128) ← 100 "ID codes"torch.Size([100, 128])Show Code
import torch.nn.functional as F
def nearest_training_pair(sentence):
q = encode_to_context(sentence) # (128,)
sims = F.cosine_similarity(q.unsqueeze(0), train_contexts) # (100,)
best = sims.argmax().item()
en, ar = pairs[best]
return best, sims[best].item(), en, ar
for test in ["sad happy hungry me", "see her please"]:
idx, sim, en, ar = nearest_training_pair(test)
print(f"input: {test}")
print(f"model translates: {translate(test)}")
print(f"nearest training: [{idx}] {en!r} → {ar} (cos={sim:.2f})")
print()input: sad happy hungry me
model translates: الأب يقرأ الجريدة
nearest training: [97] 'father reads the newspaper' → الأب يقرأ الجريدة (cos=0.73)
input: see her please
model translates: أعيش في القاهرة
nearest training: [38] 'i live in cairo' → أعيش في القاهرة (cos=0.83)
Show Code
base = "i am happy"
variants = ["i am happy", "happy i am", "am happy i", "i am sad"]
ref = encode_to_context(base)
for v in variants:
sim = F.cosine_similarity(ref.unsqueeze(
0), encode_to_context(v).unsqueeze(0)).item()
print(f"cos({base!r}, {v!r}) = {sim:.2f} → {translate(v)}")cos('i am happy', 'i am happy') = 1.00 → أنا سعيد
cos('i am happy', 'happy i am') = 0.86 → أنا عطشان
cos('i am happy', 'am happy i') = 0.87 → أنا سعيد
cos('i am happy', 'i am sad') = 0.86 → أنا حزينShow Code
print("Similiarty score between (i am thirsty) and (happy i am):",
F.cosine_similarity(encode_to_context("i am thirsty").unsqueeze(0),
encode_to_context("happy i am").unsqueeze(0)).item())
print("Similiarty score between (happy i am) and (i am happy):",
F.cosine_similarity(encode_to_context("happy i am").unsqueeze(0),
encode_to_context("i am happy").unsqueeze(0)).item())
print("--------------------")
print(f"Translation: i am thirsty --> {translate("i am thirsty")}")
print(f"Translation: happy i am --> {translate("happy i am")}")Similiarty score between (i am thirsty) and (happy i am): 0.8218614459037781
Similiarty score between (happy i am) and (i am happy): 0.8635380864143372
--------------------
Translation: i am thirsty --> أنا عطشان
Translation: happy i am --> أنا عطشانWhat does the above tell us??
If the hypothsis that the model just picks the translation sentance based on the nearest similarity score is correct, then we should see the model translates happy i am to the same translation of i am happy which is أنا سعيد, because the similarty is the highest 0.86, but it picks the translation to be أنا عطشان, why??
Show Code
import torch.nn.functional as F
def translate_verbose(sentence, max_len=10, topk=3):
model.eval()
with torch.no_grad():
src = torch.tensor(encode_source(tokenize(sentence))).unsqueeze(0)
hidden, cell = model.encoder(src)
current = torch.tensor([[arabic_word_to_index[SOS]]])
for step in range(max_len):
logits, hidden, cell = model.decoder(current, hidden, cell)
probs = F.softmax(logits[0, -1], dim=-1)
top_p, top_i = probs.topk(topk)
shown = ", ".join(f"{arabic_index_to_word[i.item()]}={p:.2f}"
for p, i in zip(top_p, top_i))
best = top_i[0].item()
print(f"step {step}: {shown}")
if best == arabic_word_to_index[EOS]:
break
current = torch.tensor([[best]])
translate_verbose("happy i am")step 0: أنا=0.94, هي=0.01, لا=0.01
step 1: عطشان=0.37, بخير=0.22, متعب=0.12
step 2: <eos>=1.00, الشاي=0.00, سعيد=0.00
Comments