-
Notifications
You must be signed in to change notification settings - Fork 139
/
a833bed8-2fa8-4cfe-af05-58c1cc48bc30.txt
3880 lines (3813 loc) · 254 KB
/
a833bed8-2fa8-4cfe-af05-58c1cc48bc30.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True,
backend='newtonschulz5', backend_steps=5):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
# generate weight updates in distributed fashion
total_params = sum(p.numel() for p in group['params'])
updates_flat = torch.zeros(total_params, device='cuda', dtype=torch.bfloat16)
curr_idx = 0
for i, p in enumerate(group['params']):
# luckily this will perfectly distribute a transformer with multiple of 4 layers to 8 GPUs
if i % int(os.environ['WORLD_SIZE']) == int(os.environ['RANK']):
g = p.grad
assert g is not None
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
g = zeropower_backend(g, steps=group['backend_steps'])
g *= max(1, g.size(0)/g.size(1))**0.5
updates_flat[curr_idx:curr_idx+p.numel()] = g.flatten()
curr_idx += p.numel()
# sync updates across devices. we are not memory-constrained so can do this simple deserialization
dist.all_reduce(updates_flat, op=dist.ReduceOp.SUM)
# deserialize and apply updates
curr_idx = 0
for p in group['params']:
g = updates_flat[curr_idx:curr_idx+p.numel()].view_as(p.data).type_as(p.data)
p.data.add_(g, alpha=-lr)
curr_idx += p.numel()
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos().bfloat16()
self.sin_cached = freqs.sin().bfloat16()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3).type_as(x)
class CastedLinear(nn.Linear):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def forward(self, x):
return F.linear(x, self.weight.to(x.dtype))
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
self.c_q = CastedLinear(self.n_embd, self.n_embd, bias=False)
self.c_k = CastedLinear(self.n_embd, self.n_embd, bias=False)
self.c_v = CastedLinear(self.n_embd, self.n_embd, bias=False)
# output projection
self.c_proj = CastedLinear(self.n_embd, self.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
self.rotary = Rotary(self.head_dim)
self.lamb = nn.Parameter(torch.tensor(0.5)) # @Grad62304977
def forward(self, x, v1=None):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_head, self.head_dim)
if v1 is None:
v1 = v # This happens if we are in the first block. v needs to be accessed by subsequent blocks
v = (1 - self.lamb) * v + self.lamb * v1.view_as(v) # @Grad62304977
cos, sin = self.rotary(q)
q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),)) # QK norm suggested by @Grad62304977
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y, v1
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = CastedLinear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = CastedLinear(4 * config.n_embd, config.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x):
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
self.lambdas = nn.Parameter(torch.tensor([1., 0.]))
def forward(self, x, v1, x0):
x = self.lambdas[0] * x + self.lambdas[1] * x0
x1, v1 = self.attn(F.rms_norm(x, (x.size(-1),)), v1)
x = x + x1
x = x + self.mlp(F.rms_norm(x, (x.size(-1),)))
return x, v1
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = CastedLinear(config.n_embd, config.vocab_size, bias=False)
self.lm_head.weight.data.zero_() # @Grad62304977
def forward(self, idx, target):
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
x = F.rms_norm(x, (x.size(-1),)) # @Grad62304977
x0 = x
v1 = None
for block in self.transformer.h:
x, v1 = block(x, v1, x0)
x = F.rms_norm(x, (x.size(-1),))
logits = self.lm_head(x)
logits = 30 * torch.tanh(logits / 30) # @Grad62304977
logits = logits.float()
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), target.view(-1))
return loss.float()
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 3242 # number of iterations to run
warmup_iters : int = 0
warmdown_iters : int = 926 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda().bfloat16()
for m in model.modules():
if isinstance(m, CastedLinear):
m.float()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
# CUDNN attention is ~4ms faster than Flash, but doesn't get selected by default in PyTorch 2.5.1
from torch.backends.cuda import enable_cudnn_sdp, enable_flash_sdp, enable_math_sdp, enable_mem_efficient_sdp
enable_cudnn_sdp(True)
enable_flash_sdp(False)
enable_mem_efficient_sdp(False)
enable_math_sdp(False)
# init the optimizer(s)
optimizer1 = torch.optim.Adam([raw_model.transformer.wte.weight], lr=0.3, betas=(0.9, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.002, betas=(0.9, 0.95), fused=True)
params = list(raw_model.transformer.h.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2]
optimizer3 = Muon(matrix_params, lr=0.02, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.02, betas=(0.9, 0.95), fused=True) # note that this learning rate is neither sensitive nor tuned
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
with torch.no_grad():
x_val, y_val = val_loader.next_batch()
val_loss += model(x_val, y_val)
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
loss = model(x, y)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# momentum warmup for Muon
frac = min(step/500, 1)
optimizer3.param_groups[0]['momentum'] = (1 - frac) * 0.85 + frac * 0.95
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.5.1+cu124 compiled for CUDA 12.4
nvidia-smi:
Sat Nov 9 02:14:49 2024
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 560.35.03 Driver Version: 560.35.03 CUDA Version: 12.6 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA H100 80GB HBM3 Off | 00000000:18:00.0 Off | 0 |
| N/A 31C P0 115W / 700W | 6787MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 Off | 00000000:2A:00.0 Off | 0 |
| N/A 33C P0 116W / 700W | 5278MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 Off | 00000000:3A:00.0 Off | 0 |
| N/A 32C P0 119W / 700W | 5278MiB / 81559MiB | 7% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 Off | 00000000:5D:00.0 Off | 0 |
| N/A 30C P0 117W / 700W | 5204MiB / 81559MiB | 1% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 Off | 00000000:84:00.0 Off | 0 |
| N/A 31C P0 121W / 700W | 5278MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 Off | 00000000:8B:00.0 Off | 0 |
| N/A 33C P0 119W / 700W | 5204MiB / 81559MiB | 5% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 Off | 00000000:91:00.0 Off | 0 |
| N/A 31C P0 123W / 700W | 5204MiB / 81559MiB | 5% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 Off | 00000000:E4:00.0 Off | 0 |
| N/A 30C P0 118W / 700W | 4964MiB / 81559MiB | 5% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 47031 C /usr/bin/python3 0MiB |
| 0 N/A N/A 91635 C /usr/bin/python3 0MiB |
| 1 N/A N/A 91636 C /usr/bin/python3 0MiB |
| 2 N/A N/A 91637 C /usr/bin/python3 0MiB |
| 3 N/A N/A 91638 C /usr/bin/python3 0MiB |
| 4 N/A N/A 91639 C /usr/bin/python3 0MiB |
| 5 N/A N/A 91640 C /usr/bin/python3 0MiB |
| 6 N/A N/A 91641 C /usr/bin/python3 0MiB |
| 7 N/A N/A 91642 C /usr/bin/python3 0MiB |
+-----------------------------------------------------------------------------------------+
====================================================================================================
step:0/3242 val_loss:10.8258 train_time:407ms step_avg:nanms
step:1/3242 train_loss:10.8258 train_time:27560ms step_avg:nanms
step:2/3242 train_loss:10.4302 train_time:27660ms step_avg:nanms
step:3/3242 train_loss:9.9632 train_time:27800ms step_avg:nanms
step:4/3242 train_loss:9.0524 train_time:27941ms step_avg:nanms
step:5/3242 train_loss:8.0705 train_time:28082ms step_avg:nanms
step:6/3242 train_loss:7.5294 train_time:28222ms step_avg:nanms
step:7/3242 train_loss:7.0088 train_time:28362ms step_avg:nanms
step:8/3242 train_loss:7.3020 train_time:28507ms step_avg:nanms
step:9/3242 train_loss:6.9019 train_time:28656ms step_avg:nanms
step:10/3242 train_loss:6.7843 train_time:28800ms step_avg:nanms
step:11/3242 train_loss:6.7989 train_time:99ms step_avg:nanms
step:12/3242 train_loss:6.6594 train_time:243ms step_avg:nanms
step:13/3242 train_loss:6.4964 train_time:386ms step_avg:128.78ms
step:14/3242 train_loss:6.4658 train_time:527ms step_avg:131.72ms
step:15/3242 train_loss:6.4421 train_time:672ms step_avg:134.40ms
step:16/3242 train_loss:6.4228 train_time:817ms step_avg:136.24ms
step:17/3242 train_loss:6.4070 train_time:962ms step_avg:137.44ms
step:18/3242 train_loss:6.4398 train_time:1106ms step_avg:138.19ms
step:19/3242 train_loss:6.2903 train_time:1249ms step_avg:138.81ms
step:20/3242 train_loss:6.3081 train_time:1391ms step_avg:139.07ms
step:21/3242 train_loss:6.0176 train_time:1532ms step_avg:139.29ms
step:22/3242 train_loss:6.3354 train_time:1677ms step_avg:139.74ms
step:23/3242 train_loss:6.5671 train_time:1820ms step_avg:140.02ms
step:24/3242 train_loss:6.2245 train_time:1965ms step_avg:140.36ms
step:25/3242 train_loss:6.3745 train_time:2108ms step_avg:140.56ms
step:26/3242 train_loss:6.0819 train_time:2251ms step_avg:140.72ms
step:27/3242 train_loss:6.0082 train_time:2395ms step_avg:140.90ms
step:28/3242 train_loss:6.1830 train_time:2537ms step_avg:140.95ms
step:29/3242 train_loss:5.8549 train_time:2680ms step_avg:141.05ms
step:30/3242 train_loss:6.1104 train_time:2826ms step_avg:141.29ms
step:31/3242 train_loss:5.9467 train_time:2971ms step_avg:141.49ms
step:32/3242 train_loss:5.9090 train_time:3114ms step_avg:141.54ms
step:33/3242 train_loss:5.7645 train_time:3257ms step_avg:141.61ms
step:34/3242 train_loss:6.0378 train_time:3399ms step_avg:141.63ms
step:35/3242 train_loss:5.9721 train_time:3543ms step_avg:141.71ms
step:36/3242 train_loss:6.0987 train_time:3685ms step_avg:141.74ms
step:37/3242 train_loss:6.0356 train_time:3831ms step_avg:141.90ms
step:38/3242 train_loss:5.9364 train_time:3976ms step_avg:142.01ms
step:39/3242 train_loss:5.8163 train_time:4119ms step_avg:142.03ms
step:40/3242 train_loss:5.8390 train_time:4261ms step_avg:142.04ms
step:41/3242 train_loss:5.7589 train_time:4405ms step_avg:142.08ms
step:42/3242 train_loss:5.7571 train_time:4546ms step_avg:142.07ms
step:43/3242 train_loss:5.6705 train_time:4689ms step_avg:142.09ms
step:44/3242 train_loss:5.7407 train_time:4836ms step_avg:142.23ms
step:45/3242 train_loss:5.7322 train_time:4981ms step_avg:142.30ms
step:46/3242 train_loss:5.8853 train_time:5126ms step_avg:142.38ms
step:47/3242 train_loss:5.6683 train_time:5269ms step_avg:142.40ms
step:48/3242 train_loss:5.5351 train_time:5412ms step_avg:142.41ms
step:49/3242 train_loss:5.7383 train_time:5555ms step_avg:142.44ms
step:50/3242 train_loss:5.6122 train_time:5698ms step_avg:142.44ms
step:51/3242 train_loss:5.7661 train_time:5842ms step_avg:142.49ms
step:52/3242 train_loss:5.6261 train_time:5987ms step_avg:142.55ms
step:53/3242 train_loss:5.4832 train_time:6132ms step_avg:142.60ms
step:54/3242 train_loss:5.6084 train_time:6275ms step_avg:142.60ms
step:55/3242 train_loss:5.4819 train_time:6417ms step_avg:142.59ms
step:56/3242 train_loss:5.8096 train_time:6560ms step_avg:142.60ms
step:57/3242 train_loss:5.4634 train_time:6704ms step_avg:142.64ms
step:58/3242 train_loss:5.3540 train_time:6848ms step_avg:142.68ms
step:59/3242 train_loss:5.4882 train_time:6992ms step_avg:142.69ms
step:60/3242 train_loss:5.4475 train_time:7135ms step_avg:142.71ms
step:61/3242 train_loss:5.5358 train_time:7278ms step_avg:142.71ms
step:62/3242 train_loss:5.2983 train_time:7420ms step_avg:142.70ms
step:63/3242 train_loss:5.4061 train_time:7565ms step_avg:142.73ms
step:64/3242 train_loss:5.3810 train_time:7708ms step_avg:142.75ms
step:65/3242 train_loss:5.1976 train_time:7852ms step_avg:142.77ms
step:66/3242 train_loss:5.1995 train_time:7996ms step_avg:142.79ms
step:67/3242 train_loss:5.3449 train_time:8138ms step_avg:142.77ms
step:68/3242 train_loss:5.2081 train_time:8281ms step_avg:142.78ms
step:69/3242 train_loss:5.4659 train_time:8426ms step_avg:142.82ms
step:70/3242 train_loss:5.1226 train_time:8570ms step_avg:142.84ms
step:71/3242 train_loss:5.1952 train_time:8714ms step_avg:142.84ms
step:72/3242 train_loss:5.3446 train_time:8857ms step_avg:142.85ms
step:73/3242 train_loss:5.2647 train_time:8999ms step_avg:142.85ms
step:74/3242 train_loss:5.1605 train_time:9142ms step_avg:142.85ms
step:75/3242 train_loss:5.2702 train_time:9285ms step_avg:142.85ms
step:76/3242 train_loss:5.2542 train_time:9431ms step_avg:142.89ms
step:77/3242 train_loss:5.1968 train_time:9574ms step_avg:142.89ms
step:78/3242 train_loss:5.2921 train_time:9716ms step_avg:142.88ms
step:79/3242 train_loss:5.3989 train_time:9859ms step_avg:142.89ms
step:80/3242 train_loss:5.1484 train_time:10004ms step_avg:142.91ms
step:81/3242 train_loss:5.2329 train_time:10146ms step_avg:142.90ms
step:82/3242 train_loss:5.0085 train_time:10289ms step_avg:142.91ms
step:83/3242 train_loss:5.1838 train_time:10433ms step_avg:142.91ms
step:84/3242 train_loss:5.1261 train_time:10576ms step_avg:142.91ms
step:85/3242 train_loss:5.1156 train_time:10718ms step_avg:142.91ms
step:86/3242 train_loss:4.9789 train_time:10861ms step_avg:142.91ms
step:87/3242 train_loss:5.1865 train_time:11005ms step_avg:142.93ms
step:88/3242 train_loss:5.1005 train_time:11149ms step_avg:142.93ms
step:89/3242 train_loss:5.1317 train_time:11292ms step_avg:142.94ms
step:90/3242 train_loss:5.1066 train_time:11436ms step_avg:142.95ms
step:91/3242 train_loss:5.0214 train_time:11578ms step_avg:142.94ms
step:92/3242 train_loss:5.0198 train_time:11723ms step_avg:142.97ms
step:93/3242 train_loss:5.1317 train_time:11868ms step_avg:142.99ms
step:94/3242 train_loss:4.9734 train_time:12012ms step_avg:142.99ms
step:95/3242 train_loss:4.9726 train_time:12156ms step_avg:143.01ms
step:96/3242 train_loss:5.0216 train_time:12299ms step_avg:143.01ms
step:97/3242 train_loss:4.9259 train_time:12442ms step_avg:143.01ms
step:98/3242 train_loss:5.0029 train_time:12586ms step_avg:143.02ms
step:99/3242 train_loss:4.9233 train_time:12731ms step_avg:143.04ms
step:100/3242 train_loss:5.0289 train_time:12875ms step_avg:143.06ms
step:101/3242 train_loss:5.0006 train_time:13018ms step_avg:143.05ms
step:102/3242 train_loss:4.8886 train_time:13161ms step_avg:143.06ms
step:103/3242 train_loss:5.0158 train_time:13306ms step_avg:143.07ms
step:104/3242 train_loss:4.9623 train_time:13449ms step_avg:143.07ms
step:105/3242 train_loss:4.8369 train_time:13592ms step_avg:143.07ms
step:106/3242 train_loss:4.8834 train_time:13735ms step_avg:143.08ms
step:107/3242 train_loss:5.0613 train_time:13879ms step_avg:143.08ms
step:108/3242 train_loss:4.8798 train_time:14023ms step_avg:143.09ms
step:109/3242 train_loss:4.6789 train_time:14167ms step_avg:143.10ms
step:110/3242 train_loss:4.8389 train_time:14310ms step_avg:143.10ms
step:111/3242 train_loss:4.8364 train_time:14454ms step_avg:143.11ms
step:112/3242 train_loss:4.7936 train_time:14597ms step_avg:143.11ms
step:113/3242 train_loss:4.9118 train_time:14740ms step_avg:143.11ms
step:114/3242 train_loss:4.8216 train_time:14885ms step_avg:143.12ms
step:115/3242 train_loss:4.6913 train_time:15030ms step_avg:143.14ms
step:116/3242 train_loss:4.8353 train_time:15174ms step_avg:143.15ms
step:117/3242 train_loss:4.7529 train_time:15316ms step_avg:143.14ms
step:118/3242 train_loss:4.6948 train_time:15460ms step_avg:143.15ms
step:119/3242 train_loss:4.8631 train_time:15603ms step_avg:143.15ms
step:120/3242 train_loss:4.7854 train_time:15746ms step_avg:143.15ms
step:121/3242 train_loss:4.6873 train_time:15890ms step_avg:143.15ms
step:122/3242 train_loss:4.6209 train_time:16035ms step_avg:143.17ms
step:123/3242 train_loss:4.7617 train_time:16177ms step_avg:143.16ms
step:124/3242 train_loss:4.6071 train_time:16320ms step_avg:143.16ms
step:125/3242 train_loss:4.9038 train_time:16464ms step_avg:143.16ms
step:125/3242 val_loss:4.7185 train_time:16508ms step_avg:143.54ms
step:126/3242 train_loss:4.7514 train_time:16617ms step_avg:143.25ms
step:127/3242 train_loss:4.7097 train_time:16767ms step_avg:143.31ms
step:128/3242 train_loss:4.7528 train_time:16909ms step_avg:143.29ms
step:129/3242 train_loss:4.6543 train_time:17052ms step_avg:143.29ms
step:130/3242 train_loss:4.9583 train_time:17194ms step_avg:143.28ms
step:131/3242 train_loss:4.6788 train_time:17338ms step_avg:143.29ms
step:132/3242 train_loss:4.6923 train_time:17482ms step_avg:143.30ms
step:133/3242 train_loss:4.6415 train_time:17627ms step_avg:143.31ms
step:134/3242 train_loss:4.7051 train_time:17773ms step_avg:143.33ms
step:135/3242 train_loss:4.5687 train_time:17919ms step_avg:143.35ms
step:136/3242 train_loss:4.6985 train_time:18062ms step_avg:143.35ms
step:137/3242 train_loss:4.4804 train_time:18206ms step_avg:143.35ms
step:138/3242 train_loss:4.6368 train_time:18348ms step_avg:143.34ms
step:139/3242 train_loss:4.5603 train_time:18490ms step_avg:143.33ms
step:140/3242 train_loss:4.6165 train_time:18633ms step_avg:143.33ms
step:141/3242 train_loss:4.6947 train_time:18779ms step_avg:143.35ms
step:142/3242 train_loss:4.5639 train_time:18924ms step_avg:143.36ms
step:143/3242 train_loss:4.5683 train_time:19067ms step_avg:143.36ms
step:144/3242 train_loss:4.4654 train_time:19210ms step_avg:143.36ms
step:145/3242 train_loss:4.5819 train_time:19353ms step_avg:143.35ms
step:146/3242 train_loss:4.5527 train_time:19496ms step_avg:143.36ms
step:147/3242 train_loss:4.4320 train_time:19640ms step_avg:143.35ms
step:148/3242 train_loss:4.5577 train_time:19785ms step_avg:143.37ms
step:149/3242 train_loss:4.5774 train_time:19931ms step_avg:143.39ms
step:150/3242 train_loss:4.5405 train_time:20075ms step_avg:143.39ms
step:151/3242 train_loss:4.6285 train_time:20219ms step_avg:143.39ms
step:152/3242 train_loss:4.4961 train_time:20362ms step_avg:143.39ms
step:153/3242 train_loss:4.4935 train_time:20505ms step_avg:143.39ms
step:154/3242 train_loss:4.5686 train_time:20648ms step_avg:143.39ms
step:155/3242 train_loss:4.5383 train_time:20793ms step_avg:143.40ms
step:156/3242 train_loss:4.4719 train_time:20937ms step_avg:143.40ms
step:157/3242 train_loss:4.5210 train_time:21081ms step_avg:143.41ms
step:158/3242 train_loss:4.6027 train_time:21223ms step_avg:143.40ms
step:159/3242 train_loss:4.4247 train_time:21367ms step_avg:143.40ms
step:160/3242 train_loss:4.4907 train_time:21509ms step_avg:143.39ms
step:161/3242 train_loss:4.3063 train_time:21652ms step_avg:143.39ms
step:162/3242 train_loss:4.5156 train_time:21796ms step_avg:143.39ms
step:163/3242 train_loss:4.5298 train_time:21941ms step_avg:143.40ms
step:164/3242 train_loss:4.5122 train_time:22086ms step_avg:143.41ms
step:165/3242 train_loss:4.3657 train_time:22228ms step_avg:143.41ms
step:166/3242 train_loss:4.4424 train_time:22370ms step_avg:143.40ms
step:167/3242 train_loss:4.5313 train_time:22514ms step_avg:143.40ms
step:168/3242 train_loss:4.3645 train_time:22659ms step_avg:143.41ms
step:169/3242 train_loss:4.4420 train_time:22801ms step_avg:143.41ms
step:170/3242 train_loss:4.3215 train_time:22945ms step_avg:143.41ms
step:171/3242 train_loss:4.1950 train_time:23089ms step_avg:143.41ms
step:172/3242 train_loss:4.3513 train_time:23232ms step_avg:143.41ms
step:173/3242 train_loss:4.3625 train_time:23375ms step_avg:143.41ms
step:174/3242 train_loss:4.4088 train_time:23518ms step_avg:143.40ms
step:175/3242 train_loss:4.5781 train_time:23663ms step_avg:143.41ms
step:176/3242 train_loss:4.4075 train_time:23806ms step_avg:143.41ms
step:177/3242 train_loss:4.2622 train_time:23950ms step_avg:143.41ms
step:178/3242 train_loss:4.2245 train_time:24092ms step_avg:143.41ms
step:179/3242 train_loss:4.3226 train_time:24235ms step_avg:143.40ms
step:180/3242 train_loss:4.2856 train_time:24379ms step_avg:143.41ms
step:181/3242 train_loss:4.2624 train_time:24522ms step_avg:143.41ms
step:182/3242 train_loss:4.4288 train_time:24665ms step_avg:143.40ms
step:183/3242 train_loss:4.2983 train_time:24808ms step_avg:143.40ms
step:184/3242 train_loss:4.2846 train_time:24951ms step_avg:143.40ms
step:185/3242 train_loss:4.2729 train_time:25094ms step_avg:143.39ms
step:186/3242 train_loss:4.3524 train_time:25239ms step_avg:143.40ms
step:187/3242 train_loss:4.3058 train_time:25384ms step_avg:143.41ms
step:188/3242 train_loss:4.3845 train_time:25527ms step_avg:143.41ms
step:189/3242 train_loss:4.3124 train_time:25786ms step_avg:144.05ms
step:190/3242 train_loss:4.2404 train_time:26059ms step_avg:144.77ms
step:191/3242 train_loss:4.3424 train_time:26199ms step_avg:144.75ms
step:192/3242 train_loss:4.2288 train_time:26340ms step_avg:144.72ms
step:193/3242 train_loss:4.1556 train_time:26480ms step_avg:144.70ms
step:194/3242 train_loss:4.3859 train_time:26622ms step_avg:144.69ms
step:195/3242 train_loss:4.2911 train_time:26764ms step_avg:144.67ms
step:196/3242 train_loss:4.4863 train_time:26906ms step_avg:144.65ms
step:197/3242 train_loss:4.3337 train_time:27058ms step_avg:144.70ms
step:198/3242 train_loss:4.1725 train_time:27203ms step_avg:144.70ms
step:199/3242 train_loss:4.2970 train_time:27346ms step_avg:144.69ms
step:200/3242 train_loss:4.1521 train_time:27489ms step_avg:144.68ms
step:201/3242 train_loss:4.2495 train_time:27630ms step_avg:144.66ms
step:202/3242 train_loss:4.1249 train_time:27773ms step_avg:144.65ms
step:203/3242 train_loss:4.3579 train_time:27917ms step_avg:144.65ms
step:204/3242 train_loss:4.1998 train_time:28064ms step_avg:144.66ms
step:205/3242 train_loss:4.3094 train_time:28208ms step_avg:144.66ms
step:206/3242 train_loss:4.3680 train_time:28351ms step_avg:144.65ms
step:207/3242 train_loss:4.0671 train_time:28492ms step_avg:144.63ms
step:208/3242 train_loss:4.2109 train_time:28634ms step_avg:144.62ms
step:209/3242 train_loss:4.2030 train_time:28777ms step_avg:144.61ms
step:210/3242 train_loss:4.3542 train_time:28922ms step_avg:144.61ms
step:211/3242 train_loss:4.2866 train_time:29068ms step_avg:144.61ms
step:212/3242 train_loss:4.1751 train_time:29211ms step_avg:144.61ms
step:213/3242 train_loss:4.1976 train_time:29354ms step_avg:144.60ms
step:214/3242 train_loss:4.1553 train_time:29497ms step_avg:144.59ms
step:215/3242 train_loss:4.2226 train_time:29641ms step_avg:144.59ms
step:216/3242 train_loss:4.0508 train_time:29784ms step_avg:144.58ms
step:217/3242 train_loss:4.1125 train_time:29927ms step_avg:144.58ms
step:218/3242 train_loss:4.1189 train_time:30072ms step_avg:144.58ms
step:219/3242 train_loss:4.1918 train_time:30217ms step_avg:144.58ms
step:220/3242 train_loss:4.1741 train_time:30362ms step_avg:144.58ms
step:221/3242 train_loss:4.1874 train_time:30505ms step_avg:144.58ms
step:222/3242 train_loss:4.2107 train_time:30648ms step_avg:144.57ms
step:223/3242 train_loss:4.1272 train_time:30791ms step_avg:144.56ms
step:224/3242 train_loss:4.0808 train_time:30933ms step_avg:144.55ms
step:225/3242 train_loss:4.4009 train_time:31077ms step_avg:144.55ms
step:226/3242 train_loss:4.0132 train_time:31221ms step_avg:144.54ms
step:227/3242 train_loss:4.0872 train_time:31365ms step_avg:144.54ms
step:228/3242 train_loss:4.0939 train_time:31508ms step_avg:144.53ms
step:229/3242 train_loss:4.2471 train_time:31650ms step_avg:144.52ms
step:230/3242 train_loss:4.0331 train_time:31793ms step_avg:144.51ms
step:231/3242 train_loss:4.1465 train_time:31937ms step_avg:144.51ms
step:232/3242 train_loss:4.0080 train_time:32081ms step_avg:144.51ms
step:233/3242 train_loss:4.0782 train_time:32225ms step_avg:144.50ms
step:234/3242 train_loss:4.2054 train_time:32368ms step_avg:144.50ms
step:235/3242 train_loss:4.1207 train_time:32511ms step_avg:144.49ms
step:236/3242 train_loss:4.0062 train_time:32654ms step_avg:144.49ms
step:237/3242 train_loss:4.1702 train_time:32797ms step_avg:144.48ms
step:238/3242 train_loss:4.1849 train_time:32941ms step_avg:144.48ms
step:239/3242 train_loss:4.0314 train_time:33086ms step_avg:144.48ms
step:240/3242 train_loss:4.1800 train_time:33229ms step_avg:144.47ms
step:241/3242 train_loss:4.2123 train_time:33372ms step_avg:144.47ms
step:242/3242 train_loss:4.0651 train_time:33515ms step_avg:144.46ms
step:243/3242 train_loss:4.2393 train_time:33659ms step_avg:144.46ms
step:244/3242 train_loss:4.1175 train_time:33802ms step_avg:144.45ms
step:245/3242 train_loss:4.1730 train_time:33945ms step_avg:144.45ms
step:246/3242 train_loss:4.2402 train_time:34088ms step_avg:144.44ms
step:247/3242 train_loss:4.1599 train_time:34230ms step_avg:144.43ms
step:248/3242 train_loss:4.0988 train_time:34373ms step_avg:144.42ms
step:249/3242 train_loss:4.2078 train_time:34517ms step_avg:144.42ms
step:250/3242 train_loss:4.0164 train_time:34660ms step_avg:144.42ms
step:250/3242 val_loss:4.1060 train_time:34703ms step_avg:144.59ms
step:251/3242 train_loss:4.0646 train_time:34814ms step_avg:144.46ms
step:252/3242 train_loss:4.1763 train_time:34963ms step_avg:144.48ms
step:253/3242 train_loss:4.2470 train_time:35107ms step_avg:144.47ms
step:254/3242 train_loss:4.0354 train_time:35248ms step_avg:144.46ms
step:255/3242 train_loss:3.9782 train_time:35390ms step_avg:144.45ms
step:256/3242 train_loss:4.1545 train_time:35531ms step_avg:144.44ms
step:257/3242 train_loss:4.0635 train_time:35672ms step_avg:144.42ms
step:258/3242 train_loss:4.0790 train_time:35817ms step_avg:144.43ms
step:259/3242 train_loss:4.0590 train_time:35965ms step_avg:144.44ms
step:260/3242 train_loss:4.1156 train_time:36110ms step_avg:144.44ms
step:261/3242 train_loss:4.1448 train_time:36252ms step_avg:144.43ms
step:262/3242 train_loss:4.1162 train_time:36394ms step_avg:144.42ms
step:263/3242 train_loss:4.0735 train_time:36535ms step_avg:144.41ms
step:264/3242 train_loss:3.9911 train_time:36678ms step_avg:144.40ms
step:265/3242 train_loss:4.0758 train_time:36822ms step_avg:144.40ms
step:266/3242 train_loss:3.9505 train_time:36968ms step_avg:144.41ms
step:267/3242 train_loss:4.0015 train_time:37113ms step_avg:144.41ms
step:268/3242 train_loss:4.0114 train_time:37255ms step_avg:144.40ms
step:269/3242 train_loss:4.0435 train_time:37398ms step_avg:144.39ms
step:270/3242 train_loss:3.9429 train_time:37542ms step_avg:144.39ms
step:271/3242 train_loss:4.1879 train_time:37683ms step_avg:144.38ms
step:272/3242 train_loss:4.0692 train_time:37827ms step_avg:144.38ms
step:273/3242 train_loss:3.9991 train_time:37973ms step_avg:144.38ms
step:274/3242 train_loss:4.0416 train_time:38117ms step_avg:144.38ms
step:275/3242 train_loss:4.1150 train_time:38260ms step_avg:144.38ms
step:276/3242 train_loss:4.1503 train_time:38402ms step_avg:144.37ms
step:277/3242 train_loss:4.3128 train_time:38543ms step_avg:144.35ms
step:278/3242 train_loss:4.1151 train_time:38685ms step_avg:144.35ms
step:279/3242 train_loss:4.1670 train_time:38828ms step_avg:144.34ms
step:280/3242 train_loss:4.0798 train_time:38973ms step_avg:144.34ms
step:281/3242 train_loss:4.2081 train_time:39117ms step_avg:144.34ms
step:282/3242 train_loss:4.0408 train_time:39263ms step_avg:144.35ms
step:283/3242 train_loss:4.0277 train_time:39405ms step_avg:144.34ms
step:284/3242 train_loss:3.9944 train_time:39546ms step_avg:144.33ms
step:285/3242 train_loss:4.1311 train_time:39688ms step_avg:144.32ms
step:286/3242 train_loss:4.1408 train_time:39831ms step_avg:144.32ms
step:287/3242 train_loss:4.1728 train_time:39974ms step_avg:144.31ms
step:288/3242 train_loss:4.0002 train_time:40118ms step_avg:144.31ms
step:289/3242 train_loss:4.1024 train_time:40264ms step_avg:144.32ms
step:290/3242 train_loss:3.9555 train_time:40407ms step_avg:144.31ms
step:291/3242 train_loss:3.9466 train_time:40550ms step_avg:144.30ms
step:292/3242 train_loss:4.0123 train_time:40693ms step_avg:144.30ms
step:293/3242 train_loss:3.9486 train_time:40835ms step_avg:144.30ms
step:294/3242 train_loss:3.9953 train_time:40978ms step_avg:144.29ms
step:295/3242 train_loss:4.0413 train_time:41123ms step_avg:144.29ms
step:296/3242 train_loss:3.9200 train_time:41267ms step_avg:144.29ms
step:297/3242 train_loss:3.9453 train_time:41410ms step_avg:144.29ms
step:298/3242 train_loss:3.9448 train_time:41551ms step_avg:144.28ms
step:299/3242 train_loss:4.0474 train_time:41696ms step_avg:144.28ms
step:300/3242 train_loss:3.9062 train_time:41839ms step_avg:144.27ms
step:301/3242 train_loss:4.0463 train_time:41982ms step_avg:144.27ms
step:302/3242 train_loss:4.0614 train_time:42128ms step_avg:144.27ms
step:303/3242 train_loss:4.0137 train_time:42274ms step_avg:144.28ms
step:304/3242 train_loss:4.0544 train_time:42416ms step_avg:144.27ms
step:305/3242 train_loss:4.0381 train_time:42559ms step_avg:144.27ms
step:306/3242 train_loss:4.5345 train_time:42704ms step_avg:144.27ms
step:307/3242 train_loss:4.0132 train_time:42845ms step_avg:144.26ms
step:308/3242 train_loss:3.9206 train_time:42988ms step_avg:144.26ms
step:309/3242 train_loss:4.0678 train_time:43131ms step_avg:144.25ms
step:310/3242 train_loss:3.9394 train_time:43275ms step_avg:144.25ms
step:311/3242 train_loss:4.1709 train_time:43418ms step_avg:144.25ms
step:312/3242 train_loss:4.0131 train_time:43561ms step_avg:144.24ms
step:313/3242 train_loss:3.9520 train_time:43705ms step_avg:144.24ms
step:314/3242 train_loss:4.0263 train_time:43847ms step_avg:144.23ms
step:315/3242 train_loss:4.1653 train_time:43990ms step_avg:144.23ms
step:316/3242 train_loss:4.0322 train_time:44134ms step_avg:144.23ms
step:317/3242 train_loss:3.8735 train_time:44277ms step_avg:144.22ms
step:318/3242 train_loss:3.9482 train_time:44422ms step_avg:144.23ms
step:319/3242 train_loss:3.9933 train_time:44565ms step_avg:144.22ms
step:320/3242 train_loss:3.9682 train_time:44707ms step_avg:144.22ms
step:321/3242 train_loss:4.0855 train_time:44850ms step_avg:144.21ms
step:322/3242 train_loss:4.0333 train_time:44994ms step_avg:144.21ms
step:323/3242 train_loss:4.0086 train_time:45137ms step_avg:144.21ms
step:324/3242 train_loss:4.0905 train_time:45281ms step_avg:144.21ms
step:325/3242 train_loss:4.0285 train_time:45423ms step_avg:144.20ms
step:326/3242 train_loss:4.0950 train_time:45568ms step_avg:144.20ms
step:327/3242 train_loss:3.9653 train_time:45711ms step_avg:144.20ms
step:328/3242 train_loss:4.4721 train_time:45854ms step_avg:144.19ms
step:329/3242 train_loss:4.1493 train_time:45997ms step_avg:144.19ms
step:330/3242 train_loss:3.8866 train_time:46140ms step_avg:144.19ms
step:331/3242 train_loss:3.8399 train_time:46284ms step_avg:144.19ms
step:332/3242 train_loss:4.0584 train_time:46427ms step_avg:144.18ms
step:333/3242 train_loss:3.9894 train_time:46570ms step_avg:144.18ms
step:334/3242 train_loss:3.9560 train_time:46713ms step_avg:144.18ms
step:335/3242 train_loss:3.9195 train_time:46855ms step_avg:144.17ms
step:336/3242 train_loss:4.0924 train_time:46998ms step_avg:144.17ms
step:337/3242 train_loss:4.0353 train_time:47141ms step_avg:144.16ms
step:338/3242 train_loss:4.5050 train_time:47284ms step_avg:144.16ms
step:339/3242 train_loss:4.0169 train_time:47426ms step_avg:144.15ms
step:340/3242 train_loss:3.9571 train_time:47570ms step_avg:144.15ms
step:341/3242 train_loss:4.0119 train_time:47712ms step_avg:144.15ms
step:342/3242 train_loss:3.9289 train_time:47854ms step_avg:144.14ms
step:343/3242 train_loss:3.8943 train_time:47998ms step_avg:144.14ms
step:344/3242 train_loss:3.9282 train_time:48141ms step_avg:144.14ms
step:345/3242 train_loss:4.0744 train_time:48285ms step_avg:144.13ms
step:346/3242 train_loss:3.9123 train_time:48429ms step_avg:144.13ms
step:347/3242 train_loss:3.8475 train_time:48573ms step_avg:144.13ms
step:348/3242 train_loss:3.8871 train_time:48715ms step_avg:144.13ms
step:349/3242 train_loss:3.9396 train_time:48858ms step_avg:144.12ms
step:350/3242 train_loss:3.9044 train_time:49002ms step_avg:144.12ms
step:351/3242 train_loss:3.6498 train_time:49144ms step_avg:144.12ms
step:352/3242 train_loss:3.9052 train_time:49289ms step_avg:144.12ms
step:353/3242 train_loss:4.2401 train_time:49433ms step_avg:144.12ms
step:354/3242 train_loss:3.7367 train_time:49577ms step_avg:144.12ms
step:355/3242 train_loss:4.0064 train_time:49720ms step_avg:144.12ms
step:356/3242 train_loss:3.8647 train_time:49865ms step_avg:144.12ms
step:357/3242 train_loss:3.9759 train_time:50008ms step_avg:144.12ms
step:358/3242 train_loss:3.8798 train_time:50151ms step_avg:144.11ms
step:359/3242 train_loss:3.9301 train_time:50294ms step_avg:144.11ms
step:360/3242 train_loss:3.9145 train_time:50436ms step_avg:144.10ms
step:361/3242 train_loss:3.5274 train_time:50579ms step_avg:144.10ms
step:362/3242 train_loss:4.1019 train_time:50722ms step_avg:144.10ms
step:363/3242 train_loss:3.9984 train_time:50866ms step_avg:144.10ms
step:364/3242 train_loss:3.9241 train_time:51009ms step_avg:144.09ms
step:365/3242 train_loss:3.8244 train_time:51152ms step_avg:144.09ms
step:366/3242 train_loss:3.9956 train_time:51295ms step_avg:144.09ms
step:367/3242 train_loss:3.9482 train_time:51438ms step_avg:144.08ms
step:368/3242 train_loss:3.9460 train_time:51581ms step_avg:144.08ms
step:369/3242 train_loss:3.9291 train_time:51723ms step_avg:144.07ms
step:370/3242 train_loss:3.8277 train_time:51867ms step_avg:144.07ms
step:371/3242 train_loss:3.9718 train_time:52010ms step_avg:144.07ms
step:372/3242 train_loss:3.8351 train_time:52152ms step_avg:144.07ms
step:373/3242 train_loss:3.7810 train_time:52295ms step_avg:144.06ms
step:374/3242 train_loss:4.0033 train_time:52439ms step_avg:144.06ms
step:375/3242 train_loss:3.9236 train_time:52582ms step_avg:144.06ms
step:375/3242 val_loss:3.9191 train_time:52624ms step_avg:144.18ms
step:376/3242 train_loss:3.8980 train_time:52737ms step_avg:144.09ms
step:377/3242 train_loss:3.9560 train_time:52885ms step_avg:144.10ms
step:378/3242 train_loss:3.8708 train_time:53145ms step_avg:144.42ms
step:379/3242 train_loss:3.9272 train_time:53294ms step_avg:144.43ms
step:380/3242 train_loss:3.9537 train_time:53562ms step_avg:144.76ms
step:381/3242 train_loss:4.0312 train_time:53703ms step_avg:144.75ms
step:382/3242 train_loss:3.9296 train_time:53844ms step_avg:144.74ms
step:383/3242 train_loss:3.8946 train_time:53985ms step_avg:144.73ms
step:384/3242 train_loss:3.8745 train_time:54126ms step_avg:144.72ms
step:385/3242 train_loss:3.9581 train_time:54267ms step_avg:144.71ms