Show Code 
import numpy as np
import importlib
import model_functions
importlib.reload(model_functions)

from model_functions import MyDeepModel, NetworkInspector

X = np.array([[1, 2, 3], [7, 5, 3], [9, 6, 4]], dtype=float)
Y = np.array([10, 28, 40], dtype=float)

model = MyDeepModel(layer_sizes=[X.shape[1], 5, 1], alpha=0.001, epochs=1000)  # ✅ more epochs
model.initializer = 'he'  
model.fit(X, Y)

y_pred = model.predict(X)
print(f"Prediction: {y_pred}")
print(f"Targets:    {Y}")
Epoch 0: Loss = 152.72494119066516
Epoch 10: Loss = 8.29751356002957
Epoch 20: Loss = 5.914366845569869
Epoch 30: Loss = 4.542595123463403
Epoch 40: Loss = 3.7160852444694776
Epoch 50: Loss = 3.207687072145982
Epoch 60: Loss = 2.8901106844354154
Epoch 70: Loss = 2.6887467527462383
Epoch 80: Loss = 2.558681408887955
Epoch 90: Loss = 2.472478571481533
Epoch 100: Loss = 2.4200592158887435
Epoch 110: Loss = 2.381671166239418
Epoch 120: Loss = 2.356774304842727
Epoch 130: Loss = 2.3545357305072243
Epoch 140: Loss = 2.306884854851161
Epoch 150: Loss = 2.3098658065705133
Epoch 160: Loss = 2.26815134073914
Epoch 170: Loss = 2.2475511957502197
Epoch 180: Loss = 2.2354253689714585
Epoch 190: Loss = 2.215241857182004
Epoch 200: Loss = 2.197262101449963
Epoch 210: Loss = 2.18670191240526
Epoch 220: Loss = 2.166998380539455
Epoch 230: Loss = 2.149878547933978
Epoch 240: Loss = 2.1331571974528036
Epoch 250: Loss = 2.1218056117361566
Epoch 260: Loss = 2.1037531234634006
Epoch 270: Loss = 2.0871867703438145
Epoch 280: Loss = 2.0708370659000805
Epoch 290: Loss = 2.069704753970347
Epoch 300: Loss = 2.0419016659767135
Epoch 310: Loss = 2.0254648847373695
Epoch 320: Loss = 2.009253893236116
Epoch 330: Loss = 1.993156317009733
Epoch 340: Loss = 1.9829761126170855
Epoch 350: Loss = 1.9643544936134958
Epoch 360: Loss = 1.9481700684770284
Epoch 370: Loss = 1.9321282475067958
Epoch 380: Loss = 1.9161699093727151
Epoch 390: Loss = 1.9042039545259337
Epoch 400: Loss = 1.8875077737134547
Epoch 410: Loss = 1.8714789548692792
Epoch 420: Loss = 1.8555625653276806
Epoch 430: Loss = 1.8642117722516989
Epoch 440: Loss = 1.8273686849996522
Epoch 450: Loss = 1.8111994851270043
Epoch 460: Loss = 1.7953001715106776
Epoch 470: Loss = 1.7794969954588424
Epoch 480: Loss = 1.7701836845687602
Epoch 490: Loss = 1.7512951201155929
Epoch 500: Loss = 1.7353764850254982
Epoch 510: Loss = 1.7196072645000342
Epoch 520: Loss = 1.704462851417584
Epoch 530: Loss = 1.691970749444638
Epoch 540: Loss = 1.675795365731369
Epoch 550: Loss = 1.6600531904183102
Epoch 560: Loss = 1.6444207230793733
Epoch 570: Loss = 1.6345400409141548
Epoch 580: Loss = 1.616576107583957
Epoch 590: Loss = 1.600847924529887
Epoch 600: Loss = 1.5852745092769875
Epoch 610: Loss = 1.5878611365762705
Epoch 620: Loss = 1.5578590257811493
Epoch 630: Loss = 1.5420313102416354
Epoch 640: Loss = 1.526517516340711
Epoch 650: Loss = 1.512392287656086
Epoch 660: Loss = 1.4996562831161773
Epoch 670: Loss = 1.483599379319015
Epoch 680: Loss = 1.468163592506239
Epoch 690: Loss = 1.4528581184701128
Epoch 700: Loss = 1.4423633116762122
Epoch 710: Loss = 1.4256227515432902
Epoch 720: Loss = 1.4102702155912787
Epoch 730: Loss = 1.3950782237025348
Epoch 740: Loss = 1.3856344100652347
Epoch 750: Loss = 1.3681289945477026
Epoch 760: Loss = 1.3528775165537843
Epoch 770: Loss = 1.3378195473707046
Epoch 780: Loss = 1.3288244715559225
Epoch 790: Loss = 1.3111510428580002
Epoch 800: Loss = 1.2960367686913823
Epoch 810: Loss = 1.2811377535445123
Epoch 820: Loss = 1.2726281999864355
Epoch 830: Loss = 1.2547871586681085
Epoch 840: Loss = 1.2398260851488734
Epoch 850: Loss = 1.2251059987129873
Epoch 860: Loss = 1.217140068674203
Epoch 870: Loss = 1.1991157725412969
Epoch 880: Loss = 1.1843186835844404
Epoch 890: Loss = 1.1697965343613543
Epoch 900: Loss = 1.161443013533825
Epoch 910: Loss = 1.1441480154425028
Epoch 920: Loss = 1.1295728557099076
Epoch 930: Loss = 1.115275763039753
Epoch 940: Loss = 1.1067158020894279
Epoch 950: Loss = 1.0900145533459407
Epoch 960: Loss = 1.0756780363920022
Epoch 970: Loss = 1.091887820415213
Epoch 980: Loss = 1.0525114214016813
Epoch 990: Loss = 1.0367442147373493
Prediction: [ 9.9701253  29.3859158  38.92927894]
Targets:    [10. 28. 40.]
Show Code 
# Activations for the hidden layer for all exapmples
for i in X:
    print(f"Activations for input {i}: {model.forward_pass(i)[1]}")
Activations for input [1. 2. 3.]: [[0.         0.         4.75458402 0.         4.24851748]]
Activations for input [7. 5. 3.]: [[ 0.          0.         14.78986467  0.01923097  8.11682474]]
Activations for input [9. 6. 4.]: [[ 0.          0.         19.38197683  0.44758179 10.24998548]]

The above indicate that only the second neuron is active for all examples, which is a sign of dead neurons. This is likely due to the small weight initialization and the ReLU activation, which can cause neurons to output zero and never recover. To fix this, we can use He initialization for the weights and increase the number of epochs to allow the model more time to learn.

Invistige what is causing the dead neurons

We saw from the above that the model is only depending on the second neuron to reach the prediction, while all other 4 neurons are inactive. This is the dead neurons issue.

Show Code 
model = MyDeepModel(layer_sizes=[X.shape[1], 5, 1], alpha=0.001, epochs=1000)  
model.initializer = 'OTHER'  
model.fit(X, Y)

y_pred = model.predict(X)
print(f"Prediction: {y_pred}")
print(f"Targets:    {Y}")
Epoch 0: Loss = 828.0101770685595
Epoch 10: Loss = 70.01127809867062
Epoch 20: Loss = 3.030471454103683
Epoch 30: Loss = 2.9898909848607502
Epoch 40: Loss = 2.9567027866211504
Epoch 50: Loss = 2.9284683770635405
Epoch 60: Loss = 2.9035642410427798
Epoch 70: Loss = 2.880908190329658
Epoch 80: Loss = 2.8597786273194425
Epoch 90: Loss = 2.8396942047763947
Epoch 100: Loss = 2.820333641170913
Epoch 110: Loss = 2.8014822813680875
Epoch 120: Loss = 2.7829964849264837
Epoch 130: Loss = 2.764779900285969
Epoch 140: Loss = 2.7467676625945447
Epoch 150: Loss = 2.728915872342759
Epoch 160: Loss = 2.7111945922514367
Epoch 170: Loss = 2.6935831872890024
Epoch 180: Loss = 2.6760672246417125
Epoch 190: Loss = 2.6586364118861234
Epoch 200: Loss = 2.641283225908682
Epoch 210: Loss = 2.6240020012708043
Epoch 220: Loss = 2.6067883240929075
Epoch 230: Loss = 2.589638629052153
Epoch 240: Loss = 2.572549931385393
Epoch 250: Loss = 2.555519648609694
Epoch 260: Loss = 2.5385454818555284
Epoch 270: Loss = 2.5216253368043673
Epoch 280: Loss = 2.5047572709367762
Epoch 290: Loss = 2.4879394582592864
Epoch 300: Loss = 2.4711701656454754
Epoch 310: Loss = 2.4544477368961526
Epoch 320: Loss = 2.437770581933863
Epoch 330: Loss = 2.421137169415473
Epoch 340: Loss = 2.4045460216246637
Epoch 350: Loss = 2.3879957108882923
Epoch 360: Loss = 2.3714848570159277
Epoch 370: Loss = 2.355012125429507
Epoch 380: Loss = 2.3385762257628313
Epoch 390: Loss = 2.3221759107840114
Epoch 400: Loss = 2.3058099755434625
Epoch 410: Loss = 2.2894772566826376
Epoch 420: Loss = 2.27317663186014
Epoch 430: Loss = 2.256907019266002
Epoch 440: Loss = 2.2406673772048387
Epoch 450: Loss = 2.2244567037339227
Epoch 460: Loss = 2.208274036347635
Epoch 470: Loss = 2.1921184517010803
Epoch 480: Loss = 2.1759890653686997
Epoch 490: Loss = 2.159885031633935
Epoch 500: Loss = 2.1438055433074954
Epoch 510: Loss = 2.1277498315717485
Epoch 520: Loss = 2.111717165849535
Epoch 530: Loss = 2.0957068536956105
Epoch 540: Loss = 2.0797182407092674
Epoch 550: Loss = 2.063750710466564
Epoch 560: Loss = 2.047803684470983
Epoch 570: Loss = 2.0318766221209272
Epoch 580: Loss = 2.015969020692945
Epoch 590: Loss = 2.0000804153392107
Epoch 600: Loss = 1.9842103790980248
Epoch 610: Loss = 1.9683585229158602
Epoch 620: Loss = 1.952524495679872
Epoch 630: Loss = 1.9367079842592645
Epoch 640: Loss = 1.9209087135542742
Epoch 650: Loss = 1.9051264465514712
Epoch 660: Loss = 1.889360984383736
Epoch 670: Loss = 1.8736121663938974
Epoch 680: Loss = 1.8578798702002441
Epoch 690: Loss = 1.8421640117626137
Epoch 700: Loss = 1.8264645454478317
Epoch 710: Loss = 1.810781464092561
Epoch 720: Loss = 1.7951147990626957
Epoch 730: Loss = 1.7794646203072133
Epoch 740: Loss = 1.7638310364054932
Epoch 750: Loss = 1.7482141946063592
Epoch 760: Loss = 1.732614280857253
Epoch 770: Loss = 1.7170315198223525
Epoch 780: Loss = 1.70146617488782
Epoch 790: Loss = 1.6859185481527639
Epoch 800: Loss = 1.670388980404504
Epoch 810: Loss = 1.6548778510764768
Epoch 820: Loss = 1.6393855781874578
Epoch 830: Loss = 1.6239126182604295
Epoch 840: Loss = 1.6084594662197607
Epoch 850: Loss = 1.5930266552652759
Epoch 860: Loss = 1.577614756721635
Epoch 870: Loss = 1.562224379861718
Epoch 880: Loss = 1.546856171702717
Epoch 890: Loss = 1.531510816773423
Epoch 900: Loss = 1.5161890368515574
Epoch 910: Loss = 1.5008915906697673
Epoch 920: Loss = 1.4856192735892737
Epoch 930: Loss = 1.4703729172397484
Epoch 940: Loss = 1.4551533891245036
Epoch 950: Loss = 1.4399615921899567
Epoch 960: Loss = 1.4247984643582432
Epoch 970: Loss = 1.4096649780222315
Epoch 980: Loss = 1.3945621395020895
Epoch 990: Loss = 1.3794909884624527
Prediction: [ 9.96936618 29.59798159 38.75948632]
Targets:    [10. 28. 40.]

Comments