Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Unet3d rcp fix v3.1 #327

Merged
merged 7 commits into from
Sep 28, 2023
Merged

Unet3d rcp fix v3.1 #327

Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c2b65c4
[UNET3D] Update RCPs due to !523 and !625
mmarcinkiewicz Apr 24, 2023
868390a
Update "when" fields
mmarcinkiewicz Apr 24, 2023
12d92a6
Update rcps_unet3d.json
mmarcinkiewicz Apr 24, 2023
18ee0bd
Fix RCPs for UNET3D GBS=80
mmarcinkiewicz Aug 3, 2023
a6aa39c
Add total samples instead of epochs for UNET3D RCPs
mmarcinkiewicz Aug 3, 2023
5c07d0c
Merge branch 'master' of https://github.com/mmarcinkiewicz/logging in…
mmarcinkiewicz Sep 27, 2023
7966690
Add total samples instead of epochs for UNET3D RCPs
mmarcinkiewicz Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 80 additions & 47 deletions mlperf_logging/rcp_checker/training_3.0.0/rcps_unet3d.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,23 @@
"opt_weight_decay": 0.0
},
"Epochs to converge": [
3420, 3420, 1440, 2320, 2940, 2240, 2600, 2840, 3320, 2360,
4040, 2920, 3360, 2080, 3060, 2900, 4000, 3120, 2120, 2540,
1880, 2640, 2660, 2160, 1420, 2880, 2360, 2260, 2900, 2640,
2380, 3060, 1880, 2420, 2560, 2580, 2180, 2960, 2480, 2140,
3500, 2420, 2500, 3860, 1620, 2260, 2160, 1280, 2320, 2140,
2580, 3020, 2480, 3300, 2140, 3400, 2940, 2520, 3680, 3380,
3080, 2660, 2980, 2740, 2140, 2140, 3000, 2820, 2960, 2420,
2760, 2940, 3280, 2660, 2200, 1660, 1520, 2320, 2180, 2280,
2960, 2140, 3280, 2980, 3580, 3280, 3420]
574560, 574560, 241920, 389760, 493920, 376320, 436800, 477120, 557760, 396480,
678720, 490560, 564480, 349440, 514080, 487200, 672000, 524160, 356160, 426720,
315840, 443520, 446880, 362880, 238560, 483840, 396480, 379680, 487200, 443520,
399840, 514080, 315840, 406560, 430080, 433440, 366240, 497280, 416640, 359520,
588000, 406560, 420000, 648480, 272160, 379680, 362880, 215040, 389760, 359520,
433440, 507360, 416640, 554400, 359520, 571200, 493920, 423360, 618240, 567840,
517440, 446880, 500640, 460320, 359520, 359520, 504000, 473760, 497280, 406560,
463680, 493920, 551040, 446880, 369600, 278880, 255360, 389760, 366240, 383040,
497280, 359520, 551040, 500640, 601440, 551040, 574560
]
},

"unet3d_ref_32_fp32":
{
"Benchmark": "unet3d",
"Creator": "NVIDIA",
"When": "Prior to 1.0 submission",
"When": "Prior to 3.1 submission",
"Platform": "TBD",
"BS": 32,
"Hyperparams": {
Expand All @@ -45,23 +46,24 @@
"opt_weight_decay": 0.0
},
"Epochs to converge": [
1775, 1955, 1649, 1901, 1793, 1667, 2261, 2333, 2009, 1559,
1523, 1667, 1793, 1901, 1775, 1847, 1667, 1757, 2099, 2081,
1793, 1757, 1847, 2549, 1703, 2585, 2207, 1523, 1847, 1793,
2927, 2189, 1865, 1703, 1757, 1901, 1397, 2063, 1703, 1451,
2585, 1757, 1757, 2081, 1793, 2099, 2315, 1541, 1883, 3233,
1901, 2333, 2909, 1793, 2369, 2171, 2063, 2063, 2441, 2081,
2837, 1955, 2459, 1541, 2153, 1829, 1739, 2171, 1847, 2405,
1739, 2459, 1919, 1469, 1991, 1991, 1487, 1955, 1361, 2711,
1775, 1433, 1901
258720, 389760, 507360, 312480, 682080, 329280, 436800, 245280, 470400, 288960,
426720, 305760, 386400, 362880, 342720, 339360, 359520, 336000, 285600, 319200,
655200, 440160, 372960, 322560, 379680, 453600, 299040, 396480, 329280, 467040,
352800, 366240, 500640, 524160, 379680, 362880, 386400, 201600, 517440, 409920,
336000, 537600, 557760, 446880, 275520, 231840, 349440, 262080, 369600, 507360,
292320, 433440, 399840, 372960, 477120, 299040, 386400, 551040, 416640, 339360,
430080, 372960, 467040, 604800, 366240, 386400, 393120, 406560, 265440, 369600,
292320, 356160, 315840, 332640, 581280, 265440, 534240, 309120, 295680, 443520,
312480, 409920, 372960, 530880, 456960, 342720, 299040, 305760, 490560, 406560,
393120, 379680, 215040, 315840, 339360, 396480, 336000, 339360
]
},

"unet3d_ref_56":
{
"Benchmark": "unet3d",
"Creator": "NVIDIA",
"When": "At 1.0 submission",
"When": "Prior to 3.1 submission",
"Platform": "TBD",
"BS": 56,
"Hyperparams": {
Expand All @@ -74,23 +76,52 @@
"opt_weight_decay": 0.0
},
"Epochs to converge": [
1780, 2080, 2780, 2640, 2020, 2420, 2820, 2320, 1720, 2280,
1760, 2420, 2000, 1280, 3120, 1880, 1740, 2200, 1760, 2340,
2180, 2120, 1900, 2200, 2020, 1660, 2040, 2220, 2420, 2100,
2560, 2700, 3040, 1580, 1360, 3200, 2100, 1540, 1800, 1720,
1720, 3300, 2860, 2540, 2880, 2860, 1760, 2560, 2480, 2240,
2340, 1540, 2340, 2600, 2160, 2560, 2980, 2460, 2360, 3020,
1620, 3040, 2780, 2600, 1540, 1640, 1780, 1320, 1480, 3460,
1740, 1660, 2280, 2000, 2320, 2360, 1680, 2200, 1880, 2180,
1740, 2240, 2280, 2180, 2880, 3040, 1760, 1720
436800, 268800, 376320, 530880, 524160, 420000, 342720, 416640, 473760, 420000,
436800, 450240, 517440, 305760, 322560, 319200, 305760, 480480, 446880, 362880,
241920, 295680, 480480, 356160, 456960, 265440, 299040, 409920, 339360, 305760,
658560, 329280, 409920, 409920, 339360, 262080, 272160, 339360, 319200, 406560,
315840, 443520, 329280, 507360, 446880, 551040, 265440, 631680, 285600, 426720,
319200, 356160, 265440, 342720, 567840, 285600, 288960, 440160, 487200, 332640,
275520, 514080, 376320, 359520, 618240, 372960, 420000, 245280, 336000, 278880,
500640, 359520, 299040, 520800, 393120, 332640, 278880, 483840, 352800, 349440
]
},

"unet3d_ref_64":
{
"Benchmark": "unet3d",
"Creator": "NVIDIA",
"When": "Prior to 3.1 submission",
"Platform": "TBD",
"BS": 64,
"Hyperparams": {
"opt_base_learning_rate": 2.0,
"opt_momentum": 0.9,
"opt_learning_rate_warmup_epochs": 1000,
"opt_initial_learning_rate": 1e-4,
"opt_learning_rate_decay_boundary_epochs": [],
"opt_learning_rate_decay_factor": 1.0,
"opt_weight_decay": 0.0
},
"Epochs to converge": [
306304, 430720, 454912, 506752, 347776, 548224, 409984, 399616, 568960, 340864,
461824, 441088, 489472, 489472, 382336, 403072, 448000, 378880, 389248, 403072,
333952, 468736, 437632, 295936, 530944, 520576, 330496, 458368, 399616, 479104,
486016, 261376, 399616, 534400, 458368, 385792, 392704, 544768, 344320, 562048,
534400, 365056, 344320, 306304, 361600, 472192, 368512, 413440, 551680, 385792,
461824, 354688, 551680, 434176, 541312, 295936, 389248, 316672, 434176, 378880,
482560, 375424, 548224, 368512, 340864, 344320, 406528, 492928, 389248, 517120,
472192, 506752, 365056, 399616, 354688, 475648, 327040, 451456, 392704, 534400,
454912, 382336, 392704, 323584, 572416, 468736, 396160, 465280, 323584, 396160,
292480, 233728, 568960, 406528, 365056, 347776, 389248, 330496, 354688, 368512
]
},

"unet3d_ref_80":
{
"Benchmark": "unet3d",
"Creator": "NVIDIA",
"When": "At 1.0 submission",
"When": "Prior to 3.1 submission",
"Platform": "TBD",
"BS": 80,
"Hyperparams": {
Expand All @@ -103,22 +134,22 @@
"opt_weight_decay": 0.0
},
"Epochs to converge": [
1428, 1862, 1988, 1400, 1582, 1344, 1540, 1176, 1484, 1302,
1582, 1694, 1694, 1848, 1274, 1456, 1414, 1582, 2002, 1148,
1372, 1428, 1876, 1540, 1470, 1442, 1148, 1918, 1372, 2492,
1736, 1862, 1288, 1680, 2086, 1708, 1708, 1834, 2450, 1330,
1764, 1162, 1498, 1792, 1820, 1386, 1400, 1302, 1540, 1680,
2226, 1582, 2002, 1484, 1288, 1512, 1792, 1540, 1960, 1806,
1442, 1330, 1540, 1498, 2002, 2100, 1344, 1596, 1862, 2226,
1400, 1344, 1708, 1372, 1484, 1512, 1918, 1260, 2016, 1470
342720, 446880, 477120, 336000, 379680, 322560, 369600, 282240, 356160, 312480,
379680, 406560, 406560, 443520, 305760, 349440, 339360, 379680, 480480, 275520,
329280, 342720, 450240, 369600, 352800, 346080, 275520, 460320, 329280, 598080,
416640, 446880, 309120, 403200, 500640, 409920, 409920, 440160, 588000, 319200,
423360, 278880, 359520, 430080, 436800, 332640, 336000, 312480, 369600, 403200,
534240, 379680, 480480, 356160, 309120, 362880, 430080, 369600, 470400, 433440,
346080, 319200, 369600, 359520, 480480, 504000, 322560, 383040, 446880, 534240,
336000, 322560, 409920, 329280, 356160, 362880, 460320, 302400, 483840, 352800
]
},

"unet3d_ref_84":
{
"Benchmark": "unet3d",
"Creator": "NVIDIA",
"When": "At 1.1 submission",
"When": "Prior to 3.1 submission",
"Platform": "TBD",
"BS": 84,
"Hyperparams": {
Expand All @@ -131,14 +162,16 @@
"opt_weight_decay": 0.0
},
"Epochs to converge": [
2500, 2420, 2500, 2440, 2160, 2020, 2340, 2900, 2760, 1980,
2380, 1900, 2420, 1660, 2600, 1800, 1840, 2140, 2180, 2020,
2280, 2820, 2060, 2380, 2180, 2960, 1960, 2200, 2700, 1880,
2180, 1720, 2800, 2220, 2440, 2360, 2340, 1940, 2040, 2280,
1960, 2260, 2400, 2680, 2180, 2240, 2240, 2240, 2280, 1740,
1360, 2500, 2260, 2300, 2680, 2120, 2200, 2460, 1880, 2400,
2320, 2120, 1940, 2580, 1520, 2680, 1960, 2040, 2620, 2600,
2140, 2060, 2180, 1620, 2040, 2080, 2580, 1980, 2040, 2480
440160, 507360, 403200, 517440, 295680, 339360, 477120, 470400, 302400, 443520,
423360, 366240, 325920, 349440, 420000, 342720, 544320, 379680, 322560, 440160,
372960, 413280, 288960, 386400, 477120, 215040, 433440, 359520, 473760, 376320,
426720, 379680, 362880, 278880, 403200, 423360, 305760, 409920, 430080, 386400,
336000, 453600, 349440, 416640, 389760, 302400, 336000, 376320, 527520, 275520,
420000, 342720, 362880, 346080, 346080, 305760, 433440, 416640, 383040, 507360,
325920, 423360, 540960, 423360, 366240, 507360, 399840, 332640, 329280, 346080,
319200, 399840, 336000, 302400, 420000, 265440, 362880, 456960, 544320, 470400,
362880, 359520, 325920, 493920, 366240, 490560, 299040, 332640, 319200, 376320,
339360, 403200
]
}

Expand Down
Loading