You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1846 lines
29 KiB
1846 lines
29 KiB
{
|
|
"num_cuts": 20,
|
|
"valid_buckets": [
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 6,
|
|
"suggested_batch_size": 9893,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 6,
|
|
"suggested_batch_size": 9893,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 2,
|
|
"num_shards": 6,
|
|
"suggested_batch_size": 9893,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 3,
|
|
"num_shards": 6,
|
|
"suggested_batch_size": 9893,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 4,
|
|
"num_shards": 6,
|
|
"suggested_batch_size": 9893,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 5,
|
|
"num_shards": 6,
|
|
"suggested_batch_size": 9893,
|
|
"num_samples": 203563
|
|
},
|
|
{
|
|
"src_range": [
|
|
7,
|
|
10
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 8676,
|
|
"num_samples": 340378
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
2,
|
|
6
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 7610,
|
|
"num_samples": 60712
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 7338,
|
|
"num_samples": 246093
|
|
},
|
|
{
|
|
"src_range": [
|
|
7,
|
|
10
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 6648,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
7,
|
|
10
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 6648,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
7,
|
|
10
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 2,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 6648,
|
|
"num_samples": 349480
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 5736,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 5736,
|
|
"num_samples": 205922
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
6,
|
|
9
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 5460,
|
|
"num_samples": 125173
|
|
},
|
|
{
|
|
"src_range": [
|
|
3,
|
|
7
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 5768,
|
|
"num_samples": 40896
|
|
},
|
|
{
|
|
"src_range": [
|
|
7,
|
|
10
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 5323,
|
|
"num_samples": 381643
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 4885,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 4885,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 2,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 4885,
|
|
"num_samples": 301534
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 4600,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 4600,
|
|
"num_samples": 298824
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 4265,
|
|
"num_samples": 196251
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
9,
|
|
12
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 4096,
|
|
"num_samples": 50917
|
|
},
|
|
{
|
|
"src_range": [
|
|
7,
|
|
10
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 4431,
|
|
"num_samples": 72695
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 4132,
|
|
"num_samples": 375924
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 3777,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 3777,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 2,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 3777,
|
|
"num_samples": 245160
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 3713,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 3713,
|
|
"num_samples": 334015
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 3456,
|
|
"num_samples": 262724
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
12,
|
|
15
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 3366,
|
|
"num_samples": 83908
|
|
},
|
|
{
|
|
"src_range": [
|
|
10,
|
|
13
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 3620,
|
|
"num_samples": 78134
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 3412,
|
|
"num_samples": 363728
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 3296,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 3296,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 2,
|
|
"num_shards": 3,
|
|
"suggested_batch_size": 3296,
|
|
"num_samples": 158791
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 3072,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 3072,
|
|
"num_samples": 333461
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2924,
|
|
"num_samples": 304005
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2711,
|
|
"num_samples": 116917
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
15,
|
|
18
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2688,
|
|
"num_samples": 38784
|
|
},
|
|
{
|
|
"src_range": [
|
|
13,
|
|
16
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 3072,
|
|
"num_samples": 95381
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2882,
|
|
"num_samples": 330392
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2704,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2704,
|
|
"num_samples": 342321
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2607,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2607,
|
|
"num_samples": 286879
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2561,
|
|
"num_samples": 323796
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 142503
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
18,
|
|
21
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 52991
|
|
},
|
|
{
|
|
"src_range": [
|
|
16,
|
|
19
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2620,
|
|
"num_samples": 105371
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2531,
|
|
"num_samples": 313708
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 239773
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 229969
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2207,
|
|
"num_samples": 320096
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2116,
|
|
"num_samples": 153611
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
21,
|
|
24
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1948,
|
|
"num_samples": 74537
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2275,
|
|
"num_samples": 112513
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2186,
|
|
"num_samples": 265260
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2023,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2023,
|
|
"num_samples": 153848
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2048,
|
|
"num_samples": 262144
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 1,
|
|
"num_shards": 2,
|
|
"suggested_batch_size": 2048,
|
|
"num_samples": 162349
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1865,
|
|
"num_samples": 294639
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1792,
|
|
"num_samples": 184325
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
24,
|
|
27
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1706,
|
|
"num_samples": 62132
|
|
},
|
|
{
|
|
"src_range": [
|
|
19,
|
|
22
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2078,
|
|
"num_samples": 36826
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 2048,
|
|
"num_samples": 103902
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1935,
|
|
"num_samples": 224578
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1778,
|
|
"num_samples": 344236
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1792,
|
|
"num_samples": 353912
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1709,
|
|
"num_samples": 319859
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1706,
|
|
"num_samples": 140480
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
27,
|
|
30
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1647,
|
|
"num_samples": 51709
|
|
},
|
|
{
|
|
"src_range": [
|
|
22,
|
|
25
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1784,
|
|
"num_samples": 36371
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1802,
|
|
"num_samples": 93056
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1703,
|
|
"num_samples": 187649
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1703,
|
|
"num_samples": 274758
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1675,
|
|
"num_samples": 368934
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1572,
|
|
"num_samples": 240236
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1515,
|
|
"num_samples": 108486
|
|
},
|
|
{
|
|
"src_range": [
|
|
46,
|
|
51
|
|
],
|
|
"tgt_range": [
|
|
30,
|
|
33
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1364,
|
|
"num_samples": 45152
|
|
},
|
|
{
|
|
"src_range": [
|
|
25,
|
|
28
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1706,
|
|
"num_samples": 34732
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1698,
|
|
"num_samples": 80245
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1633,
|
|
"num_samples": 153742
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1536,
|
|
"num_samples": 294532
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1404,
|
|
"num_samples": 284588
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1364,
|
|
"num_samples": 178019
|
|
},
|
|
{
|
|
"src_range": [
|
|
46,
|
|
51
|
|
],
|
|
"tgt_range": [
|
|
33,
|
|
36
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1321,
|
|
"num_samples": 92992
|
|
},
|
|
{
|
|
"src_range": [
|
|
28,
|
|
31
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1536,
|
|
"num_samples": 37615
|
|
},
|
|
{
|
|
"src_range": [
|
|
31,
|
|
34
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1397,
|
|
"num_samples": 81139
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1440,
|
|
"num_samples": 207002
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1354,
|
|
"num_samples": 300853
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1279,
|
|
"num_samples": 285080
|
|
},
|
|
{
|
|
"src_range": [
|
|
46,
|
|
51
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1294,
|
|
"num_samples": 215477
|
|
},
|
|
{
|
|
"src_range": [
|
|
51,
|
|
57
|
|
],
|
|
"tgt_range": [
|
|
36,
|
|
40
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1217,
|
|
"num_samples": 93366
|
|
},
|
|
{
|
|
"src_range": [
|
|
34,
|
|
38
|
|
],
|
|
"tgt_range": [
|
|
40,
|
|
44
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1344,
|
|
"num_samples": 71380
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
40,
|
|
44
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1308,
|
|
"num_samples": 147421
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
40,
|
|
44
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1248,
|
|
"num_samples": 215979
|
|
},
|
|
{
|
|
"src_range": [
|
|
46,
|
|
51
|
|
],
|
|
"tgt_range": [
|
|
40,
|
|
44
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1208,
|
|
"num_samples": 260457
|
|
},
|
|
{
|
|
"src_range": [
|
|
51,
|
|
57
|
|
],
|
|
"tgt_range": [
|
|
40,
|
|
44
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1161,
|
|
"num_samples": 171544
|
|
},
|
|
{
|
|
"src_range": [
|
|
57,
|
|
64
|
|
],
|
|
"tgt_range": [
|
|
40,
|
|
44
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1080,
|
|
"num_samples": 62525
|
|
},
|
|
{
|
|
"src_range": [
|
|
38,
|
|
42
|
|
],
|
|
"tgt_range": [
|
|
44,
|
|
49
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1145,
|
|
"num_samples": 60061
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
44,
|
|
49
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1152,
|
|
"num_samples": 122161
|
|
},
|
|
{
|
|
"src_range": [
|
|
46,
|
|
51
|
|
],
|
|
"tgt_range": [
|
|
44,
|
|
49
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1104,
|
|
"num_samples": 231107
|
|
},
|
|
{
|
|
"src_range": [
|
|
51,
|
|
57
|
|
],
|
|
"tgt_range": [
|
|
44,
|
|
49
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1023,
|
|
"num_samples": 260571
|
|
},
|
|
{
|
|
"src_range": [
|
|
57,
|
|
64
|
|
],
|
|
"tgt_range": [
|
|
44,
|
|
49
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1023,
|
|
"num_samples": 146568
|
|
},
|
|
{
|
|
"src_range": [
|
|
64,
|
|
73
|
|
],
|
|
"tgt_range": [
|
|
44,
|
|
49
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 960,
|
|
"num_samples": 47280
|
|
},
|
|
{
|
|
"src_range": [
|
|
42,
|
|
46
|
|
],
|
|
"tgt_range": [
|
|
49,
|
|
55
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1050,
|
|
"num_samples": 37630
|
|
},
|
|
{
|
|
"src_range": [
|
|
46,
|
|
51
|
|
],
|
|
"tgt_range": [
|
|
49,
|
|
55
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 1027,
|
|
"num_samples": 104480
|
|
},
|
|
{
|
|
"src_range": [
|
|
51,
|
|
57
|
|
],
|
|
"tgt_range": [
|
|
49,
|
|
55
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 971,
|
|
"num_samples": 209505
|
|
},
|
|
{
|
|
"src_range": [
|
|
57,
|
|
64
|
|
],
|
|
"tgt_range": [
|
|
49,
|
|
55
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 938,
|
|
"num_samples": 219416
|
|
},
|
|
{
|
|
"src_range": [
|
|
64,
|
|
73
|
|
],
|
|
"tgt_range": [
|
|
49,
|
|
55
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 864,
|
|
"num_samples": 120140
|
|
},
|
|
{
|
|
"src_range": [
|
|
51,
|
|
57
|
|
],
|
|
"tgt_range": [
|
|
55,
|
|
62
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 903,
|
|
"num_samples": 79331
|
|
},
|
|
{
|
|
"src_range": [
|
|
57,
|
|
64
|
|
],
|
|
"tgt_range": [
|
|
55,
|
|
62
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 815,
|
|
"num_samples": 164553
|
|
},
|
|
{
|
|
"src_range": [
|
|
64,
|
|
73
|
|
],
|
|
"tgt_range": [
|
|
55,
|
|
62
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 767,
|
|
"num_samples": 183494
|
|
},
|
|
{
|
|
"src_range": [
|
|
73,
|
|
85
|
|
],
|
|
"tgt_range": [
|
|
55,
|
|
62
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 751,
|
|
"num_samples": 81554
|
|
},
|
|
{
|
|
"src_range": [
|
|
57,
|
|
64
|
|
],
|
|
"tgt_range": [
|
|
62,
|
|
71
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 758,
|
|
"num_samples": 55379
|
|
},
|
|
{
|
|
"src_range": [
|
|
64,
|
|
73
|
|
],
|
|
"tgt_range": [
|
|
62,
|
|
71
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 739,
|
|
"num_samples": 138294
|
|
},
|
|
{
|
|
"src_range": [
|
|
73,
|
|
85
|
|
],
|
|
"tgt_range": [
|
|
62,
|
|
71
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 661,
|
|
"num_samples": 150243
|
|
},
|
|
{
|
|
"src_range": [
|
|
85,
|
|
102
|
|
],
|
|
"tgt_range": [
|
|
62,
|
|
71
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 587,
|
|
"num_samples": 47918
|
|
},
|
|
{
|
|
"src_range": [
|
|
64,
|
|
73
|
|
],
|
|
"tgt_range": [
|
|
71,
|
|
82
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 644,
|
|
"num_samples": 35458
|
|
},
|
|
{
|
|
"src_range": [
|
|
73,
|
|
85
|
|
],
|
|
"tgt_range": [
|
|
71,
|
|
82
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 609,
|
|
"num_samples": 102402
|
|
},
|
|
{
|
|
"src_range": [
|
|
85,
|
|
102
|
|
],
|
|
"tgt_range": [
|
|
71,
|
|
82
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 543,
|
|
"num_samples": 99900
|
|
},
|
|
{
|
|
"src_range": [
|
|
85,
|
|
102
|
|
],
|
|
"tgt_range": [
|
|
82,
|
|
99
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 480,
|
|
"num_samples": 84894
|
|
},
|
|
{
|
|
"src_range": [
|
|
102,
|
|
129
|
|
],
|
|
"tgt_range": [
|
|
82,
|
|
99
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 405,
|
|
"num_samples": 65260
|
|
},
|
|
{
|
|
"src_range": [
|
|
102,
|
|
129
|
|
],
|
|
"tgt_range": [
|
|
99,
|
|
128
|
|
],
|
|
"shard_idx": 0,
|
|
"num_shards": 1,
|
|
"suggested_batch_size": 312,
|
|
"num_samples": 57765
|
|
}
|
|
],
|
|
"min_samples": 32768,
|
|
"max_samples": 262144,
|
|
"total_samples": 24949534,
|
|
"total_original_samples": 25872309,
|
|
"discarded_samples": 922775,
|
|
"total_padding": 73671652,
|
|
"total_actual_tokens": 1292478969
|
|
}
|