13 lines
225 B
JSON
13 lines
225 B
JSON
{
|
|
"attn_dropout_p": 0.0,
|
|
"d_model": 256,
|
|
"ff_dim": 512,
|
|
"ffn_dropout_p": 0.2,
|
|
"learn_te": true,
|
|
"n_heads": 4,
|
|
"n_layers": 4,
|
|
"resid_dropout_p": 0.2,
|
|
"s1_bits": 10,
|
|
"s2_bits": 10,
|
|
"token_dropout_p": 0.0
|
|
} |