in the latest version of spacy, roberta transformer cannot be initialize due to piece_encoder

[smamidik]

Hi, I'm training transformer+ner pipes. I'm interested in en language only based transformer AKA Roberta.
So I took the example config from spacy-curated-transformers and modified the transformer to the config in en_core_web_trf and ran into the error below, it's complaining the list cannot be split. i didn't dig into the code as to what is vocab_merges object is what is merges implies. I'm not supplying any vocab.
For the transformer model = roberta (v1 and v2), I have tried all combinations of piece_encoder and piecer_encoder_loader and it didn't help.
Any help in debugging this issue is appreciated.

on a side note en_core_web_trf 3.8 has compatibility issues and it would not let install latest spacy-curated-transformers
PACKAGES:

curated-transformers              2.0.1
spacy                             4.0.0.dev3
spacy-curated-transformers        2.1.1
spacy-legacy                      4.0.0.dev1
spacy-loggers                     1.0.5
spacy-lookups-data                1.0.5
transformers                      4.51.3

CONFIG:

[paths]
train = null
dev = null

[system]
gpu_allocator = "pytorch"
seed = 1

[nlp]
lang = "en"
pipeline = ["transformer","ner","doc_cleaner"]
disabled = []
before_creation = null
after_creation = null
after_pipeline_creation = null
batch_size = 1000
tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
vectors = {"@vectors":"spacy.Vectors.v1"}

[components]
[components.doc_cleaner]
factory = "doc_cleaner"

[components.transformer]
factory = "curated_transformer"
all_layer_outputs = True

[components.transformer.model]
@architectures = "spacy-curated-transformers.RobertaTransformer.v1"
vocab_size = 50265
hidden_width = 768
piece_encoder = {"@architectures":"spacy-curated-transformers.BertWordpieceEncoder.v1"}
attention_probs_dropout_prob = 0.1
hidden_act = "gelu"
hidden_dropout_prob = 0.1
intermediate_width = 3072
layer_norm_eps = 0.00001
max_position_embeddings = 514
model_max_length = 512
num_attention_heads = 12
num_hidden_layers = 12
padding_idx = 1
type_vocab_size = 1
torchscript = false
mixed_precision = false
wrapped_listener = null


[components.transformer.model.with_spans]
@architectures = "spacy-curated-transformers.WithStridedSpans.v1"
stride = 96
window = 128

[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers":"spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100

[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = false
nO = null

[components.ner.model.tok2vec]
@architectures = "spacy-curated-transformers.LastTransformerLayerListener.v1"
width = ${components.transformer.model.hidden_width}
upstream = "transformer"
pooling = {"@layers":"reduce_mean.v1"}
grad_factor = 1.0





[corpora]

[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
gold_preproc = false
max_length = 0
limit = 0
augmenter = null

[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
gold_preproc = false
max_length = 0
limit = 0
augmenter = null

[training]
train_corpus = "corpora.train"
dev_corpus = "corpora.dev"
seed = ${system:seed}
gpu_allocator = ${system:gpu_allocator}
dropout = 0.1
accumulate_gradient = 3
patience = 5000
max_epochs = 0
max_steps = 16000
eval_frequency = 5
frozen_components = []
before_to_disk = null
annotating_components = []

[training.before_update]
@callbacks = "spacy-curated-transformers.gradual_transformer_unfreezing.v1"
target_pipes = { "*": 1000}

[training.batcher]
@batchers = "spacy.batch_by_sequence.v1"
get_length = null
size = 2000

[training.logger]
@loggers = "spacy.ConsoleLogger.v1"
progress_bar = true

[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = true
eps = 0.00000001

[training.optimizer.learn_rate]
@schedules = "warmup_linear.v1"
warmup_steps = 5
total_steps = 20
initial_rate = 0.00005

[training.score_weights]
tag_acc = 0.1
pos_acc = 0.1
morph_acc = 0.09
morph_per_feat = null
dep_uas = 0.0
dep_las = 0.29
dep_las_per_type = null
sents_p = null
sents_r = null
sents_f = 0.04
lemma_acc = 0.1
ents_f = 0.29
ents_p = 0.0
ents_r = 0.0
speed = 0.0

[pretraining]

[initialize]
vectors = null

[initialize.components]

[initialize.components.doc_cleaner]

[initialize.components.transformer]

[initialize.components.transformer.encoder_loader]
@model_loaders = "spacy-curated-transformers.HFTransformerEncoderLoader.v1"
name = "roberta-base"
revision = "main"

[initialize.components.transformer.piecer_loader]
@model_loaders = "spacy-curated-transformers.HFPieceEncoderLoader.v1"
name = "roberta-base"
revision = "main"

ERROR

smamidi@spark-spacy-train-a2-ultra-single-20250416232957-m:~$ python -m spacy train ./spacy_train_config.cfg  --paths.train ./train.spacy  --paths.dev ./test.spacy   --training.max_epochs 3 --gpu-id 0
ℹ No output directory provided
ℹ Using GPU: 0

=========================== Initializing pipeline ===========================
Traceback (most recent call last):
  File "<frozen runpy>", line 198, in _run_module_as_main
  File "<frozen runpy>", line 88, in _run_code
  File "/opt/conda/default/lib/python3.11/site-packages/spacy/__main__.py", line 4, in <module>
    setup_cli()
  File "/opt/conda/default/lib/python3.11/site-packages/spacy/cli/_util.py", line 88, in setup_cli
    command(prog_name=COMMAND)
  File "/opt/conda/default/lib/python3.11/site-packages/click/core.py", line 1161, in __call__
    return self.main(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/typer/core.py", line 783, in main
    return _main(
           ^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/typer/core.py", line 225, in _main
    rv = self.invoke(ctx)
         ^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/click/core.py", line 1697, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/click/core.py", line 1443, in invoke
    return ctx.invoke(self.callback, **ctx.params)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/click/core.py", line 788, in invoke
    return __callback(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/typer/main.py", line 683, in wrapper
    return callback(**use_params)  # type: ignore
           ^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/spacy/cli/train.py", line 54, in train_cli
    train(config_path, output_path, use_gpu=use_gpu, overrides=overrides)
  File "/opt/conda/default/lib/python3.11/site-packages/spacy/cli/train.py", line 81, in train
    nlp = init_nlp(config, use_gpu=use_gpu)
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/spacy/training/initialize.py", line 90, in init_nlp
    nlp.initialize(lambda: train_corpus(nlp), sgd=optimizer)
  File "/opt/conda/default/lib/python3.11/site-packages/spacy/language.py", line 1463, in initialize
    proc.initialize(get_examples, nlp=self, **p_settings)
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/pipeline/transformer.py", line 357, in initialize
    self.model.initialize(X=doc_sample)
  File "/opt/conda/default/lib/python3.11/site-packages/thinc/model.py", line 318, in initialize
    self.init(self, X=X, Y=Y)
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/models/architectures.py", line 1164, in transformer_model_init
    model.layers[0].initialize(X, Y)
  File "/opt/conda/default/lib/python3.11/site-packages/thinc/model.py", line 318, in initialize
    self.init(self, X=X, Y=Y)
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/models/with_non_ws_tokens.py", line 95, in with_non_ws_tokens_init
    model.layers[0].initialize(X=_filter_tokens(X)[0] if X is not None else None, Y=Y)
  File "/opt/conda/default/lib/python3.11/site-packages/thinc/model.py", line 318, in initialize
    self.init(self, X=X, Y=Y)
  File "/opt/conda/default/lib/python3.11/site-packages/thinc/layers/chain.py", line 89, in init
    layer.initialize(X=curr_input)
  File "/opt/conda/default/lib/python3.11/site-packages/thinc/model.py", line 318, in initialize
    self.init(self, X=X, Y=Y)
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/tokenization/hf_loader.py", line 39, in __call__
    return _convert_encoder(model, tokenizer)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/tokenization/hf_loader.py", line 63, in _convert_encoder
    return _convert_byte_bpe_encoder(model, tokenizer)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/tokenization/hf_loader.py", line 88, in _convert_byte_bpe_encoder
    merges = [tuple(merge.split(" ")) for merge in vocab_merges["merges"]]
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/default/lib/python3.11/site-packages/spacy_curated_transformers/tokenization/hf_loader.py", line 88, in <listcomp>
    merges = [tuple(merge.split(" ")) for merge in vocab_merges["merges"]]
                    ^^^^^^^^^^^
AttributeError: 'list' object has no attribute 'split'

--update
given the below code in hf_loader, I checked tokernizer , merges is a list of string . not sure where the problem is.

def _convert_byte_bpe_encoder(
    model: Tok2PiecesModelT,
    tokenizer: "transformers.RobertaTokenizerFast",
) -> Tok2PiecesModelT:
    # Seems like we cannot get the vocab file name for a RoBERTa vocabulary? And
    # neither the merges from the fast tokenizer. So we'll get them from the
    # JSON serialization.
    serialized = tokenizer.backend_tokenizer.to_str(True)  # type: ignore
    deserialized = json.loads(serialized)
    vocab_merges = deserialized["model"]
    merges = [tuple(merge.split(" ")) for merge in vocab_merges["merges"]]
    model.attrs["byte_bpe_processor"] = ByteBPEProcessor(vocab_merges["vocab"], merges)
    model.attrs["bos_piece"] = tokenizer.bos_token  # type: ignore
    model.attrs["eos_piece"] = tokenizer.eos_token  # type: ignore
    model.attrs["unk_piece"] = tokenizer.unk_token  # type: ignore

    return model
    ```