Index

_

_dtype (multimeditron.model.modalities.BaseModality attribute) _num_patches_per_entry (multimeditron.model.modalities.image_modality.ImageProcessor attribute) (multimeditron.model.modalities.ImageProcessor attribute)

_registry (multimeditron.dataset.loader.AutoModalityLoader attribute) (multimeditron.model.modalities.AutoModality attribute)

A

add_generation_prompt (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) ALIGNMENT (multimeditron.train.trainer.TrainingMode attribute) apertus() (multimeditron.model.model.ChatTemplate static method)

attachment_token (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) AutoDatasetPreprocessor (class in multimeditron.dataset.preprocessor) AutoModality (class in multimeditron.model.modalities) AutoModalityLoader (class in multimeditron.dataset.loader)

B

base_model_prefix (multimeditron.model.model.MultiModalModelForCausalLM attribute) BaseDatasetPreprocessor (class in multimeditron.dataset.preprocessor) BaseModality (class in multimeditron.model.modalities) BaseModalityConfig (class in multimeditron.model.modalities)

BaseModalityLoader (class in multimeditron.dataset.loader) BaseModalityProcessor (class in multimeditron.model.modalities) bootstrap() (in module multimeditron.model.model) build_datasets() (in module multimeditron.cli.train)

C

chat_template (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) ChatTemplate (class in multimeditron.model.model) clip_name (multimeditron.model.modalities.image_modality.ImageConfig attribute) (multimeditron.model.modalities.ImageConfig attribute) collate_fn() (in module multimeditron.verl) compute_loss() (multimeditron.train.trainer.MultimodalTrainer method) compute_score() (in module multimeditron.verl) compute_token_range() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) config (multimeditron.model.modalities.BaseModality attribute) (multimeditron.model.modalities.BaseModalityProcessor attribute)

config_class (multimeditron.model.modalities.BaseModality attribute) (multimeditron.model.modalities.image_modality.ImageModality attribute) (multimeditron.model.modalities.image_modality_moe.MOEImageModality attribute) (multimeditron.model.modalities.ImageModality attribute) (multimeditron.model.modalities.moe.gating.GatingNetwork attribute) (multimeditron.model.modalities.MOEImageModality attribute) (multimeditron.model.modalities.MOEImageModalityPEP attribute) (multimeditron.model.model.MultiModalModelForCausalLM attribute) config_from_dict() (multimeditron.model.modalities.AutoModality class method) convert_tokens_to_ids() (multimeditron.model.prompt_tokenizers.PromptTokenizer method)

D

DataCollatorForMultimodal (class in multimeditron.model.data_loader)

delimiters (multimeditron.model.model.ChatTemplate attribute)

E

embed_modalities_with_text() (multimeditron.model.model.MultiModalModelForCausalLM method) embedding_size (multimeditron.model.modalities.MOEImageModalityPEP property)

END2END (multimeditron.train.trainer.TrainingMode attribute) ensure_config_empty_dict() (in module multimeditron.cli) expand_attachment_input_tokens() (multimeditron.model.prompt_tokenizers.PromptTokenizer method)

F

FileSystemImageLoader (class in multimeditron.dataset.loader) (class in multimeditron.dataset.loader.image.fs) find_tag_pos() (in module multimeditron.model.prompt_tokenizers) forward() (multimeditron.model.modalities.image_modality.ImageModality method) (multimeditron.model.modalities.image_modality_moe.MOEImageModality method) (multimeditron.model.modalities.ImageModality method) (multimeditron.model.modalities.moe.gating.GatingNetwork method) (multimeditron.model.modalities.MOEImageModality method) (multimeditron.model.modalities.MOEImageModalityPEP method) (multimeditron.model.model.MultiModalModelForCausalLM method) freeze_all() (multimeditron.model.modalities.BaseModality method) freeze_for_alignment() (multimeditron.model.model.MultiModalModelForCausalLM method) freeze_for_end2end() (multimeditron.model.model.MultiModalModelForCausalLM method)

freeze_for_lm() (multimeditron.model.model.MultiModalModelForCausalLM method) freeze_modality_embedder() (multimeditron.model.modalities.BaseModality method) (multimeditron.model.modalities.image_modality.ImageModality method) (multimeditron.model.modalities.image_modality_moe.MOEImageModality method) (multimeditron.model.modalities.ImageModality method) (multimeditron.model.modalities.MOEImageModality method) (multimeditron.model.modalities.MOEImageModalityPEP method) from_dict() (multimeditron.model.modalities.image_modality.ImageModality class method) (multimeditron.model.modalities.ImageModality class method) (multimeditron.model.model.MultimodalConfig class method) from_name() (multimeditron.dataset.loader.AutoModalityLoader class method) (multimeditron.model.model.ChatTemplate static method) from_pretrained() (multimeditron.model.modalities.AutoModality class method) FULL (multimeditron.train.trainer.TrainingMode attribute)

G

GatingNetwork (class in multimeditron.model.modalities.moe.gating) GatingNetworkConfig (class in multimeditron.model.modalities.moe.gating) generate() (multimeditron.model.model.MultiModalModelForCausalLM method) get() (multimeditron.dataset.preprocessor.AutoDatasetPreprocessor class method) get_config() (multimeditron.model.modalities.BaseModality method)

get_input_embeddings() (multimeditron.model.model.MultiModalModelForCausalLM method) get_model() (multimeditron.model.model.MultiModalModelForCausalLM method) get_num_embeddings() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) get_torch_dtype() (in module multimeditron.utils) get_train_dataloader() (multimeditron.train.trainer.MultimodalTrainer method)

H

hidden_size (multimeditron.model.modalities.BaseModalityConfig attribute) (multimeditron.model.modalities.image_modality.ImageConfig attribute) (multimeditron.model.modalities.ImageConfig attribute)

I

image_processor (multimeditron.model.modalities.image_modality.ImageProcessor attribute) (multimeditron.model.modalities.ImageProcessor attribute) ImageConfig (class in multimeditron.model.modalities) (class in multimeditron.model.modalities.image_modality) ImageModality (class in multimeditron.model.modalities) (class in multimeditron.model.modalities.image_modality)

ImageProcessor (class in multimeditron.model.modalities) (class in multimeditron.model.modalities.image_modality) is_dataset_folder() (in module multimeditron.cli.train) is_jsonl() (in module multimeditron.cli.train) is_main_process() (in module multimeditron.cli.train)

J

JSONLGenerator (class in multimeditron.utils.jsonl)

L

llama() (multimeditron.model.model.ChatTemplate static method) LM_ONLY (multimeditron.train.trainer.TrainingMode attribute) load() (multimeditron.dataset.loader.BaseModalityLoader method) (multimeditron.dataset.loader.FileSystemImageLoader method) (multimeditron.dataset.loader.image.bytes.RawImageLoader method) (multimeditron.dataset.loader.image.fs.FileSystemImageLoader method) (multimeditron.dataset.loader.RawImageLoader method)

load_modalities() (multimeditron.dataset.loader.BaseModalityLoader static method)

M

modality_loaders (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) modality_processors (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) modality_type (multimeditron.model.modalities.BaseModalityConfig attribute) model_type (multimeditron.model.modalities.image_modality.ImageConfig attribute) (multimeditron.model.modalities.image_modality_moe.MOEImageConfig attribute) (multimeditron.model.modalities.ImageConfig attribute) (multimeditron.model.modalities.moe.gating.GatingNetworkConfig attribute) (multimeditron.model.modalities.MOEImageConfig attribute) (multimeditron.model.modalities.MOEImageConfigPEP attribute) (multimeditron.model.model.MultimodalConfig attribute) module multimeditron multimeditron.cli multimeditron.cli.preprocess multimeditron.cli.train multimeditron.cli.utils multimeditron.cli.verl multimeditron.config multimeditron.dataset multimeditron.dataset.loader multimeditron.dataset.loader.image multimeditron.dataset.loader.image.bytes multimeditron.dataset.loader.image.fs multimeditron.dataset.preprocessor multimeditron.dataset.preprocessor.python multimeditron.dataset.preprocessor.shuffle multimeditron.dataset.sample_preprocessor multimeditron.model multimeditron.model.constants multimeditron.model.data_loader multimeditron.model.modalities multimeditron.model.modalities.base multimeditron.model.modalities.image_modality multimeditron.model.modalities.image_modality_moe multimeditron.model.modalities.moe multimeditron.model.modalities.moe.gating multimeditron.model.model multimeditron.model.prompt_tokenizers multimeditron.profiling multimeditron.tools multimeditron.train multimeditron.train.trainer multimeditron.utils multimeditron.utils.jsonl multimeditron.verl MOEImageConfig (class in multimeditron.model.modalities) (class in multimeditron.model.modalities.image_modality_moe) MOEImageConfigPEP (class in multimeditron.model.modalities) MOEImageModality (class in multimeditron.model.modalities) (class in multimeditron.model.modalities.image_modality_moe) MOEImageModalityPEP (class in multimeditron.model.modalities) MOEImageProcessor (class in multimeditron.model.modalities) (class in multimeditron.model.modalities.image_modality_moe) MOEImageProcessorPEP (class in multimeditron.model.modalities) multimeditron module multimeditron.cli module multimeditron.cli.preprocess module multimeditron.cli.train module

multimeditron.cli.utils module multimeditron.cli.verl module multimeditron.config module multimeditron.dataset module multimeditron.dataset.loader module multimeditron.dataset.loader.image module multimeditron.dataset.loader.image.bytes module multimeditron.dataset.loader.image.fs module multimeditron.dataset.preprocessor module multimeditron.dataset.preprocessor.python module multimeditron.dataset.preprocessor.shuffle module multimeditron.dataset.sample_preprocessor module multimeditron.model module multimeditron.model.constants module multimeditron.model.data_loader module multimeditron.model.modalities module multimeditron.model.modalities.base module multimeditron.model.modalities.image_modality module multimeditron.model.modalities.image_modality_moe module multimeditron.model.modalities.moe module multimeditron.model.modalities.moe.gating module multimeditron.model.model module multimeditron.model.prompt_tokenizers module multimeditron.profiling module multimeditron.tools module multimeditron.train module multimeditron.train.trainer module multimeditron.utils module multimeditron.utils.jsonl module multimeditron.verl module MultimodalConfig (class in multimeditron.model.model) MultiModalModelForCausalLM (class in multimeditron.model.model) MultimodalTrainer (class in multimeditron.train.trainer)

N

name (multimeditron.dataset.loader.BaseModalityLoader attribute) (multimeditron.dataset.loader.FileSystemImageLoader attribute) (multimeditron.dataset.loader.image.bytes.RawImageLoader attribute) (multimeditron.dataset.loader.image.fs.FileSystemImageLoader attribute) (multimeditron.dataset.loader.RawImageLoader attribute) (multimeditron.dataset.preprocessor.python.PythonFilterProcessor attribute) (multimeditron.dataset.preprocessor.python.PythonProcessor attribute) (multimeditron.dataset.preprocessor.PythonFilterProcessor attribute) (multimeditron.dataset.preprocessor.PythonProcessor attribute) (multimeditron.dataset.preprocessor.shuffle.ShuffleProcessor attribute) (multimeditron.dataset.preprocessor.ShuffleProcessor attribute) (multimeditron.model.model.ChatTemplate attribute)

numpy_call() (multimeditron.model.data_loader.DataCollatorForMultimodal method) NvtxAnnotationCallback (class in multimeditron.profiling)

O

on_epoch_begin() (multimeditron.profiling.NvtxAnnotationCallback method) on_epoch_end() (multimeditron.profiling.NvtxAnnotationCallback method) on_evaluate() (multimeditron.profiling.NvtxAnnotationCallback method) on_forward_begin() (multimeditron.profiling.NvtxAnnotationCallback method) on_forward_end() (multimeditron.profiling.NvtxAnnotationCallback method) on_init_end() (multimeditron.profiling.NvtxAnnotationCallback method) on_log() (multimeditron.profiling.NvtxAnnotationCallback method) on_optimizer_step() (multimeditron.profiling.NvtxAnnotationCallback method) on_pre_optimizer_step() (multimeditron.profiling.NvtxAnnotationCallback method)

on_predict() (multimeditron.profiling.NvtxAnnotationCallback method) on_prediction_step() (multimeditron.profiling.NvtxAnnotationCallback method) on_prepare_inputs_begin() (multimeditron.profiling.NvtxAnnotationCallback method) on_prepare_inputs_end() (multimeditron.profiling.NvtxAnnotationCallback method) on_save() (multimeditron.profiling.NvtxAnnotationCallback method) on_step_begin() (multimeditron.profiling.NvtxAnnotationCallback method) on_step_end() (multimeditron.profiling.NvtxAnnotationCallback method) on_substep_end() (multimeditron.profiling.NvtxAnnotationCallback method) on_train_begin() (multimeditron.profiling.NvtxAnnotationCallback method) on_train_end() (multimeditron.profiling.NvtxAnnotationCallback method)

P

pad_tokenized() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) preprocess_images() (multimeditron.model.modalities.moe.gating.GatingNetwork method) preprocessor_class (multimeditron.model.modalities.BaseModality attribute) (multimeditron.model.modalities.image_modality.ImageModality attribute) (multimeditron.model.modalities.image_modality_moe.MOEImageModality attribute) (multimeditron.model.modalities.ImageModality attribute) (multimeditron.model.modalities.MOEImageModality attribute) (multimeditron.model.modalities.MOEImageModalityPEP attribute) preprocessor_from_name() (multimeditron.model.modalities.AutoModality class method) process() (multimeditron.dataset.preprocessor.BaseDatasetPreprocessor method) (multimeditron.model.modalities.BaseModalityProcessor method) (multimeditron.model.modalities.image_modality.ImageProcessor method) (multimeditron.model.modalities.image_modality_moe.MOEImageProcessor method) (multimeditron.model.modalities.ImageProcessor method) (multimeditron.model.modalities.MOEImageProcessor method) (multimeditron.model.modalities.MOEImageProcessorPEP method)

process_modality_to_tensor() (multimeditron.dataset.sample_preprocessor.SamplePreprocessor method) processor (multimeditron.model.modalities.moe.gating.GatingNetwork attribute) processors() (multimeditron.model.model.MultiModalModelForCausalLM method) projection_type (multimeditron.model.modalities.image_modality.ImageConfig attribute) (multimeditron.model.modalities.ImageConfig attribute) PromptTokenizer (class in multimeditron.model.prompt_tokenizers) PythonFilterProcessor (class in multimeditron.dataset.preprocessor) (class in multimeditron.dataset.preprocessor.python) PythonProcessor (class in multimeditron.dataset.preprocessor) (class in multimeditron.dataset.preprocessor.python)

Q

qwen3() (multimeditron.model.model.ChatTemplate static method)

R

RawImageLoader (class in multimeditron.dataset.loader) (class in multimeditron.dataset.loader.image.bytes) register() (multimeditron.dataset.loader.AutoModalityLoader class method) (multimeditron.dataset.preprocessor.AutoDatasetPreprocessor class method) (multimeditron.model.modalities.AutoModality class method)

replace_between_tags_v2() (in module multimeditron.model.prompt_tokenizers) reset() (multimeditron.utils.jsonl.JSONLGenerator method) resnet (multimeditron.model.modalities.moe.gating.GatingNetwork attribute) return_tensors (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) run_preprocessors() (in module multimeditron.dataset.preprocessor)

S

SamplePreprocessor (class in multimeditron.dataset.sample_preprocessor) set_input_embeddings() (multimeditron.model.model.MultiModalModelForCausalLM method) ShuffleProcessor (class in multimeditron.dataset.preprocessor) (class in multimeditron.dataset.preprocessor.shuffle)

special_tokens (multimeditron.model.model.ChatTemplate attribute) split_host_port() (in module multimeditron.cli.utils) supports_gradient_checkpointing (multimeditron.model.model.MultiModalModelForCausalLM attribute)

T

tf_call() (multimeditron.model.data_loader.DataCollatorForMultimodal method) to_dict() (multimeditron.model.model.MultimodalConfig method) tokenize() (multimeditron.dataset.sample_preprocessor.SamplePreprocessor method) tokenize_conversation() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) tokenize_samples() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) tokenize_text() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) tokenizer (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) (multimeditron.model.modalities.BaseModality attribute)

top_k (multimeditron.model.modalities.moe.gating.GatingNetwork attribute) (multimeditron.model.modalities.moe.gating.GatingNetworkConfig attribute) torch_call() (multimeditron.model.data_loader.DataCollatorForMultimodal method) train() (multimeditron.model.modalities.image_modality_moe.MOEImageModality method) (multimeditron.model.modalities.MOEImageModality method) (multimeditron.model.modalities.MOEImageModalityPEP method) (multimeditron.train.trainer.MultimodalTrainer method) training_step() (multimeditron.train.trainer.MultimodalTrainer method) TrainingMode (class in multimeditron.train.trainer)

U

unfreeze() (multimeditron.model.model.MultiModalModelForCausalLM method) unfreeze_all() (multimeditron.model.modalities.BaseModality method) unfreeze_modality_embedder() (multimeditron.model.modalities.BaseModality method) (multimeditron.model.modalities.image_modality.ImageModality method) (multimeditron.model.modalities.image_modality_moe.MOEImageModality method) (multimeditron.model.modalities.ImageModality method) (multimeditron.model.modalities.MOEImageModality method) (multimeditron.model.modalities.MOEImageModalityPEP method) unfreeze_projection() (multimeditron.model.modalities.BaseModality method) (multimeditron.model.modalities.image_modality.ImageModality method) (multimeditron.model.modalities.image_modality_moe.MOEImageModality method) (multimeditron.model.modalities.ImageModality method) (multimeditron.model.modalities.MOEImageModality method) (multimeditron.model.modalities.MOEImageModalityPEP method)

update_with_token_range() (multimeditron.model.prompt_tokenizers.PromptTokenizer method) use_2d_position_ids (multimeditron.model.data_loader.DataCollatorForMultimodal attribute) (multimeditron.model.modalities.image_modality.ImageConfig attribute) (multimeditron.model.modalities.ImageConfig attribute)

V

validate_tokenized_results() (multimeditron.model.prompt_tokenizers.PromptTokenizer method)

vocab_size (multimeditron.model.prompt_tokenizers.PromptTokenizer property)