multimeditron.dataset.preprocessor package

Submodules

multimeditron.dataset.preprocessor.python module

class multimeditron.dataset.preprocessor.python.PythonFilterProcessor

Bases: BaseDatasetPreprocessor

name = 'python-filter'

class multimeditron.dataset.preprocessor.python.PythonProcessor

Bases: BaseDatasetPreprocessor

name = 'python'

multimeditron.dataset.preprocessor.shuffle module

class multimeditron.dataset.preprocessor.shuffle.ShuffleProcessor

Bases: BaseDatasetPreprocessor

name = 'shuffle'

Module contents

class multimeditron.dataset.preprocessor.AutoDatasetPreprocessor

Bases: object

classmethod get(name: str) → BaseDatasetPreprocessor

classmethod register(name: str)

class multimeditron.dataset.preprocessor.BaseDatasetPreprocessor

Bases: ABC

process(ds: Dataset, num_processes: int, **kwargs) → Dataset

class multimeditron.dataset.preprocessor.PythonFilterProcessor

Bases: BaseDatasetPreprocessor

name = 'python-filter'

class multimeditron.dataset.preprocessor.PythonProcessor

Bases: BaseDatasetPreprocessor

name = 'python'

class multimeditron.dataset.preprocessor.ShuffleProcessor

Bases: BaseDatasetPreprocessor

name = 'shuffle'

multimeditron.dataset.preprocessor.run_preprocessors(ds: Dataset, num_processes: int, processors: list) → Dataset