105 lines
2.1 KiB
ReStructuredText
105 lines
2.1 KiB
ReStructuredText
datasets package
|
|
================
|
|
|
|
.. mdinclude :: ../../../megatron/core/datasets/readme.md
|
|
|
|
Submodules
|
|
----------
|
|
|
|
datasets.blended\_megatron\_dataset\_config module
|
|
---------------------------------------------------
|
|
|
|
.. automodule:: core.datasets.blended_megatron_dataset_config
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.blended\_megatron\_dataset\_builder module
|
|
---------------------------------------------------
|
|
|
|
.. automodule:: core.datasets.blended_megatron_dataset_builder
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.megatron\_tokenizer module
|
|
-----------------------------------
|
|
|
|
.. automodule:: core.datasets.megatron_tokenizer
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.indexed\_dataset module
|
|
--------------------------------
|
|
|
|
.. automodule:: core.datasets.indexed_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.megatron\_dataset module
|
|
---------------------------------
|
|
|
|
.. automodule:: core.datasets.megatron_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.gpt\_dataset module
|
|
----------------------------
|
|
|
|
.. automodule:: core.datasets.gpt_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.masked\_dataset module
|
|
-------------------------------
|
|
|
|
.. automodule:: core.datasets.masked_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.bert\_dataset module
|
|
-----------------------------
|
|
|
|
.. automodule:: core.datasets.bert_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.t5\_dataset module
|
|
---------------------------
|
|
|
|
.. automodule:: core.datasets.t5_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.blended\_dataset module
|
|
----------------------------------
|
|
|
|
.. automodule:: core.datasets.blended_dataset
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
datasets.utils module
|
|
---------------------
|
|
|
|
.. automodule:: core.datasets.utils
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|
|
Module contents
|
|
---------------
|
|
|
|
.. automodule:: core.datasets
|
|
:members:
|
|
:undoc-members:
|
|
:show-inheritance:
|
|
|