tfds.load(
name: str,
*,
split: Optional[Tree[splits_lib.SplitArg]] = None,
data_dir: Optional[str] = None,
batch_size: tfds.typing.Dim = None,
shuffle_files: bool = False,
download: bool = True,
as_supervised: bool = False,
decoders: Optional[TreeDict[decode.partial_decode.DecoderArg]] = None,
read_config: Optional[tfds.ReadConfig] = None,
with_info: bool = False,
builder_kwargs: Optional[Dict[str, Any]] = None,
download_and_prepare_kwargs: Optional[Dict[str, Any]] = None,
as_dataset_kwargs: Optional[Dict[str, Any]] = None,
try_gcs: bool = False
)
函数说明
tf.load函数相当于同时执行了下面三个函数：
builder = tfds.builder(name, data_dir=data_dir, **builder_kwargs)
builder.download_and_prepare(**download_and_prepare_kwargs)
ds = builder.as_dataset(
split=split,
as_supervised=as_supervised,
shuffle_files=shuffle_files,
read_config=read_config,
decoders=decoders,
**as_dataset_kwargs,
)
参数name表示数据集的名字，参数split表示数据集的分隔方式，参数data_dir表示读写数据集的目录，参数batch_size指定数据集中每一条数据的大小，默认为1。
参数shuffle_files表示是否打乱输入的数据，参数download表示是否将数据集下载到本地，默认为True，如果设置为False，相当于在三个步骤中少了builder.download_and_prepare()这一步。
参数as_supervised为True表示数据集的每一条数据保存为监督学习的方式–2元组（input, label)，如果为False表示数据集的每一条数据保存为字典类型{feature1:input, feature:label}。
参数with_info如果为True，表示返回的数据集为一个元组 (tf.data.Dataset, tfds.core.DatasetInfo)；如果为False，则返回为一个tf.data.Dataset对象。
函数使用这里使用cola数据集作为示例，展示各个参数的作用：
1、split参数
# 正常情况，返回train, validation, test三个数据集，是一个dict类型，
# 键为train,validation,test中的一种，是一个字符串，值为一个PrefetchDataset对象
ds = tfds.load("glue/cola")
# 指定split，返回train, validation, test数据集中的一种和几种
# 情况一：只返回train数据集，是一个PrefetchDataset对象
ds_1 = tfds.load("glue/cola", split="train")
# 情况二：只返回train数据集的前50%，或者前50条
ds_21 = tfds.load("glue/cola", split="train[:50%]")
ds_22 = tfds.load("glue/cola", split="train[:50]")
# 情况三：将train和test数据集的全部或者部分结合起来，并返回一个PrefetchDataset对象
ds_31 = tfds.load("glue/cola", split="train+test")
ds_32 = tfds.load("glue/cola", split="train+validation+test")
ds_33 = tfds.load("glue/cola", split="all")
ds_34 = tfds.load("glue/cola", split="train[:50]+test[:50%]")
# 情况四：返回一个包含train+test, validation两个数据集的列表，每一个元素是一个PrefetchDataset对象
ds_41 = tfds.load("glue/cola", split=["train", "validation"])
ds_42 = tfds.load("glue/cola", split=["train+test", "validation"])
2、data_dir参数
# 指定数据集下载目录，默认下载到环境变量TFDS_DATA_DIR指定的目录
# 这里指定为当前路径的datasets目录，下载的数据文件将会存放到dataset/glue/cola文件夹中
ds = tfds.load("glue/cola", data_dir="datasets")
3、batch_size参数
# 没有指定batch_size时，每个PrefetchDataset中的每一项只由一个数据组成
# 指定batch_size后，每个PrefetchDataset的每一项由多条数据组成
datasets_0 = tfds.load("glue/cola")
datasets_1 = tfds.load("glue/cola", batch_size=4)
train_dataset_0 = datasets_0["train"]
train_dataset_1 = datasets_1["train"]
for data in train_dataset_0.take(1):
print(data)
for data in train_dataset_1.take(1):
print(data)
{'idx': ,
'label': ,
'sentence': }
{'idx': ,
'label': ,
'sentence': }
4、with_info参数
# 指定split为train，返回为一个训练集tf.data.Dataset对象
# 指定with_info为True，返回为(tf.data.Dataset, tfds.core.DatasetInfo)二元组
datasets = tfds.load("glue/cola", split="train", with_info=True)
print(datasets)
(, tfds.core.DatasetInfo( name='glue', full_name='glue/cola/2.0.0', description=""" GLUE, the General Language Understanding Evaluation benchmark (https://gluebenchmark.com/) is a collection of resources for training, evaluating, and analyzing natural language understanding systems. """, config_description=""" The Corpus of Linguistic Acceptability consists of English acceptability judgments drawn from books and journal articles on linguistic theory. Each example is a sequence of words annotated with whether it is a grammatical English sentence. """, homepage='https://nyu-mll.github.io/CoLA/', data_path='C:\Users\ASUS\tensorflow_datasets\glue\cola\2.0.0', download_size=368.14 KiB, dataset_size=965.49 KiB, features=FeaturesDict({ 'idx': tf.int32, 'label': ClassLabel(shape=(), dtype=tf.int64, num_classes=2), 'sentence': Text(shape=(), dtype=tf.string), }), supervised_keys=None, disable_shuffling=False, splits={ 'test': , 'train': , 'validation': , }, citation="""@article{warstadt2018neural, title={Neural Network Acceptability Judgments}, author={Warstadt, Alex and Singh, Amanpreet and Bowman, Samuel R}, journal={arXiv preprint arXiv:1805.12471}, year={2018} } @inproceedings{wang2019glue, title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding}, author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.}, note={In the Proceedings of ICLR.}, year={2019} } Note that each GLUE dataset has its own citation. Please see the source to see the correct citation for each contained dataset.""", ))
