ez_transfer.datasets

csv_reader

class easytransfer.datasets.csv_reader.CSVReader(input_glob, batch_size, is_training, thread_num=1, input_queue=None, output_queue=None, job_name='DISTCSVReader', **kwargs)

Read csv format

Parameters

• input_glob -- input file fp

• batch_size -- input batch size

• is_training -- True or False

• thread_num -- thread number

process(input_data)

close()

class easytransfer.datasets.csv_reader.BundleCSVReader(input_glob, batch_size, worker_hosts, task_index, is_training=False, **kwargs)

Read group of csv formats

Parameters

• input_glob -- input file fp

• batch_size -- input batch size

• worker_hosts -- worker hosts

• task_index -- task index

• is_training -- True or False

csv_writer

class easytransfer.datasets.csv_writer.CSVWriter(output_glob, output_schema, input_queue=None, **kwargs)

Writer csv format

Parameters

• output_glob -- output file fp

• output_schema -- output_schema

odps_table_reader

class easytransfer.datasets.odps_table_reader.OdpsTableReader(input_glob, batch_size, is_training, thread_num=1, input_queue=None, output_queue=None, slice_id=0, slice_count=1, job_name='DISTOdpsTableReader', **kwargs)

Read odps table

Parameters

• input_glob -- input file fp

• batch_size -- input batch size

• is_training -- True or False

process(input_data)

odps_table_writer

class easytransfer.datasets.odps_table_writer.OdpsTableWriter(output_glob, output_schema, slice_id, input_queue, job_name='DistOdpsTableWriter', **kwargs)

Writer odps table

Parameters

• output_glob -- output file fp

• output_schema -- output_schema

tfrecord_reader

class easytransfer.datasets.tfrecord_reader.TFRecordReader(input_glob, batch_size, is_training, thread_num=1, input_queue=None, output_queue=None, job_name='DISTTFRecordReader', **kwargs)

Read tfrecords

Parameters

• input_glob -- input file fp

• batch_size -- input batch size

• is_training -- True or False

class easytransfer.datasets.tfrecord_reader.BundleTFRecordReader(input_glob, batch_size, worker_hosts, task_index, distribution_strategy, is_training=False, **kwargs)

tfrecord_writer

class easytransfer.datasets.tfrecord_writer.TFRecordWriter(output_glob, output_schema, input_queue=None)

Writer tfrecords

Parameters

• output_glob -- output file fp

• output_schema -- output_schema