diff --git a/PaddleRec/ctr/dcn/infer.py b/PaddleRec/ctr/dcn/infer.py deleted file mode 100644 index f79e36d965..0000000000 --- a/PaddleRec/ctr/dcn/infer.py +++ /dev/null @@ -1,113 +0,0 @@ -import logging -import random - -import numpy as np -import pickle - -# disable gpu training for this example -import os -os.environ['CUDA_VISIBLE_DEVICES'] = '' -import paddle -import paddle.fluid as fluid - -from config import parse_args -from reader import CriteoDataset -from network import DCN -from collections import OrderedDict -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger('fluid') -logger.setLevel(logging.INFO) - - -def infer(): - args = parse_args() - print(args) - - place = fluid.CPUPlace() - inference_scope = fluid.Scope() - - test_valid_files = [ - os.path.join(args.test_valid_data_dir, fname) - for fname in next(os.walk(args.test_valid_data_dir))[2] - ] - test_files = random.sample(test_valid_files, - int(len(test_valid_files) * 0.5)) - if not test_files: - test_files = test_valid_files - print('test files num {}'.format(len(test_files))) - - criteo_dataset = CriteoDataset() - criteo_dataset.setup(args.vocab_dir) - test_reader = criteo_dataset.test_reader(test_files, args.batch_size, 100) - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_output_dir, - 'epoch_' + args.test_epoch, "checkpoint") - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - cat_feat_dims_dict = OrderedDict() - for line in open(args.cat_feat_num): - spls = line.strip().split() - assert len(spls) == 2 - cat_feat_dims_dict[spls[0]] = int(spls[1]) - dcn_model = DCN(args.cross_num, args.dnn_hidden_units, - args.l2_reg_cross, args.use_bn, args.clip_by_norm, - cat_feat_dims_dict, args.is_sparse) - dcn_model.build_network(is_test=True) - - exe = fluid.Executor(place) - feeder = fluid.DataFeeder( - feed_list=dcn_model.data_list, place=place) - - exe.run(startup_program) - fluid.load(fluid.default_main_program(), cur_model_path) - - for var in dcn_model.auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - loss_all = 0 - num_ins = 0 - for batch_id, data_test in enumerate(test_reader()): - loss_val, auc_val = exe.run(test_program, - feed=feeder.feed(data_test), - fetch_list=[ - dcn_model.avg_logloss.name, - dcn_model.auc_var.name - ]) - # num_ins += len(data_test) - num_ins += 1 - loss_all += loss_val - logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( - batch_id, loss_all / num_ins, auc_val)) - - print( - 'The last log info is the total Logloss and AUC for all test data. ' - ) - - -def set_zero(var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - infer() diff --git a/PaddleRec/ctr/dcn/local_train.py b/PaddleRec/ctr/dcn/local_train.py deleted file mode 100644 index 411fb916bd..0000000000 --- a/PaddleRec/ctr/dcn/local_train.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import print_function, absolute_import, division -import os -import random -import sys -import time -from collections import OrderedDict - -import paddle.fluid as fluid - -from config import parse_args -from network import DCN -import utils -""" -train DCN model -""" - - -def train(args): - """train and save DCN model - - :param args: hyperparams of model - :return: - """ - # ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - cat_feat_dims_dict = OrderedDict() - for line in open(args.cat_feat_num): - spls = line.strip().split() - assert len(spls) == 2 - cat_feat_dims_dict[spls[0]] = int(spls[1]) - dcn_model = DCN(args.cross_num, args.dnn_hidden_units, args.l2_reg_cross, - args.use_bn, args.clip_by_norm, cat_feat_dims_dict, - args.is_sparse) - dcn_model.build_network() - dcn_model.backward(args.lr) - - # config dataset - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(dcn_model.data_list) - pipe_command = 'python reader.py {}'.format(args.vocab_dir) - dataset.set_pipe_command(pipe_command) - dataset.set_batch_size(args.batch_size) - dataset.set_thread(args.num_thread) - train_filelist = [ - os.path.join(args.train_data_dir, fname) - for fname in next(os.walk(args.train_data_dir))[2] - ] - dataset.set_filelist(train_filelist) - num_epoch = args.num_epoch - if args.steps: - epoch = args.steps * args.batch_size / 41000000 - full_epoch = int(epoch // 1) - last_epoch = epoch % 1 - train_filelists = [train_filelist for _ in range(full_epoch)] + [ - random.sample(train_filelist, int( - len(train_filelist) * last_epoch)) - ] - num_epoch = full_epoch + 1 - print("train epoch: {}".format(num_epoch)) - - # Executor - exe = fluid.Executor(fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - - for epoch_id in range(num_epoch): - start = time.time() - sys.stderr.write('\nepoch%d start ...\n' % (epoch_id + 1)) - dataset.set_filelist(train_filelists[epoch_id]) - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[ - dcn_model.loss, dcn_model.avg_logloss, dcn_model.auc_var - ], - fetch_info=['total_loss', 'avg_logloss', 'auc'], - debug=False, - print_period=args.print_steps) - model_dir = os.path.join(args.model_output_dir, - 'epoch_' + str(epoch_id + 1), "checkpoint") - sys.stderr.write('epoch%d is finished and takes %f s\n' % ( - (epoch_id + 1), time.time() - start)) - fluid.save(fluid.default_main_program(), model_dir) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - args = parse_args() - print(args) - utils.check_version() - train(args) diff --git a/PaddleRec/ctr/deepfm/criteo_reader.py b/PaddleRec/ctr/deepfm/criteo_reader.py deleted file mode 100644 index 037fe909bd..0000000000 --- a/PaddleRec/ctr/deepfm/criteo_reader.py +++ /dev/null @@ -1,76 +0,0 @@ -import sys -import paddle.fluid.incubate.data_generator as dg -try: - import cPickle as pickle -except ImportError: - import pickle -from collections import Counter -import os - - -class CriteoDataset(dg.MultiSlotDataGenerator): - def setup(self, feat_dict_name): - self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - self.cont_max_ = [ - 5775, 257675, 65535, 969, 23159456, 431037, 56311, 6047, 29019, 46, - 231, 4008, 7393 - ] - self.cont_diff_ = [ - self.cont_max_[i] - self.cont_min_[i] - for i in range(len(self.cont_min_)) - ] - self.continuous_range_ = range(1, 14) - self.categorical_range_ = range(14, 40) - self.feat_dict_ = pickle.load(open(feat_dict_name, 'rb')) - - def _process_line(self, line): - features = line.rstrip('\n').split('\t') - feat_idx = [] - feat_value = [] - for idx in self.continuous_range_: - if features[idx] == '': - feat_idx.append(0) - feat_value.append(0.0) - else: - feat_idx.append(self.feat_dict_[idx]) - feat_value.append( - (float(features[idx]) - self.cont_min_[idx - 1]) / - self.cont_diff_[idx - 1]) - for idx in self.categorical_range_: - if features[idx] == '' or features[idx] not in self.feat_dict_: - feat_idx.append(0) - feat_value.append(0.0) - else: - feat_idx.append(self.feat_dict_[features[idx]]) - feat_value.append(1.0) - label = [int(features[0])] - return feat_idx, feat_value, label - - def test(self, filelist): - def local_iter(): - for fname in filelist: - with open(fname.strip(), 'r') as fin: - for line in fin: - feat_idx, feat_value, label = self._process_line(line) - yield [feat_idx, feat_value, label] - - return local_iter - - def generate_sample(self, line): - def data_iter(): - feat_idx, feat_value, label = self._process_line(line) - yield [('feat_idx', feat_idx), ('feat_value', feat_value), ('label', - label)] - - return data_iter - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - criteo_dataset = CriteoDataset() - if len(sys.argv) <= 1: - sys.stderr.write("feat_dict needed for criteo reader.") - exit(1) - criteo_dataset.setup(sys.argv[1]) - criteo_dataset.run_from_stdin() diff --git a/PaddleRec/ctr/deepfm/infer.py b/PaddleRec/ctr/deepfm/infer.py deleted file mode 100644 index 35f18a6efd..0000000000 --- a/PaddleRec/ctr/deepfm/infer.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging -import numpy as np -import pickle - -# disable gpu training for this example -import os -os.environ['CUDA_VISIBLE_DEVICES'] = '' -import paddle -import paddle.fluid as fluid - -from args import parse_args -from criteo_reader import CriteoDataset -from network_conf import ctr_deepfm_model -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger('fluid') -logger.setLevel(logging.INFO) - - -def infer(): - args = parse_args() - - place = fluid.CPUPlace() - inference_scope = fluid.Scope() - - test_files = [ - os.path.join(args.test_data_dir, x) - for x in os.listdir(args.test_data_dir) - ] - criteo_dataset = CriteoDataset() - criteo_dataset.setup(args.feat_dict) - test_reader = fluid.io.batch( - criteo_dataset.test(test_files), batch_size=args.batch_size) - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_output_dir, - 'epoch_' + args.test_epoch) - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - loss, auc, data_list, auc_states = ctr_deepfm_model( - args.embedding_size, args.num_field, args.num_feat, - args.layer_sizes, args.act, args.reg) - - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=data_list, place=place) - main_program = fluid.default_main_program() - fluid.load(main_program, cur_model_path, exe) - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - loss_all = 0 - num_ins = 0 - for batch_id, data_test in enumerate(test_reader()): - loss_val, auc_val = exe.run(test_program, - feed=feeder.feed(data_test), - fetch_list=[loss.name, auc.name]) - num_ins += len(data_test) - loss_all += loss_val - logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( - batch_id, loss_all / num_ins, auc_val)) - - print( - 'The last log info is the total Logloss and AUC for all test data. ' - ) - - -def set_zero(var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - infer() diff --git a/PaddleRec/ctr/deepfm/local_train.py b/PaddleRec/ctr/deepfm/local_train.py deleted file mode 100644 index 5b4d724af1..0000000000 --- a/PaddleRec/ctr/deepfm/local_train.py +++ /dev/null @@ -1,72 +0,0 @@ -from args import parse_args -import os -import paddle.fluid as fluid -import sys -from network_conf import ctr_deepfm_model -import time -import numpy -import pickle -import utils - - -def train(): - args = parse_args() - # add ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - print('---------- Configuration Arguments ----------') - for key, value in args.__dict__.items(): - print(key + ':' + str(value)) - - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - loss, auc, data_list, auc_states = ctr_deepfm_model( - args.embedding_size, args.num_field, args.num_feat, args.layer_sizes, - args.act, args.reg) - optimizer = fluid.optimizer.SGD( - learning_rate=args.lr, - regularization=fluid.regularizer.L2DecayRegularizer(args.reg)) - optimizer.minimize(loss) - - exe = fluid.Executor(fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(data_list) - pipe_command = 'python criteo_reader.py {}'.format(args.feat_dict) - dataset.set_pipe_command(pipe_command) - dataset.set_batch_size(args.batch_size) - dataset.set_thread(args.num_thread) - train_filelist = [ - os.path.join(args.train_data_dir, x) - for x in os.listdir(args.train_data_dir) - ] - - print('---------------------------------------------') - for epoch_id in range(args.num_epoch): - start = time.time() - dataset.set_filelist(train_filelist) - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[loss, auc], - fetch_info=['epoch %d batch loss' % (epoch_id + 1), "auc"], - print_period=1000, - debug=False) - model_dir = os.path.join(args.model_output_dir, - 'epoch_' + str(epoch_id + 1)) - sys.stderr.write('epoch%d is finished and takes %f s\n' % ( - (epoch_id + 1), time.time() - start)) - main_program = fluid.default_main_program() - fluid.io.save(main_program, model_dir) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/ctr/din/infer.py b/PaddleRec/ctr/din/infer.py deleted file mode 100644 index d6d8d39eb3..0000000000 --- a/PaddleRec/ctr/din/infer.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import argparse -import logging -import numpy as np -import os -import paddle -import paddle.fluid as fluid -import reader - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle DIN example") - parser.add_argument( - '--model_path', - type=str, - required=True, - help="path of model parameters") - parser.add_argument( - '--test_path', - type=str, - default='data/paddle_test.txt.bak', - help='dir of test file') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether to use gpu') - - return parser.parse_args() - - -def calc_auc(raw_arr): - # sort by pred value, from small to big - arr = sorted(raw_arr, key=lambda d: d[2]) - auc = 0.0 - fp1, tp1, fp2, tp2 = 0.0, 0.0, 0.0, 0.0 - for record in arr: - fp2 += record[0] # noclick - tp2 += record[1] # click - auc += (fp2 - fp1) * (tp2 + tp1) - fp1, tp1 = fp2, tp2 - # if all nonclick or click, disgard - threshold = len(arr) - 1e-3 - if tp2 > threshold or fp2 > threshold: - return -0.5 - if tp2 * fp2 > 0.0: # normal auc - return (1.0 - auc / (2.0 * tp2 * fp2)) - else: - return None - - -def infer(): - args = parse_args() - model_path = args.model_path - use_cuda = True if args.use_cuda else False - data_reader, _ = reader.prepare_reader(args.test_path, 32 * 16) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - inference_scope = fluid.Scope() - - exe = fluid.Executor(place) - - #with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - - loader = fluid.io.DataLoader.from_generator( - feed_list=[ - inference_program.block(0).var(e) for e in feed_target_names - ], - capacity=10000, - iterable=True) - loader.set_sample_list_generator(data_reader, places=place) - - loss_sum = 0.0 - score = [] - count = 0 - for data in loader(): - res = exe.run(inference_program, feed=data, fetch_list=fetch_targets) - loss_sum += res[0] - label_data = list(np.array(data[0]["label"])) - for i in range(len(label_data)): - if label_data[i] > 0.5: - score.append([0, 1, res[1][i]]) - else: - score.append([1, 0, res[1][i]]) - count += 1 - auc = calc_auc(score) - logger.info("TEST --> loss: {}, auc: {}".format(loss_sum / count, auc)) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - infer() diff --git a/PaddleRec/ctr/din/train.py b/PaddleRec/ctr/din/train.py deleted file mode 100644 index f309459ab8..0000000000 --- a/PaddleRec/ctr/din/train.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import os -import sys -import logging -import time -import numpy as np -import argparse -import paddle.fluid as fluid -import paddle -import time -import network -import reader -import random - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser("din") - parser.add_argument( - '--config_path', - type=str, - default='data/config.txt', - help='dir of config') - parser.add_argument( - '--train_dir', - type=str, - default='data/paddle_train.txt', - help='dir of train file') - parser.add_argument( - '--model_dir', - type=str, - default='din_amazon', - help='dir of saved model') - parser.add_argument( - '--batch_size', type=int, default=16, help='number of batch size') - parser.add_argument( - '--epoch_num', type=int, default=200, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether to use gpu') - parser.add_argument( - '--parallel', - type=int, - default=0, - help='whether to use parallel executor') - parser.add_argument( - '--base_lr', type=float, default=0.85, help='based learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - parser.add_argument('--batch_num', type=int, help="batch num for ce") - args = parser.parse_args() - return args - - -def train(): - args = parse_args() - - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - config_path = args.config_path - train_path = args.train_dir - epoch_num = args.epoch_num - use_cuda = True if args.use_cuda else False - use_parallel = True if args.parallel else False - - logger.info("reading data begins") - user_count, item_count, cat_count = reader.config_read(config_path) - data_reader, max_len = reader.prepare_reader(train_path, args.batch_size * - args.num_devices) - logger.info("reading data completes") - - avg_cost, pred, feed_list = network.network(item_count, cat_count) - - clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0) - base_lr = args.base_lr - boundaries = [410000] - values = [base_lr, 0.2] - sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.piecewise_decay( - boundaries=boundaries, values=values), - grad_clip=clip) - sgd_optimizer.minimize(avg_cost) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - loader = fluid.io.DataLoader.from_generator( - feed_list=feed_list, capacity=10000, iterable=True) - loader.set_sample_list_generator(data_reader, places=place) - if use_parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - logger.info("train begins") - - global_step = 0 - PRINT_STEP = 1000 - - total_time = [] - ce_info = [] - start_time = time.time() - loss_sum = 0.0 - for id in range(epoch_num): - epoch = id + 1 - for data in loader(): - global_step += 1 - results = train_exe.run(feed=data, - fetch_list=[avg_cost.name, pred.name], - return_numpy=True) - loss_sum += results[0].mean() - - if global_step % PRINT_STEP == 0: - ce_info.append(loss_sum / PRINT_STEP) - total_time.append(time.time() - start_time) - logger.info( - "epoch: %d\tglobal_step: %d\ttrain_loss: %.4f\t\ttime: %.2f" - % (epoch, global_step, loss_sum / PRINT_STEP, - time.time() - start_time)) - start_time = time.time() - loss_sum = 0.0 - - if (global_step > 400000 and global_step % PRINT_STEP == 0) or ( - global_step <= 400000 and global_step % 50000 == 0): - save_dir = os.path.join(args.model_dir, - "global_step_" + str(global_step)) - feed_var_name = [ - "hist_item_seq", "hist_cat_seq", "target_item", - "target_cat", "label", "mask", "target_item_seq", - "target_cat_seq" - ] - fetch_vars = [avg_cost, pred] - fluid.io.save_inference_model(save_dir, feed_var_name, - fetch_vars, exe) - logger.info("model saved in " + save_dir) - if args.enable_ce and global_step >= args.batch_num: - break - # only for ce - if args.enable_ce: - gpu_num = get_cards(args) - ce_loss = 0 - ce_time = 0 - try: - ce_loss = ce_info[-1] - ce_time = total_time[-1] - except: - print("ce info error") - print("kpis\teach_pass_duration_card%s\t%s" % (gpu_num, ce_time)) - print("kpis\ttrain_loss_card%s\t%s" % (gpu_num, ce_loss)) - - -def get_cards(args): - if args.enable_ce: - cards = os.environ.get('CUDA_VISIBLE_DEVICES') - num = len(cards.split(",")) - return num - else: - return args.num_devices - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - train() diff --git a/PaddleRec/ctr/wide_deep/infer.py b/PaddleRec/ctr/wide_deep/infer.py deleted file mode 100644 index 3828dce284..0000000000 --- a/PaddleRec/ctr/wide_deep/infer.py +++ /dev/null @@ -1,75 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from net import wide_deep -import logging -import paddle -import args -import utils -import time -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def set_zero(var_name,scope=fluid.global_scope(), place=fluid.CPUPlace(),param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -def run_infer(args,test_data_path): - wide_deep_model = wide_deep() - test_data_generator = utils.Dataset() - test_reader = fluid.io.batch(test_data_generator.test(test_data_path), batch_size=args.batch_size) - inference_scope = fluid.Scope() - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - - cur_model_path = os.path.join(args.model_dir, 'epoch_' + str(args.test_epoch), "checkpoint") - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - inputs = wide_deep_model.input_data() - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - loss, acc, auc, batch_auc, auc_states = wide_deep_model.model(inputs, args.hidden1_units, args.hidden2_units, args.hidden3_units) - exe = fluid.Executor(place) - - fluid.load(fluid.default_main_program(), cur_model_path,exe) - loader = fluid.io.DataLoader.from_generator(feed_list=inputs, capacity=args.batch_size, iterable=True) - loader.set_sample_list_generator(test_reader, places=place) - - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - mean_acc = [] - mean_auc = [] - for batch_id, data in enumerate(loader()): - begin = time.time() - acc_val,auc_val = exe.run(program=test_program, - feed=data, - fetch_list=[acc.name, auc.name], - return_numpy=True - ) - mean_acc.append(np.array(acc_val)[0]) - mean_auc.append(np.array(auc_val)[0]) - end = time.time() - logger.info("batch_id: {}, batch_time: {:.5f}s, acc: {:.5f}, auc: {:.5f}".format( - batch_id, end-begin, np.array(acc_val)[0], np.array(auc_val)[0])) - - logger.info("mean_acc:{:.5f}, mean_auc:{:.5f}".format(np.mean(mean_acc), np.mean(mean_auc))) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - run_infer(args, args.test_data_path) - \ No newline at end of file diff --git a/PaddleRec/ctr/wide_deep/train.py b/PaddleRec/ctr/wide_deep/train.py deleted file mode 100644 index 781feb8fd9..0000000000 --- a/PaddleRec/ctr/wide_deep/train.py +++ /dev/null @@ -1,52 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from net import wide_deep -import logging -import paddle -import args -import utils -import time - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args, train_data_path): - wide_deep_model = wide_deep() - inputs = wide_deep_model.input_data() - train_data_generator = utils.Dataset() - train_reader = fluid.io.batch(train_data_generator.train(train_data_path), batch_size=args.batch_size) - - loss, acc, auc, batch_auc, auc_states = wide_deep_model.model(inputs, args.hidden1_units, args.hidden2_units, args.hidden3_units) - optimizer = fluid.optimizer.AdagradOptimizer(learning_rate=0.01) - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - loader = fluid.io.DataLoader.from_generator( - feed_list=inputs, capacity=args.batch_size, iterable=True) - loader.set_sample_list_generator(train_reader, places=place) - - for epoch in range(args.epochs): - for batch_id, data in enumerate(loader()): - begin = time.time() - loss_val, acc_val, auc_val = exe.run(program=fluid.default_main_program(), - feed=data, - fetch_list=[loss.name, acc.name, auc.name], - return_numpy=True) - end = time.time() - logger.info("epoch:{}, batch_time:{:.5f}s, loss:{:.5f}, acc:{:.5f}, auc:{:.5f}".format(epoch, end-begin, np.array(loss_val)[0], - np.array(acc_val)[0], np.array(auc_val)[0])) - - model_dir = os.path.join(args.model_dir, 'epoch_' + str(epoch + 1), "checkpoint") - main_program = fluid.default_main_program() - fluid.io.save(main_program,model_dir) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - train(args, args.train_data_path) diff --git a/PaddleRec/ctr/xdeepfm/criteo_reader.py b/PaddleRec/ctr/xdeepfm/criteo_reader.py deleted file mode 100644 index a6d32f4a47..0000000000 --- a/PaddleRec/ctr/xdeepfm/criteo_reader.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys -import paddle.fluid.incubate.data_generator as dg -import pickle -from collections import Counter -import os - - -class CriteoDataset(dg.MultiSlotDataGenerator): - def _process_line(self, line): - features = line.strip('\n').split('\t') - feat_idx = [] - feat_value = [] - for idx in range(1, 40): - feat_idx.append(int(features[idx])) - feat_value.append(1.0) - label = [int(features[0])] - return feat_idx, feat_value, label - - def test(self, filelist): - def local_iter(): - for fname in filelist: - with open(fname.strip(), 'r') as fin: - for line in fin: - feat_idx, feat_value, label = self._process_line(line) - yield [feat_idx, feat_value, label] - - return local_iter - - def generate_sample(self, line): - def data_iter(): - feat_idx, feat_value, label = self._process_line(line) - feature_name = ['feat_idx', 'feat_value', 'label'] - yield [('feat_idx', feat_idx), ('feat_value', feat_value), ('label', - label)] - - return data_iter - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - criteo_dataset = CriteoDataset() - criteo_dataset.run_from_stdin() diff --git a/PaddleRec/ctr/xdeepfm/infer.py b/PaddleRec/ctr/xdeepfm/infer.py deleted file mode 100644 index 98210fb2d4..0000000000 --- a/PaddleRec/ctr/xdeepfm/infer.py +++ /dev/null @@ -1,96 +0,0 @@ -import logging -import numpy as np -import pickle -import os -import paddle -import paddle.fluid as fluid - -from args import parse_args -from criteo_reader import CriteoDataset -import network_conf -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger('fluid') -logger.setLevel(logging.INFO) - - -def infer(): - args = parse_args() - print(args) - - if args.use_gpu == 1: - place = fluid.CUDAPlace(0) - else: - place = fluid.CPUPlace() - inference_scope = fluid.Scope() - - test_files = [ - os.path.join(args.test_data_dir, x) - for x in os.listdir(args.test_data_dir) - ] - criteo_dataset = CriteoDataset() - test_reader = fluid.io.batch( - criteo_dataset.test(test_files), batch_size=args.batch_size) - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_output_dir, - 'epoch_' + args.test_epoch, "checkpoint") - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - loss, auc, data_list, auc_states = eval( - 'network_conf.' + args.model_name)( - args.embedding_size, args.num_field, args.num_feat, - args.layer_sizes_dnn, args.act, args.reg, - args.layer_sizes_cin) - - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=data_list, place=place) - - exe.run(startup_program) - fluid.load(fluid.default_main_program(), cur_model_path) - - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - loss_all = 0 - num_ins = 0 - for batch_id, data_test in enumerate(test_reader()): - loss_val, auc_val = exe.run(test_program, - feed=feeder.feed(data_test), - fetch_list=[loss.name, auc.name]) - - num_ins += len(data_test) - loss_all += loss_val * len(data_test) - logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( - batch_id, loss_all / num_ins, auc_val)) - - print( - 'The last log info is the total Logloss and AUC for all test data. ' - ) - - -def set_zero(var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - infer() diff --git a/PaddleRec/ctr/xdeepfm/local_train.py b/PaddleRec/ctr/xdeepfm/local_train.py deleted file mode 100644 index 5743503340..0000000000 --- a/PaddleRec/ctr/xdeepfm/local_train.py +++ /dev/null @@ -1,68 +0,0 @@ -from args import parse_args -import os -import paddle.fluid as fluid -import sys -import network_conf -import time -import utils - - -def train(): - args = parse_args() - # add ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - print(args) - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - loss, auc, data_list, auc_states = eval('network_conf.' + args.model_name)( - args.embedding_size, args.num_field, args.num_feat, - args.layer_sizes_dnn, args.act, args.reg, args.layer_sizes_cin) - optimizer = fluid.optimizer.SGD( - learning_rate=args.lr, - regularization=fluid.regularizer.L2DecayRegularizer(args.reg)) - optimizer.minimize(loss) - - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(data_list) - dataset.set_pipe_command('python criteo_reader.py') - dataset.set_batch_size(args.batch_size) - dataset.set_filelist([ - os.path.join(args.train_data_dir, x) - for x in os.listdir(args.train_data_dir) - ]) - - if args.use_gpu == 1: - exe = fluid.Executor(fluid.CUDAPlace(0)) - dataset.set_thread(1) - else: - exe = fluid.Executor(fluid.CPUPlace()) - dataset.set_thread(args.num_thread) - exe.run(fluid.default_startup_program()) - - for epoch_id in range(args.num_epoch): - start = time.time() - sys.stderr.write('\nepoch%d start ...\n' % (epoch_id + 1)) - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[loss, auc], - fetch_info=['loss', 'auc'], - debug=False, - print_period=args.print_steps) - model_dir = os.path.join(args.model_output_dir, - 'epoch_' + str(epoch_id + 1), "checkpoint") - sys.stderr.write('epoch%d is finished and takes %f s\n' % ( - (epoch_id + 1), time.time() - start)) - fluid.save(fluid.default_main_program(), model_dir) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/dssm/infer.py b/PaddleRec/dssm/infer.py deleted file mode 100644 index 06eb0ba5d5..0000000000 --- a/PaddleRec/dssm/infer.py +++ /dev/null @@ -1,43 +0,0 @@ -import paddle.fluid as fluid -import numpy as np -import sys -import args -import logging - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def infer(args): - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model(args.model_dir, exe) - - #构造测试数据 - sample_size = 100 - l_Qs = [] - pos_l_Ds = [] - - for i in range(sample_size): - l_Q = np.random.rand(1, args.TRIGRAM_D) - l_Qs.append(l_Q) - - l_D = np.random.rand(1, args.TRIGRAM_D) - pos_l_Ds.append(l_D) - - res = [] - for i in range(sample_size): - con_sim = exe.run(infer_program, - feed={"query": l_Qs[i].astype('float32').reshape(1,args.TRIGRAM_D), - "doc_pos": pos_l_Ds[i].astype('float32').reshape(1,args.TRIGRAM_D)}, - fetch_list=fetch_vars, - return_numpy=True) - - logger.info("query_doc_sim: {:.5f}".format(np.array(con_sim).reshape(-1,1)[0][0])) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - infer(args) \ No newline at end of file diff --git a/PaddleRec/gnn/infer.py b/PaddleRec/gnn/infer.py deleted file mode 100644 index 2bd58b0fcb..0000000000 --- a/PaddleRec/gnn/infer.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import argparse -import logging -import numpy as np -import os -import paddle -import paddle.fluid as fluid -import reader -import network - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle DIN example") - parser.add_argument( - '--model_path', type=str, default='./saved_model/', help="path of model parameters") - parser.add_argument( - '--test_path', type=str, default='./data/diginetica/test.txt', help='dir of test file') - parser.add_argument( - '--config_path', type=str, default='./data/diginetica/config.txt', help='dir of config') - parser.add_argument( - '--use_cuda', type=int, default=1, help='whether to use gpu') - parser.add_argument( - '--batch_size', type=int, default=100, help='input batch size') - parser.add_argument( - '--start_index', type=int, default='0', help='start index') - parser.add_argument( - '--last_index', type=int, default='10', help='end index') - parser.add_argument( - '--hidden_size', type=int, default=100, help='hidden state size') - parser.add_argument( - '--step', type=int, default=1, help='gnn propogation steps') - return parser.parse_args() - - -def infer(args): - batch_size = args.batch_size - items_num = reader.read_config(args.config_path) - test_data = reader.Data(args.test_path, False) - place = fluid.CUDAPlace(0) if args.use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - loss, acc, py_reader, feed_datas = network.network(items_num, args.hidden_size, args.step, batch_size) - exe.run(fluid.default_startup_program()) - infer_program = fluid.default_main_program().clone(for_test=True) - - for epoch_num in range(args.start_index, args.last_index + 1): - model_path = os.path.join(args.model_path, "epoch_" + str(epoch_num)) - try: - if not os.path.exists(model_path + ".pdmodel"): - raise ValueError() - fluid.io.load(infer_program, model_path+".pdmodel", exe) - - loss_sum = 0.0 - acc_sum = 0.0 - count = 0 - py_reader.set_sample_list_generator(test_data.reader(batch_size, batch_size*20, False)) - py_reader.start() - try: - while True: - res = exe.run(infer_program, - fetch_list=[loss.name, acc.name], use_program_cache=True) - loss_sum += res[0] - acc_sum += res[1] - count += 1 - except fluid.core.EOFException: - py_reader.reset() - logger.info("TEST --> loss: %.4lf, Recall@20: %.4lf" % - (loss_sum / count, acc_sum / count)) - except ValueError as e: - logger.info("TEST --> error: there is no model in " + model_path) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = parse_args() - infer(args) diff --git a/PaddleRec/gnn/train.py b/PaddleRec/gnn/train.py deleted file mode 100644 index ae58ca3ea0..0000000000 --- a/PaddleRec/gnn/train.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import numpy as np -import os -from functools import partial -import logging -import time -import paddle -import paddle.fluid as fluid -import argparse -import network -import reader - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser("gnn") - parser.add_argument( - '--train_path', type=str, default='./data/diginetica/train.txt', help='dir of training data') - parser.add_argument( - '--config_path', type=str, default='./data/diginetica/config.txt', help='dir of config') - parser.add_argument( - '--model_path', type=str, default='./saved_model', help="path of model parameters") - parser.add_argument( - '--epoch_num', type=int, default=30, help='number of epochs to train for') - parser.add_argument( - '--batch_size', type=int, default=100, help='input batch size') - parser.add_argument( - '--hidden_size', type=int, default=100, help='hidden state size') - parser.add_argument( - '--l2', type=float, default=1e-5, help='l2 penalty') - parser.add_argument( - '--lr', type=float, default=0.001, help='learning rate') - parser.add_argument( - '--step', type=int, default=1, help='gnn propogation steps') - parser.add_argument( - '--lr_dc', type=float, default=0.1, help='learning rate decay rate') - parser.add_argument( - '--lr_dc_step', type=int, default=3, help='the number of steps after which the learning rate decay') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether to use gpu') - parser.add_argument( - '--use_parallel', type=int, default=1, help='whether to use parallel executor') - parser.add_argument( - '--enable_ce', action='store_true', help='If set, run the task with continuous evaluation logs.') - return parser.parse_args() - - -def train(): - args = parse_args() - - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - batch_size = args.batch_size - items_num = reader.read_config(args.config_path) - loss, acc, py_reader, feed_datas = network.network(items_num, args.hidden_size, - args.step, batch_size) - - data_reader = reader.Data(args.train_path, True) - logger.info("load data complete") - - use_cuda = True if args.use_cuda else False - use_parallel = True if args.use_parallel else False - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - - exe = fluid.Executor(place) - step_per_epoch = data_reader.length // batch_size - optimizer = fluid.optimizer.Adam( - learning_rate=fluid.layers.exponential_decay( - learning_rate=args.lr, - decay_steps=step_per_epoch * args.lr_dc_step, - decay_rate=args.lr_dc), - regularization=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=args.l2)) - optimizer.minimize(loss) - - exe.run(fluid.default_startup_program()) - - all_vocab = fluid.global_scope().var("all_vocab").get_tensor() - all_vocab.set( - np.arange(1, items_num).astype("int64").reshape((-1)), place) - - feed_list = [e.name for e in feed_datas] - - if use_parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=loss.name) - else: - train_exe = exe - - logger.info("begin train") - - total_time = [] - ce_info = [] - start_time = time.time() - loss_sum = 0.0 - acc_sum = 0.0 - global_step = 0 - PRINT_STEP = 500 - #py_reader.decorate_paddle_reader(data_reader.reader(batch_size, batch_size * 20, True)) - py_reader.set_sample_list_generator(data_reader.reader(batch_size, batch_size * 20, True)) - for i in range(args.epoch_num): - epoch_sum = [] - py_reader.start() - try: - while True: - res = train_exe.run(fetch_list=[loss.name, acc.name]) - loss_sum += res[0].mean() - acc_sum += res[1].mean() - epoch_sum.append(res[0].mean()) - global_step += 1 - if global_step % PRINT_STEP == 0: - ce_info.append([loss_sum / PRINT_STEP, acc_sum / PRINT_STEP]) - total_time.append(time.time() - start_time) - logger.info("global_step: %d, loss: %.4lf, train_acc: %.4lf" % ( - global_step, loss_sum / PRINT_STEP, acc_sum / PRINT_STEP)) - loss_sum = 0.0 - acc_sum = 0.0 - start_time = time.time() - except fluid.core.EOFException: - py_reader.reset() - logger.info("epoch loss: %.4lf" % (np.mean(epoch_sum))) - save_dir = os.path.join(args.model_path, "epoch_" + str(i)) - fetch_vars = [loss, acc] - fluid.save(fluid.default_main_program(), model_path=save_dir) - logger.info("model saved in " + save_dir) - - # only for ce - if args.enable_ce: - gpu_num = get_cards(args) - ce_loss = 0 - ce_acc = 0 - ce_time = 0 - try: - ce_loss = ce_info[-1][0] - ce_acc = ce_info[-1][1] - ce_time = total_time[-1] - except: - print("ce info error") - print("kpis\teach_pass_duration_card%s\t%s" % - (gpu_num, ce_time)) - print("kpis\ttrain_loss_card%s\t%f" % - (gpu_num, ce_loss)) - print("kpis\ttrain_acc_card%s\t%f" % - (gpu_num, ce_acc)) - - -def get_cards(args): - num = 0 - cards = os.environ.get('CUDA_VISIBLE_DEVICES') - num = len(cards.split(",")) - return num - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - train() diff --git a/PaddleRec/gru4rec/infer.py b/PaddleRec/gru4rec/infer.py deleted file mode 100644 index 618bc799bd..0000000000 --- a/PaddleRec/gru4rec/infer.py +++ /dev/null @@ -1,96 +0,0 @@ -import argparse -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle - -import utils - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--start_index', type=int, default='1', help='start index') - parser.add_argument( - '--last_index', type=int, default='10', help='end index') - parser.add_argument( - '--model_dir', type=str, default='model_recall20', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - args = parser.parse_args() - return args - - -def infer(test_reader, use_cuda, model_path): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - model_path, exe) - accum_num_recall = 0.0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - src_wordseq = utils.to_lodtensor([dat[0] for dat in data], place) - label_data = [dat[1] for dat in data] - dst_wordseq = utils.to_lodtensor(label_data, place) - para = exe.run( - infer_program, - feed={"src_wordseq": src_wordseq, - "dst_wordseq": dst_wordseq}, - fetch_list=fetch_vars, - return_numpy=False) - - acc_ = para[1]._get_float_element(0) - data_length = len( - np.concatenate( - label_data, axis=0).astype("int64")) - accum_num_sum += (data_length) - accum_num_recall += (data_length * acc_) - if step_id % 1 == 0: - print("step:%d recall@20:%.4f" % - (step_id, accum_num_recall / accum_num_sum)) - t1 = time.time() - print("model:%s recall@20:%.3f time_cost(s):%.2f" % - (model_path, accum_num_recall / accum_num_sum, t1 - t0)) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_size, test_reader = utils.prepare_data( - test_dir, - vocab_path, - batch_size=batch_size, - buffer_size=1000, - word_freq_threshold=0, - is_train=False) - - for epoch in range(start_index, last_index + 1): - epoch_path = model_dir + "/epoch_" + str(epoch) - infer(test_reader=test_reader, use_cuda=use_cuda, model_path=epoch_path) diff --git a/PaddleRec/gru4rec/infer_sample_neg.py b/PaddleRec/gru4rec/infer_sample_neg.py deleted file mode 100644 index 598644e9ca..0000000000 --- a/PaddleRec/gru4rec/infer_sample_neg.py +++ /dev/null @@ -1,106 +0,0 @@ -import argparse -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle -import net -import utils - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--start_index', type=int, default='1', help='start index') - parser.add_argument( - '--last_index', type=int, default='3', help='last index') - parser.add_argument( - '--model_dir', type=str, default='model_neg_recall20', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - parser.add_argument( - '--hid_size', type=int, default='100', help='batch_size') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - args = parser.parse_args() - return args - - -def infer(args, vocab_size, test_reader, use_cuda): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - hid_size = args.hid_size - batch_size = args.batch_size - with fluid.scope_guard(fluid.Scope()): - main_program = fluid.Program() - with fluid.program_guard(main_program): - acc = net.infer_network(vocab_size, batch_size, hid_size) - for epoch in range(start_index, last_index + 1): - copy_program = main_program.clone() - model_path = model_dir + "/epoch_" + str(epoch) - fluid.load(copy_program, model_path, exe) - accum_num_recall = 0.0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - label_data = [dat[1] for dat in data] - ls, lp = utils.to_lodtensor_bpr_test(data, vocab_size, - place) - para = exe.run( - copy_program, - feed={ - "src": ls, - "all_label": - np.arange(vocab_size).reshape(vocab_size, 1), - "pos_label": lp - }, - fetch_list=[acc.name], - return_numpy=False) - - acc_ = np.array(para[0])[0] - data_length = len( - np.concatenate( - label_data, axis=0).astype("int64")) - accum_num_sum += (data_length) - accum_num_recall += (data_length * acc_) - if step_id % 1 == 0: - print("step:%d recall@20:%.4f" % - (step_id, accum_num_recall / accum_num_sum)) - t1 = time.time() - print("model:%s recall@20:%.4f time_cost(s):%.2f" % - (model_path, accum_num_recall / accum_num_sum, t1 - t0)) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_size, test_reader = utils.prepare_data( - test_dir, - vocab_path, - batch_size=batch_size, - buffer_size=1000, - word_freq_threshold=0, - is_train=False) - - infer(args, vocab_size, test_reader=test_reader, use_cuda=use_cuda) diff --git a/PaddleRec/gru4rec/train.py b/PaddleRec/gru4rec/train.py deleted file mode 100644 index c1516740b1..0000000000 --- a/PaddleRec/gru4rec/train.py +++ /dev/null @@ -1,174 +0,0 @@ -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import paddle.fluid as fluid -import paddle -import time -import utils -import net - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--train_dir', type=str, default='train_data', help='train file') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - parser.add_argument( - '--is_local', type=int, default=1, help='whether is local') - parser.add_argument( - '--hid_size', type=int, default=100, help='hidden-dim size') - parser.add_argument( - '--model_dir', type=str, default='model_recall20', help='model dir') - parser.add_argument( - '--batch_size', type=int, default=5, help='num of batch size') - parser.add_argument( - '--print_batch', type=int, default=10, help='num of print batch') - parser.add_argument( - '--pass_num', type=int, default=10, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether use gpu') - parser.add_argument( - '--parallel', type=int, default=0, help='whether parallel') - parser.add_argument( - '--base_lr', type=float, default=0.01, help='learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - parser.add_argument( - '--step_num', type=int, default=1000, help='Number of steps') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - args = parser.parse_args() - return args - - -def get_cards(args): - return args.num_devices - - -def train(): - """ do training """ - args = parse_args() - if args.enable_ce: - fluid.default_startup_program().random_seed = SEED - fluid.default_main_program().random_seed = SEED - hid_size = args.hid_size - train_dir = args.train_dir - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - parallel = True if args.parallel else False - print("use_cuda:", use_cuda, "parallel:", parallel) - batch_size = args.batch_size - vocab_size, train_reader = utils.prepare_data( - train_dir, vocab_path, batch_size=batch_size * get_cards(args),\ - buffer_size=1000, word_freq_threshold=0, is_train=True) - - # Train program - src_wordseq, dst_wordseq, avg_cost, acc = net.all_vocab_network( - vocab_size=vocab_size, hid_size=hid_size) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - if parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - pass_num = args.pass_num - model_dir = args.model_dir - fetch_list = [avg_cost.name] - - ce_info = [] - total_time = 0.0 - for pass_idx in six.moves.xrange(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - - t0 = time.time() - i = 0 - newest_ppl = 0 - for data in train_reader(): - i += 1 - lod_src_wordseq = utils.to_lodtensor([dat[0] for dat in data], - place) - lod_dst_wordseq = utils.to_lodtensor([dat[1] for dat in data], - place) - ret_avg_cost = train_exe.run(feed={ - "src_wordseq": lod_src_wordseq, - "dst_wordseq": lod_dst_wordseq - }, - fetch_list=fetch_list) - avg_ppl = np.exp(ret_avg_cost[0]) - newest_ppl = np.mean(avg_ppl) - ce_info.append(newest_ppl) - if i % args.print_batch == 0: - print("step:%d ppl:%.3f" % (i, newest_ppl)) - if args.enable_ce and i > args.step_num: - break - - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, i, total_time / epoch_idx)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - feed_var_names = ["src_wordseq", "dst_wordseq"] - fetch_vars = [avg_cost, acc] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - print("model saved in %s" % save_dir) - - # only for ce - if args.enable_ce: - ce_ppl = 0 - try: - ce_ppl = ce_info[-2] - except: - print("ce info error") - epoch_idx = args.pass_num - device = get_device(args) - if args.use_cuda: - gpu_num = device[1] - print("kpis\teach_pass_duration_gpu%s\t%s" % - (gpu_num, total_time / epoch_idx)) - print("kpis\ttrain_ppl_gpu%s\t%s" % (gpu_num, ce_ppl)) - else: - cpu_num = device[1] - threads_num = device[2] - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_ppl_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_ppl)) - - print("finish training") - - -def get_device(args): - if args.use_cuda: - gpus = os.environ.get("CUDA_VISIBLE_DEVICES", 1) - gpu_num = len(gpus.split(',')) - return "gpu", gpu_num - else: - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return "cpu", int(cpu_num), int(threads_num) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/gru4rec/train_sample_neg.py b/PaddleRec/gru4rec/train_sample_neg.py deleted file mode 100644 index f45663bf60..0000000000 --- a/PaddleRec/gru4rec/train_sample_neg.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import paddle.fluid as fluid -import paddle -import time -import utils -import net - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--train_dir', type=str, default='train_data', help='train file') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - parser.add_argument( - '--is_local', type=int, default=1, help='whether is local') - parser.add_argument( - '--hid_size', type=int, default=100, help='hidden-dim size') - parser.add_argument( - '--neg_size', type=int, default=10, help='neg item size') - parser.add_argument( - '--loss', type=str, default="bpr", help='loss: bpr/cross_entropy') - parser.add_argument( - '--model_dir', type=str, default='model_neg_recall20', help='model dir') - parser.add_argument( - '--batch_size', type=int, default=5, help='num of batch size') - parser.add_argument( - '--print_batch', type=int, default=10, help='num of print batch') - parser.add_argument( - '--pass_num', type=int, default=10, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether use gpu') - parser.add_argument( - '--parallel', type=int, default=0, help='whether parallel') - parser.add_argument( - '--base_lr', type=float, default=0.01, help='learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - args = parser.parse_args() - return args - - -def get_cards(args): - return args.num_devices - - -def train(): - """ do training """ - args = parse_args() - hid_size = args.hid_size - train_dir = args.train_dir - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - parallel = True if args.parallel else False - print("use_cuda:", use_cuda, "parallel:", parallel) - batch_size = args.batch_size - vocab_size, train_reader = utils.prepare_data( - train_dir, vocab_path, batch_size=batch_size * get_cards(args),\ - buffer_size=1000, word_freq_threshold=0, is_train=True) - - # Train program - if args.loss == 'bpr': - print('bpr loss') - src, pos_label, label, avg_cost = net.train_bpr_network( - neg_size=args.neg_size, vocab_size=vocab_size, hid_size=hid_size) - else: - print('cross-entory loss') - src, pos_label, label, avg_cost = net.train_cross_entropy_network( - neg_size=args.neg_size, vocab_size=vocab_size, hid_size=hid_size) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - if parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - pass_num = args.pass_num - model_dir = args.model_dir - fetch_list = [avg_cost.name] - - total_time = 0.0 - for pass_idx in six.moves.xrange(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - - t0 = time.time() - i = 0 - newest_ppl = 0 - for data in train_reader(): - i += 1 - ls, lp, ll = utils.to_lodtensor_bpr(data, args.neg_size, vocab_size, - place) - ret_avg_cost = train_exe.run( - feed={"src": ls, - "label": ll, - "pos_label": lp}, - fetch_list=fetch_list) - avg_ppl = np.exp(ret_avg_cost[0]) - newest_ppl = np.mean(avg_ppl) - if i % args.print_batch == 0: - print("step:%d ppl:%.3f" % (i, newest_ppl)) - - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, i, total_time / epoch_idx)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - fluid.save(fluid.default_main_program(), model_path=save_dir) - print("model saved in %s" % save_dir) - - print("finish training") - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/multi_task/esmm/infer.py b/PaddleRec/multi_task/esmm/infer.py deleted file mode 100644 index 9f85e02b76..0000000000 --- a/PaddleRec/multi_task/esmm/infer.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import numpy as np -import paddle -import paddle.fluid as fluid -from net import ESMM -import args -import logging -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def set_zero(place): - auc_states_names = [ - 'auc_1.tmp_0', 'auc_0.tmp_0' - ] - for name in auc_states_names: - param = fluid.global_scope().var(name).get_tensor() - if param: - param_array = np.zeros(param._get_dims()).astype("int64") - param.set(param_array, place) - -def run_infer(args,model_path,test_data_path,vocab_size): - place = fluid.CPUPlace() - esmm_model = ESMM() - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - - with fluid.framework.program_guard(test_program, startup_program): - with fluid.unique_name.guard(): - inputs = esmm_model.input_data() - avg_cost,auc_ctr,auc_ctcvr= esmm_model.net(inputs, vocab_size, args.embed_size) - - dataset, file_list = utils.get_dataset(inputs, test_data_path,args.batch_size,args.cpu_num) - - exe = fluid.Executor(place) - fluid.load(fluid.default_main_program(),os.path.join(model_path, "checkpoint"), exe) - - set_zero(place) - - dataset.set_filelist(file_list) - exe.infer_from_dataset(program=test_program, - dataset=dataset, - fetch_list=[auc_ctr,auc_ctcvr], - fetch_info=["auc_ctr","auc_ctcvr"], - print_period=20, - debug=False) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - model_list = [] - for _, dir, _ in os.walk(args.model_dir): - for model in dir: - if "epoch" in model: - path = os.path.join(args.model_dir, model) - model_list.append(path) - - vocab_size =utils.get_vocab_size(args.vocab_path) - - for model in model_list: - logger.info("Test model {}".format(model)) - run_infer(args, model,args.test_data_path) - - \ No newline at end of file diff --git a/PaddleRec/multi_task/esmm/train.py b/PaddleRec/multi_task/esmm/train.py deleted file mode 100644 index 57abc81e13..0000000000 --- a/PaddleRec/multi_task/esmm/train.py +++ /dev/null @@ -1,45 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from net import ESMM -import paddle -import utils -import args - -def train(args, vocab_size, train_data_path): - esmm_model = ESMM() - inputs = esmm_model.input_data() - - dataset, file_list = utils.get_dataset(inputs, train_data_path,args.batch_size,args.cpu_num) - - avg_cost,auc_ctr,auc_ctcvr= esmm_model.net(inputs, vocab_size, args.embed_size) - optimizer = fluid.optimizer.Adam() - optimizer.minimize(avg_cost) - - if args.use_gpu == True: - exe = fluid.Executor(fluid.CUDAPlace(0)) - dataset.set_thread(1) - else: - exe = fluid.Executor(fluid.CPUPlace()) - dataset.set_thread(args.cpu_num) - - exe.run(fluid.default_startup_program()) - - for epoch in range(args.epochs): - dataset.set_filelist(file_list) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[avg_cost,auc_ctr,auc_ctcvr], - fetch_info=['epoch %d batch loss' % (epoch), "auc_ctr","auc_ctcvr"], - print_period=20, - debug=False) - model_dir = os.path.join(args.model_dir,'epoch_' + str(epoch + 1), "checkpoint") - main_program = fluid.default_main_program() - fluid.io.save(main_program,model_dir) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - vocab_size =utils.get_vocab_size(args.vocab_path) - train(args, vocab_size, args.train_data_path) diff --git a/PaddleRec/multiview_simnet/infer.py b/PaddleRec/multiview_simnet/infer.py deleted file mode 100644 index 9dea35b0d7..0000000000 --- a/PaddleRec/multiview_simnet/infer.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import logging -import paddle.fluid as fluid -import paddle -import time -import reader as reader -from nets import MultiviewSimnet, SimpleEncoderFactory - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def check_version(): - """ - Log error and exit when the installed version of paddlepaddle is - not satisfied. - """ - err = "PaddlePaddle version 1.6 or higher is required, " \ - "or a suitable develop version is satisfied as well. \n" \ - "Please make sure the version is good with your code." \ - - try: - fluid.require_version('1.6.0') - except Exception as e: - logger.error(err) - sys.exit(1) - - -def parse_args(): - parser = argparse.ArgumentParser("multi-view simnet") - parser.add_argument("--train_file", type=str, help="Training file") - parser.add_argument("--valid_file", type=str, help="Validation file") - parser.add_argument( - "--epochs", type=int, default=10, help="Number of epochs for training") - parser.add_argument( - "--model_dir", - type=str, - default='model_output', - help="Model output folder") - parser.add_argument( - "--query_slots", type=int, default=1, help="Number of query slots") - parser.add_argument( - "--title_slots", type=int, default=1, help="Number of title slots") - parser.add_argument( - "--query_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--title_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--query_encode_dim", - type=int, - default=128, - help="Dimension of query encoder output") - parser.add_argument( - "--title_encode_dim", - type=int, - default=128, - help="Dimension of title encoder output") - parser.add_argument( - "--batch_size", type=int, default=128, help="Batch size for training") - parser.add_argument( - "--embedding_dim", - type=int, - default=128, - help="Default Dimension of Embedding") - parser.add_argument( - "--sparse_feature_dim", - type=int, - default=1000001, - help="Sparse feature hashing space for index processing") - parser.add_argument( - "--hidden_size", type=int, default=128, help="Hidden dim") - return parser.parse_args() - - -def start_infer(args, model_path): - dataset = reader.SyntheticDataset(args.sparse_feature_dim, args.query_slots, - args.title_slots) - test_reader = fluid.io.batch( - fluid.io.shuffle( - dataset.valid(), buf_size=args.batch_size * 100), - batch_size=args.batch_size) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - args.model_dir, exe) - t0 = time.time() - step_id = 0 - feeder = fluid.DataFeeder( - program=infer_program, feed_list=feed_target_names, place=place) - for batch_id, data in enumerate(test_reader()): - step_id += 1 - loss_val, correct_val = exe.run(infer_program, - feed=feeder.feed(data), - fetch_list=fetch_vars) - logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}" - .format(step_id, batch_id, loss_val, - float(correct_val) / args.batch_size)) - - -def main(): - args = parse_args() - start_infer(args, args.model_dir) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - check_version() - main() diff --git a/PaddleRec/multiview_simnet/train.py b/PaddleRec/multiview_simnet/train.py deleted file mode 100644 index 5bbd169c53..0000000000 --- a/PaddleRec/multiview_simnet/train.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import logging -import paddle.fluid as fluid -import paddle -import time -import reader as reader -from nets import MultiviewSimnet, SimpleEncoderFactory - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser("multi-view simnet") - parser.add_argument("--train_file", type=str, help="Training file") - parser.add_argument("--valid_file", type=str, help="Validation file") - parser.add_argument( - "--epochs", type=int, default=10, help="Number of epochs for training") - parser.add_argument( - "--model_output_dir", - type=str, - default='model_output', - help="Model output folder") - parser.add_argument( - "--query_slots", type=int, default=1, help="Number of query slots") - parser.add_argument( - "--title_slots", type=int, default=1, help="Number of title slots") - parser.add_argument( - "--query_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--title_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--query_encode_dim", - type=int, - default=128, - help="Dimension of query encoder output") - parser.add_argument( - "--title_encode_dim", - type=int, - default=128, - help="Dimension of title encoder output") - parser.add_argument( - "--batch_size", type=int, default=128, help="Batch size for training") - parser.add_argument( - "--embedding_dim", - type=int, - default=128, - help="Default Dimension of Embedding") - parser.add_argument( - "--sparse_feature_dim", - type=int, - default=1000001, - help="Sparse feature hashing space" - "for index processing") - parser.add_argument( - "--hidden_size", type=int, default=128, help="Hidden dim") - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - return parser.parse_args() - - -def check_version(): - """ - Log error and exit when the installed version of paddlepaddle is - not satisfied. - """ - err = "PaddlePaddle version 1.6 or higher is required, " \ - "or a suitable develop version is satisfied as well. \n" \ - "Please make sure the version is good with your code." \ - - try: - fluid.require_version('1.6.0') - except Exception as e: - logger.error(err) - sys.exit(1) - - -def start_train(args): - if args.enable_ce: - SEED = 102 - fluid.default_startup_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - dataset = reader.SyntheticDataset(args.sparse_feature_dim, args.query_slots, - args.title_slots) - train_reader = fluid.io.batch( - fluid.io.shuffle( - dataset.train(), buf_size=args.batch_size * 100), - batch_size=args.batch_size) - place = fluid.CPUPlace() - factory = SimpleEncoderFactory() - query_encoders = [ - factory.create(args.query_encoder, args.query_encode_dim) - for i in range(args.query_slots) - ] - title_encoders = [ - factory.create(args.title_encoder, args.title_encode_dim) - for i in range(args.title_slots) - ] - m_simnet = MultiviewSimnet(args.sparse_feature_dim, args.embedding_dim, - args.hidden_size) - m_simnet.set_query_encoder(query_encoders) - m_simnet.set_title_encoder(title_encoders) - all_slots, avg_cost, correct = m_simnet.train_net() - optimizer = fluid.optimizer.Adam(learning_rate=1e-4) - optimizer.minimize(avg_cost) - startup_program = fluid.default_startup_program() - loop_program = fluid.default_main_program() - - exe = fluid.Executor(place) - exe.run(startup_program) - loader = fluid.io.DataLoader.from_generator( - feed_list=all_slots, capacity=10000, iterable=True) - loader.set_sample_list_generator(train_reader, places=place) - - total_time = 0 - ce_info = [] - for pass_id in range(args.epochs): - start_time = time.time() - for batch_id, data in enumerate(loader()): - loss_val, correct_val = exe.run(loop_program, - feed=data, - fetch_list=[avg_cost, correct]) - logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}" - .format(pass_id, batch_id, loss_val, - float(correct_val) / args.batch_size)) - ce_info.append(loss_val[0]) - end_time = time.time() - total_time += end_time - start_time - fluid.io.save_inference_model(args.model_output_dir, - [val.name for val in all_slots], - [avg_cost, correct], exe) - - # only for ce - if args.enable_ce: - threads_num, cpu_num = get_cards(args) - epoch_idx = args.epochs - ce_loss = 0 - try: - ce_loss = ce_info[-2] - except: - logger.error("ce info error") - - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_loss_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_loss)) - - -def get_cards(args): - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return int(threads_num), int(cpu_num) - - -def main(): - args = parse_args() - start_train(args) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - check_version() - main() diff --git a/PaddleRec/ncf/infer.py b/PaddleRec/ncf/infer.py deleted file mode 100644 index a9b628ab6b..0000000000 --- a/PaddleRec/ncf/infer.py +++ /dev/null @@ -1,34 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from gmf import GMF -from mlp import MLP -from neumf import NeuMF -from Dataset import Dataset -from evaluate import evaluate_model -import logging -import paddle -import args -import utils -import time -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - dataset = Dataset(args.path + args.dataset) - testRatings, testNegatives = dataset.testRatings, dataset.testNegatives - topK = 10 - - begin = time.time() - model_path = args.model_dir + "/epoch_" + str(12) - (hits, ndcgs) = evaluate_model(args, testRatings, testNegatives, topK, model_path) - hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() - end = time.time() - - logger.info("epoch: {}, epoch_time: {:.5f}s, HR: {:.5f}, NDCG: {:.5f}".format(args.epochs, end - begin, hr, ndcg)) - \ No newline at end of file diff --git a/PaddleRec/ncf/train.py b/PaddleRec/ncf/train.py deleted file mode 100644 index d1d79e59cf..0000000000 --- a/PaddleRec/ncf/train.py +++ /dev/null @@ -1,70 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from gmf import GMF -from mlp import MLP -from neumf import NeuMF -from Dataset import Dataset -import logging -import paddle -import args -import utils -import time -from evaluate import evaluate_model - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args, train_data_path): - print("use_gpu:{}, NeuMF:{}, epochs:{}, batch_size:{}, num_factors:{}, num_neg:{}, lr:{}, model_dir:{}, layers:{}".format( - args.use_gpu, args.NeuMF, args.epochs, args.batch_size, args.num_factors, args.num_neg, args.lr, args.model_dir, args.layers)) - dataset = Dataset(args.path + args.dataset) - testRatings, testNegatives = dataset.testRatings, dataset.testNegatives - - train_data_generator = utils.Dataset() - train_reader = fluid.io.batch(train_data_generator.train(train_data_path, True), batch_size=args.batch_size) - - inputs = utils.input_data(True) - if args.GMF: - model = GMF() - loss, pred = model.net(inputs, args.num_users, args.num_items, args.num_factors) - elif args.MLP: - model = MLP() - loss, pred = model.net(inputs, args.num_users, args.num_items, args.layers) - elif args.NeuMF: - model = NeuMF() - loss, pred = model.net(inputs, args.num_users, args.num_items, args.num_factors, args.layers) - - optimizer = fluid.optimizer.AdamOptimizer(args.lr) - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - loader = fluid.io.DataLoader.from_generator( - feed_list=inputs, capacity=args.batch_size, iterable=True) - loader.set_sample_list_generator(train_reader, places=place) - - for epoch in range(args.epochs): - - for batch_id, data in enumerate(loader()): - begin = time.time() - loss_val = exe.run(program=fluid.default_main_program(), - feed=data, - fetch_list=[loss.name], - return_numpy=True) - end = time.time() - logger.info("epoch: {}, batch_id: {}, batch_time: {:.5f}s, loss: {:.5f}".format(epoch, batch_id, end - begin, np.array(loss_val)[0][0])) - - save_dir = "%s/epoch_%d" % (args.model_dir, epoch) - feed_var_names = ["user_input", "item_input"] - fetch_vars = [pred] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - train(args, args.train_data_path) diff --git a/PaddleRec/rerank/listwise/infer.py b/PaddleRec/rerank/listwise/infer.py deleted file mode 100644 index f024d625c4..0000000000 --- a/PaddleRec/rerank/listwise/infer.py +++ /dev/null @@ -1,84 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -import logging -import args -import random -import time -from evaluator import BiRNN - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def set_zero(var_name, scope=fluid.global_scope(), place=fluid.CPUPlace(), param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - -def run_infer(args): - model = BiRNN() - inference_scope = fluid.Scope() - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_dir, 'epoch_' + str(args.test_epoch), "checkpoint") - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - inputs = model.input_data(args.item_len) - loss, auc_val, batch_auc, auc_states = model.net(inputs, args.hidden_size, args.batch_size*args.sample_size, args.item_vocab, args.embd_dim) - exe = fluid.Executor(place) - - fluid.load(fluid.default_main_program(), cur_model_path, exe) - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - # Build a random data set. - user_slot_names = [] - item_slot_names = [] - lens = [] - labels = [] - user_id = 0 - for i in range(args.sample_size): - user_slot_name = [] - for j in range(args.batch_size): - user_slot_name.append(user_id) - user_id += 1 - user_slot_names.append(user_slot_name) - - item_slot_name = np.random.randint(args.item_vocab, size=(args.batch_size, args.item_len)) - item_slot_names.append(item_slot_name) - lenght = np.array([args.item_len]*args.batch_size) - lens.append(lenght) - label = np.random.randint(2, size=(args.batch_size, args.item_len)) - labels.append(label) - - for i in range(args.sample_size): - begin = time.time() - loss_val, auc = exe.run(test_program, - feed={ - "user_slot_names": np.array(user_slot_names[i]).reshape(args.batch_size, 1), - "item_slot_names": item_slot_names[i].astype('int64'), - "lens": lens[i].astype('int64'), - "labels": labels[i].astype('int64') - }, - return_numpy=True, - fetch_list=[loss.name, auc_val]) - end = time.time() - logger.info("batch_time: {:.5f}s, loss: {:.5f}, auc: {:.5f}".format( - end-begin, float(np.array(loss_val)), float(np.array(auc)))) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - run_infer(args) \ No newline at end of file diff --git a/PaddleRec/rerank/listwise/train.py b/PaddleRec/rerank/listwise/train.py deleted file mode 100644 index 7bc1e58e55..0000000000 --- a/PaddleRec/rerank/listwise/train.py +++ /dev/null @@ -1,72 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -import logging -import args -import random -import time -from evaluator import BiRNN - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args): - - model = BiRNN() - inputs = model.input_data(args.item_len) - loss, auc_val, batch_auc, auc_states = model.net(inputs, args.hidden_size, args.batch_size*args.sample_size, args.item_vocab, args.embd_dim) - - optimizer = fluid.optimizer.Adam(learning_rate=args.base_lr, epsilon=1e-4) - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - # Build a random data set. - user_slot_names = [] - item_slot_names = [] - lens = [] - labels = [] - user_id = 0 - for i in range(args.sample_size): - user_slot_name = [] - for j in range(args.batch_size): - user_slot_name.append(user_id) - user_id += 1 - user_slot_names.append(user_slot_name) - - item_slot_name = np.random.randint(args.item_vocab, size=(args.batch_size, args.item_len)) - item_slot_names.append(item_slot_name) - lenght = np.array([args.item_len]*args.batch_size) - lens.append(lenght) - label = np.random.randint(2, size=(args.batch_size, args.item_len)) - labels.append(label) - - for epoch in range(args.epochs): - for i in range(args.sample_size): - begin = time.time() - loss_val, auc = exe.run(fluid.default_main_program(), - feed={ - "user_slot_names": np.array(user_slot_names[i]).reshape(args.batch_size, 1), - "item_slot_names": item_slot_names[i].astype('int64'), - "lens": lens[i].astype('int64'), - "labels": labels[i].astype('int64') - }, - return_numpy=True, - fetch_list=[loss.name, auc_val]) - end = time.time() - logger.info("epoch_id: {}, batch_time: {:.5f}s, loss: {:.5f}, auc: {:.5f}".format( - epoch, end-begin, float(np.array(loss_val)), float(np.array(auc)))) - - #save model - model_dir = os.path.join(args.model_dir, 'epoch_' + str(epoch + 1), "checkpoint") - main_program = fluid.default_main_program() - fluid.save(main_program, model_dir) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - train(args) \ No newline at end of file diff --git a/PaddleRec/tagspace/infer.py b/PaddleRec/tagspace/infer.py deleted file mode 100644 index 58fe933cc3..0000000000 --- a/PaddleRec/tagspace/infer.py +++ /dev/null @@ -1,102 +0,0 @@ -import sys -import argparse -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle -import utils - - -def parse_args(): - parser = argparse.ArgumentParser("tagspace benchmark.") - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--vocab_tag_path', - type=str, - default='vocab_tag.txt', - help='vocab path') - parser.add_argument( - '--start_index', type=int, default='1', help='start index') - parser.add_argument( - '--last_index', type=int, default='10', help='end index') - parser.add_argument( - '--model_dir', type=str, default='model_', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - args = parser.parse_args() - return args - - -def infer(test_reader, vocab_tag, use_cuda, model_path, epoch): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - model_path, exe) - t0 = time.time() - step_id = 0 - true_num = 0 - all_num = 0 - size = vocab_tag - value = [] - print("epoch " + str(epoch) + " start") - for data in test_reader(): - step_id += 1 - lod_text_seq = utils.to_lodtensor([dat[0] for dat in data], place) - lod_tag = utils.to_lodtensor([dat[1] for dat in data], place) - lod_pos_tag = utils.to_lodtensor([dat[2] for dat in data], place) - para = exe.run(infer_program, - feed={"text": lod_text_seq, - "pos_tag": lod_tag}, - fetch_list=fetch_vars, - return_numpy=False) - value.append(para[0]._get_float_element(0)) - if step_id % size == 0 and step_id > 1: - all_num += 1 - true_pos = [dat[2] for dat in data][0][0] - if value.index(max(value)) == int(true_pos): - true_num += 1 - value = [] - print("epoch:" + str(epoch) + "\tacc:" + str(1.0 * true_num / all_num)) - t1 = time.time() - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - vocab_tag_path = args.vocab_tag_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_text, vocab_tag, test_reader = utils.prepare_data( - test_dir, - "", - vocab_tag_path, - batch_size=1, - neg_size=0, - buffer_size=1000, - is_train=False) - - for epoch in range(start_index, last_index + 1): - epoch_path = model_dir + "/epoch_" + str(epoch) - infer( - test_reader=test_reader, - vocab_tag=vocab_tag, - use_cuda=False, - model_path=epoch_path, - epoch=epoch) diff --git a/PaddleRec/tagspace/train.py b/PaddleRec/tagspace/train.py deleted file mode 100644 index ee2054a7b1..0000000000 --- a/PaddleRec/tagspace/train.py +++ /dev/null @@ -1,173 +0,0 @@ -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import paddle -import paddle.fluid as fluid -import time -import utils -import net - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("TagSpace benchmark.") - parser.add_argument( - '--neg_size', type=int, default=3, help='number of neg item') - parser.add_argument( - '--train_dir', type=str, default='train_data', help='train file') - parser.add_argument( - '--vocab_text_path', type=str, default='vocab_text.txt', help='text') - parser.add_argument( - '--vocab_tag_path', type=str, default='vocab_tag.txt', help='tag') - parser.add_argument( - '--model_dir', type=str, default='model_', help='model dir') - parser.add_argument( - '--batch_size', type=int, default=5, help='num of batch size') - parser.add_argument( - '--print_batch', type=int, default=10, help='num of print batch') - parser.add_argument( - '--pass_num', type=int, default=10, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether use gpu') - parser.add_argument( - '--parallel', type=int, default=0, help='whether parallel') - parser.add_argument( - '--base_lr', type=float, default=0.01, help='learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - args = parser.parse_args() - return args - - -def get_cards(args): - return args.num_devices - - -def train(): - """ do training """ - args = parse_args() - if args.enable_ce: - fluid.default_startup_program().random_seed = SEED - fluid.default_main_program().random_seed = SEED - train_dir = args.train_dir - vocab_text_path = args.vocab_text_path - vocab_tag_path = args.vocab_tag_path - use_cuda = True if args.use_cuda else False - parallel = True if args.parallel else False - batch_size = args.batch_size - neg_size = args.neg_size - print("use_cuda: {}, parallel: {}, batch_size: {}, neg_size: {} " - .format(use_cuda, parallel, batch_size, neg_size)) - vocab_text_size, vocab_tag_size, train_reader = utils.prepare_data( - file_dir=train_dir, - vocab_text_path=vocab_text_path, - vocab_tag_path=vocab_tag_path, - neg_size=neg_size, - batch_size=batch_size * get_cards(args), - buffer_size=batch_size * 100, - is_train=True) - """ train network """ - # Train program - avg_cost, correct, cos_pos = net.network( - vocab_text_size, vocab_tag_size, neg_size=neg_size) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - if parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - pass_num = args.pass_num - model_dir = args.model_dir - fetch_list = [avg_cost.name] - total_time = 0.0 - ce_info = [] - for pass_idx in range(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - t0 = time.time() - for batch_id, data in enumerate(train_reader()): - lod_text_seq = utils.to_lodtensor([dat[0] for dat in data], place) - lod_pos_tag = utils.to_lodtensor([dat[1] for dat in data], place) - lod_neg_tag = utils.to_lodtensor([dat[2] for dat in data], place) - loss_val, correct_val = train_exe.run( - feed={ - "text": lod_text_seq, - "pos_tag": lod_pos_tag, - "neg_tag": lod_neg_tag - }, - fetch_list=[avg_cost.name, correct.name]) - ce_info.append( - float(np.sum(correct_val)) / (args.num_devices * batch_size)) - if batch_id % args.print_batch == 0: - print("TRAIN --> pass: {} batch_num: {} avg_cost: {}, acc: {}" - .format(pass_idx, (batch_id + 10) * batch_size, - np.mean(loss_val), - float(np.sum(correct_val)) / (args.num_devices * - batch_size))) - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, batch_id, total_time / epoch_idx)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - feed_var_names = ["text", "pos_tag"] - fetch_vars = [cos_pos] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - # only for ce - if args.enable_ce: - ce_acc = 0 - try: - ce_acc = ce_info[-2] - except: - logger.error("ce info error") - epoch_idx = args.pass_num - device = get_device(args) - if args.use_cuda: - gpu_num = device[1] - print("kpis\teach_pass_duration_gpu%s\t%s" % - (gpu_num, total_time / epoch_idx)) - print("kpis\ttrain_acc_gpu%s\t%s" % (gpu_num, ce_acc)) - else: - cpu_num = device[1] - threads_num = device[2] - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_acc_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_acc)) - - print("finish training") - - -def get_device(args): - if args.use_cuda: - gpus = os.environ.get("CUDA_VISIBLE_DEVICES", 1) - gpu_num = len(gpus.split(',')) - return "gpu", gpu_num - else: - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return "cpu", int(cpu_num), int(threads_num) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/tdm/tdm_demo/dataset_generator.py b/PaddleRec/tdm/tdm_demo/dataset_generator.py deleted file mode 100644 index fccd37b805..0000000000 --- a/PaddleRec/tdm/tdm_demo/dataset_generator.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding=utf8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import json -import pickle -import time -import random -import os -import numpy as np -import sys -import paddle.fluid.incubate.data_generator as dg - - -class TDMDataset(dg.MultiSlotStringDataGenerator): - """ - DacDataset: inheritance MultiSlotDataGeneratior, Implement data reading - Help document: http://wiki.baidu.com/pages/viewpage.action?pageId=728820675 - """ - - def infer_reader(self, infer_file_list, batch): - """ - Read test_data line by line & yield batch - """ - - def local_iter(): - """Read file line by line""" - for fname in infer_file_list: - with open(fname, "r") as fin: - for line in fin: - one_data = (line.strip('\n')).split('\t') - input_emb = one_data[0].split(' ') - - yield [input_emb] - - import paddle - batch_iter = fluid.io.batch(local_iter, batch) - return batch_iter - - def generate_sample(self, line): - """ - Read the data line by line and process it as a dictionary - """ - - def iterator(): - """ - This function needs to be implemented by the user, based on data format - """ - features = (line.strip('\n')).split('\t') - input_emb = features[0].split(' ') - item_label = [features[1]] - - feature_name = ["input_emb", "item_label"] - yield zip(feature_name, [input_emb] + [item_label]) - - return iterator - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - d = TDMDataset() - d.run_from_stdin() diff --git a/PaddleRec/tdm/tdm_demo/distributed_train.py b/PaddleRec/tdm/tdm_demo/distributed_train.py deleted file mode 100644 index e82e5f74ce..0000000000 --- a/PaddleRec/tdm/tdm_demo/distributed_train.py +++ /dev/null @@ -1,250 +0,0 @@ -# -*- coding=utf-8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -from __future__ import print_function -import os -import time -import numpy as np -import logging -import random -from shutil import copyfile -import paddle.fluid as fluid -import paddle.fluid.incubate.fleet.base.role_maker as role_maker - -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from paddle.fluid.incubate.fleet.utils.hdfs import HDFSClient -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory - -from args import print_arguments, parse_args -from utils import tdm_sampler_prepare, tdm_child_prepare, tdm_emb_prepare - -from train_network import TdmTrainNet - - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -hadoop_home = os.getenv("HADOOP_HOME") -configs = { - "fs.default.name": os.getenv("FS_NAME"), - "hadoop.job.ugi": os.getenv("FS_UGI") -} -client = HDFSClient(hadoop_home, configs) - - -def get_dataset(inputs, args): - """get dataset""" - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(inputs) - dataset.set_pipe_command("python ./dataset_generator.py") - dataset.set_batch_size(args.batch_size) - dataset.set_thread(int(args.cpu_num)) - file_list = [ - str(args.train_files_path) + "/%s" % x - for x in os.listdir(args.train_files_path) - ] - - # 请确保每一个训练节点都持有不同的训练文件 - # 当我们用本地多进程模拟分布式时，每个进程需要拿到不同的文件 - # 使用 fleet.split_files 可以便捷的以文件为单位分配训练样本 - if not int(args.is_cloud): - file_list = fleet.split_files(file_list) - logger.info("file list: {}".format(file_list)) - total_example_num = get_example_num(file_list) - return dataset, file_list, total_example_num - - -def train(args): - """run train""" - # set random - program = fluid.default_main_program() - program.random_seed = args.random_seed - - # 根据环境变量确定当前机器/进程在分布式训练中扮演的角色 - # 然后使用 fleet api的 init()方法初始化这个节点 - role = role_maker.PaddleCloudRoleMaker() - fleet.init(role) - - # 我们还可以进一步指定分布式的运行模式，通过 DistributeTranspilerConfig进行配置 - # 如下，我们设置分布式运行模式为异步(async)，同时将参数进行切分，以分配到不同的节点 - if args.sync_mode == "sync": - strategy = StrategyFactory.create_sync_strategy() - elif args.sync_mode == "half_async": - strategy = StrategyFactory.create_half_async_strategy() - elif args.sync_mode == "async": - strategy = StrategyFactory.create_async_strategy() - - # set model - logger.info("TDM Begin build network.") - tdm_model = TdmTrainNet(args) - inputs = tdm_model.input_data() - - logger.info("TDM Begin load tree travel & layer.") - avg_cost, acc = tdm_model.tdm(inputs) - logger.info("TDM End build network.") - # 配置分布式的optimizer，传入我们指定的strategy，构建program - optimizer = fluid.optimizer.AdamOptimizer( - learning_rate=args.learning_rate, lazy_mode=True) - - optimizer = fleet.distributed_optimizer(optimizer, strategy) - optimizer.minimize(avg_cost) - logger.info("TDM End append backward.") - - # 根据节点角色，分别运行不同的逻辑 - if fleet.is_server(): - logger.info("TDM Run server ...") - # 初始化及运行参数服务器节点 - logger.info("TDM init model path: {}".format( - args.init_model_files_path)) - # 模型中除了tdm树结构相关的变量都应该在此处初始化 - fleet.init_server(args.init_model_files_path) - lr = fluid.global_scope().find_var("learning_rate_0") - if lr: - lr.get_tensor().set(np.array(args.learning_rate).astype('float32'), - fluid.CPUPlace()) - logger.info("TDM Set learning rate {}".format(args.learning_rate)) - else: - logger.info("TDM Didn't find learning_rate_0 param") - logger.info("TDM load End") - - fleet.run_server() - logger.info("TDM Run server success!") - elif fleet.is_worker(): - logger.info("TDM Run worker ...") - # 初始化工作节点 - fleet.init_worker() - place = fluid.CPUPlace() - exe = fluid.Executor(place) - logger.info("TDM Run Startup Begin") - # 初始化含有分布式流程的fleet.startup_program - exe.run(fleet.startup_program) - - # Set Learning Rate - lr = fluid.global_scope().find_var("learning_rate_0") - if lr: - lr.get_tensor().set(np.array(args.learning_rate).astype('float32'), - place) - logger.info("TDM Set learning rate {}".format(args.learning_rate)) - - # Set TDM Variable - logger.info("TDM Begin load parameter.") - # Set TDM_Tree_Info - # 树结构相关的变量不参与网络更新，不存储于参数服务器，因此需要在本地手动Set - tdm_param_prepare_dict = tdm_sampler_prepare(args) - tdm_param_prepare_dict['info_array'] = tdm_child_prepare(args) - Numpy_model = {} - Numpy_model['TDM_Tree_Travel'] = tdm_param_prepare_dict['travel_array'] - Numpy_model['TDM_Tree_Layer'] = tdm_param_prepare_dict['layer_array'] - Numpy_model['TDM_Tree_Info'] = tdm_param_prepare_dict['info_array'] - # Numpy_model['TDM_Tree_Emb'] = tdm_emb_prepare(args) - # 分布式训练中，Emb存储与参数服务器，无需在本地set - for param_name in Numpy_model: - param_t = fluid.global_scope().find_var(param_name).get_tensor() - param_t.set(Numpy_model[str(param_name)].astype('int32'), place) - - logger.info("TDM Run Startup End") - - # Train loop - dataset, file_list, example_num = get_dataset(inputs, args) - logger.info("TDM Distributed training begin ...") - for epoch in range(args.epoch_num): - # local shuffle - random.shuffle(file_list) - dataset.set_filelist(file_list) - - # 训练节点运行的是经过分布式裁剪的fleet.mian_program - start_time = time.time() - exe.train_from_dataset(program=fleet.main_program, - dataset=dataset, - fetch_list=[acc, avg_cost], - fetch_info=["Epoch {} acc ".format( - epoch), "Epoch {} loss ".format(epoch)], - print_period=1, - debug=False) - end_time = time.time() - logger.info("Epoch {} finished, use time {} second, speed {} example/s".format( - epoch, end_time - start_time, example_num * 1.0 / (end_time - start_time))) - - # 默认使用0号节点保存模型 - if fleet.is_first_worker(): - model_path = os.path.join( - args.model_files_path, "epoch_" + str(epoch)) - fleet.save_persistables(executor=exe, dirname=model_path) - logger.info("Begin upload files") - # upload_files(model_path, warm_up=False) - # 在分布式环境下时，支持上传模型到hdfs - logger.info("TDM Before stop worker") - fleet.stop_worker() - logger.info("TDM Distributed training success!") - - -def upload_files(local_path, warm_up=False): - """ - upload files to hdfs - """ - remote = os.getenv("OUTPUT_PATH") - job_id = os.getenv("SYS_JOB_ID") - local = local_path.split('/')[-1] - remote_path = "{}/{}/{}/{}".format(remote, job_id, "model", local) - client.makedirs(remote_path) - hadoop_path = "{}/".format(remote_path) - - def is_adam_param(name): - adam_name = ['bias_beta', 'bias_moment', - 'moment1_0', 'moment2_0', 'pow_acc'] - for i in adam_name: - if i in name: - return True - return False - - if not warm_up: - infer_model_path = os.path.join(os.getcwd(), 'infer_model_'+local) - if not os.path.exists(infer_model_path): - os.makedirs(infer_model_path) - for root, _, files in os.walk(local_path): - for f in files: - if not is_adam_param(f): - copyfile(os.path.join(root, f), - os.path.join(infer_model_path, f)) - local_path = infer_model_path - - client.upload(hdfs_path=hadoop_path, local_path=local_path, - multi_processes=5, overwrite=False, - retry_times=3) - - -def get_example_num(file_list): - """ - Count the number of samples in the file - """ - count = 0 - for f in file_list: - last_count = count - for index, line in enumerate(open(f, 'r')): - count += 1 - logger.info("file : %s has %s example" % (f, count - last_count)) - logger.info("Total example : %s" % count) - return count - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - print(os.getcwd()) - args = parse_args() - print_arguments(args) - train(args) diff --git a/PaddleRec/tdm/tdm_demo/local_infer.py b/PaddleRec/tdm/tdm_demo/local_infer.py deleted file mode 100644 index aefedc1763..0000000000 --- a/PaddleRec/tdm/tdm_demo/local_infer.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding=utf-8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -from __future__ import print_function -import os -import time -import numpy as np -import logging -import argparse - -import paddle -import paddle.fluid as fluid -from paddle.fluid import profiler - -from args import print_arguments, parse_args -from infer_network import TdmInferNet -from dataset_generator import TDMDataset - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def to_tensor(data, place): - """ - Convert data to paddle tensor - """ - flattened_data = np.concatenate(data, axis=0).astype("float32") - flattened_data = flattened_data.reshape([-1, 768]) - res = fluid.Tensor() - res.set(flattened_data, place) - return res - - -def data2tensor(data, place): - """ - Dataset prepare - """ - input_emb = to_tensor([x[0] for x in data], place) - - return input_emb - - -def run_infer(args, model_path): - """run infer""" - logger.info("Infer Begin") - file_list = [ - str(args.test_files_path) + "/%s" % x - for x in os.listdir(args.test_files_path) - ] - - tdm_model = TdmInferNet(args) - inputs = tdm_model.input_data() - res_item = tdm_model.infer_net(inputs) - test_reader = TDMDataset().infer_reader(file_list, args.batch_size) - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - - path = os.path.join(args.model_files_path, model_path) - fluid.io.load_persistables( - executor=exe, - dirname=path, - main_program=fluid.default_main_program()) - - logger.info("Load persistables from \"{}\"".format(path)) - - if args.save_init_model: - logger.info("Begin Save infer model.") - model_path = (str(args.model_files_path) + "/" + "infer_model") - fluid.io.save_inference_model(executor=exe, dirname=model_path, - feeded_var_names=[ - 'input_emb', 'first_layer_node', 'first_layer_node_mask'], - target_vars=[res_item]) - logger.info("End Save infer model.") - - first_layer_node = tdm_model.first_layer_node - first_layer_nums = len(first_layer_node) - first_layer_node = np.array(first_layer_node) - first_layer_node = first_layer_node.reshape((1, -1)).astype('int64') - first_layer_node = first_layer_node.repeat(args.batch_size, axis=0) - # 在demo中，假设infer起始层的节点都不是叶子节点，mask=0 - # 若真实的起始层含有叶子节点，则对应位置的 mask=1 - first_layer_mask = ( - np.zeros((args.batch_size, first_layer_nums))).astype('int64') - - for batch_id, data in enumerate(test_reader()): - input_emb = data2tensor(data, place) - item_res = exe.run(fluid.default_main_program(), - feed={"input_emb": input_emb, - "first_layer_node": first_layer_node, - "first_layer_node_mask": first_layer_mask}, - fetch_list=[res_item]) - logger.info("TEST --> batch: {} infer_item {}".format( - batch_id, item_res)) - logger.info("Inference complete!") - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = parse_args() - print_arguments(args) - # 在此处指定infer模型所在的文件夹 - path = "epoch_0" - run_infer(args, path) diff --git a/PaddleRec/tdm/tdm_demo/local_train.py b/PaddleRec/tdm/tdm_demo/local_train.py deleted file mode 100644 index 9660a23e26..0000000000 --- a/PaddleRec/tdm/tdm_demo/local_train.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding=utf-8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -from __future__ import print_function -import os -import time -import numpy as np -import logging -import argparse - -import paddle -import paddle.fluid as fluid - -from args import print_arguments, parse_args -from utils import tdm_sampler_prepare, tdm_child_prepare -from train_network import TdmTrainNet - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def get_dataset(inputs, args): - """ - get dataset - """ - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(inputs) - dataset.set_pipe_command("python ./dataset_generator.py") - dataset.set_batch_size(args.batch_size) - dataset.set_thread(int(args.cpu_num)) - file_list = [ - str(args.train_files_path) + "/%s" % x - for x in os.listdir(args.train_files_path) - ] - dataset.set_filelist(file_list) - logger.info("file list: {}".format(file_list)) - return dataset - - -def run_train(args): - """ - run train - """ - logger.info("TDM Begin build network.") - tdm_model = TdmTrainNet(args) - inputs = tdm_model.input_data() - avg_cost, acc = tdm_model.tdm(inputs) - logger.info("TDM End build network.") - - dataset = get_dataset(inputs, args) - - optimizer = fluid.optimizer.AdamOptimizer( - learning_rate=args.learning_rate, - lazy_mode=True) - optimizer.minimize(avg_cost) - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.load_model: - # 从paddle二进制模型加载参数 - path = args.init_model_files_path - fluid.io.load_persistables( - executor=exe, - dirname=path, - main_program=fluid.default_main_program()) - lr = fluid.global_scope().find_var("learning_rate_0").get_tensor() - lr.set(np.array(args.learning_rate).astype('float32'), place) - logger.info("Load persistables from \"{}\"".format(path)) - else: - # 将明文树结构及数据，set到组网中的Variale中 - # 不使用NumpyInitialize方法是考虑到树结构相关数据size过大，有性能风险 - Numpy_model = {} - Numpy_model['TDM_Tree_Travel'] = tdm_model.tdm_param_prepare_dict['travel_array'] - Numpy_model['TDM_Tree_Layer'] = tdm_model.tdm_param_prepare_dict['layer_array'] - Numpy_model['TDM_Tree_Info'] = tdm_model.tdm_param_prepare_dict['info_array'] - Numpy_model['TDM_Tree_Emb'] = tdm_model.tdm_param_prepare_dict['emb_array'] - for param_name in Numpy_model: - param_t = fluid.global_scope().find_var(param_name).get_tensor() - if param_name == 'TDM_Tree_Emb': - param_t.set(Numpy_model[str(param_name) - ].astype('float32'), place) - else: - param_t.set(Numpy_model[str(param_name) - ].astype('int32'), place) - - if args.save_init_model or not args.load_model: - logger.info("Begin Save Init model.") - model_path = os.path.join(args.model_files_path, "init_model") - fluid.io.save_persistables(executor=exe, dirname=model_path) - logger.info("End Save Init model.") - - logger.info("TDM Local training begin ...") - for epoch in range(args.epoch_num): - start_time = time.time() - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[acc, avg_cost], - fetch_info=["Epoch {} acc".format( - epoch), "Epoch {} loss".format(epoch)], - print_period=1, - debug=False, - ) - end_time = time.time() - logger.info("Epoch %d finished, use time=%d sec\n" % - (epoch, end_time - start_time)) - - model_path = os.path.join(args.model_files_path, "epoch_" + str(epoch)) - fluid.io.save_persistables(executor=exe, dirname=model_path) - - logger.info("Local training success!") - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = parse_args() - print_arguments(args) - run_train(args) diff --git a/PaddleRec/tdm/tdm_demo/predict.py b/PaddleRec/tdm/tdm_demo/predict.py deleted file mode 100644 index 2ec6e522cf..0000000000 --- a/PaddleRec/tdm/tdm_demo/predict.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- -""" -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os -import argparse -import numpy as np -import argparse -import paddle.fluid as fluid -from paddle.fluid.core import PaddleTensor -from paddle.fluid.core import AnalysisConfig -from paddle.fluid.core import create_paddle_predictor -from dataset_generator import TDMDataset -from infer_network import TdmInferNet -from args import print_arguments, parse_args -import logging - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def to_tensor(data): - """ - Convert data to paddle tensor - """ - flattened_data = np.concatenate(data, axis=0).astype("float32") - flattened_data = flattened_data.reshape([-1, 768]) - return flattened_data - - -def data2tensor(data): - """ - Dataset prepare - """ - input_emb = to_tensor([x[0] for x in data]) - - return input_emb - - -def tdm_input(input_emb, first_layer_node, first_layer_mask): - """ - Create input of tdm pred - """ - input_emb = PaddleTensor(input_emb) - first_layer_node = PaddleTensor(first_layer_node) - first_layer_mask = PaddleTensor(first_layer_mask) - return [input_emb, first_layer_node, first_layer_mask] - - -def main(): - """Predictor main""" - args = parse_args() - - config = AnalysisConfig(args.model_files_path) - config.disable_gpu() - config.enable_profile() - # config.enable_mkldnn() - config.set_cpu_math_library_num_threads(args.cpu_num) - - predictor = create_paddle_predictor(config) - tdm_model = TdmInferNet(args) - first_layer_node = tdm_model.first_layer_node - first_layer_nums = len(first_layer_node) - first_layer_node = np.array(first_layer_node) - first_layer_node = first_layer_node.reshape((1, -1)).astype('int64') - first_layer_node = first_layer_node.repeat(args.batch_size, axis=0) - first_layer_mask = ( - np.zeros((args.batch_size, first_layer_nums))).astype('int64') - - file_list = [ - str(args.test_files_path) + "/%s" % x - for x in os.listdir(args.test_files_path) - ] - test_reader = TDMDataset().infer_reader(file_list, args.batch_size) - - for batch_id, data in enumerate(test_reader()): - input_emb = data2tensor(data) - - inputs = tdm_input(input_emb, first_layer_node, first_layer_mask) - outputs = predictor.run(inputs) - output = outputs[0] - output_data = output.as_ndarray() - - logger.info("TEST --> batch: {} infer_item {}".format( - batch_id, output_data)) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - main() diff --git a/PaddleRec/word2vec/infer.py b/PaddleRec/word2vec/infer.py deleted file mode 100644 index 4fcc20fc11..0000000000 --- a/PaddleRec/word2vec/infer.py +++ /dev/null @@ -1,208 +0,0 @@ -import argparse -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle -import net -import utils -if six.PY2: - reload(sys) - sys.setdefaultencoding('utf-8') - - -def parse_args(): - parser = argparse.ArgumentParser("PaddlePaddle Word2vec infer example") - parser.add_argument( - '--dict_path', - type=str, - default='./data/data_c/1-billion_dict_word_to_id_', - help="The path of dic") - parser.add_argument( - '--infer_epoch', - action='store_true', - required=False, - default=False, - help='infer by epoch') - parser.add_argument( - '--infer_step', - action='store_true', - required=False, - default=False, - help='infer by step') - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--print_step', type=int, default='500000', help='print step') - parser.add_argument( - '--start_index', type=int, default='0', help='start index') - parser.add_argument( - '--start_batch', type=int, default='1', help='start index') - parser.add_argument( - '--end_batch', type=int, default='13', help='start index') - parser.add_argument( - '--last_index', type=int, default='100', help='last index') - parser.add_argument( - '--model_dir', type=str, default='model', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - parser.add_argument('--emb_size', type=int, default='64', help='batch_size') - args = parser.parse_args() - return args - - -def infer_epoch(args, vocab_size, test_reader, use_cuda, i2w): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - emb_size = args.emb_size - batch_size = args.batch_size - with fluid.scope_guard(fluid.Scope()): - main_program = fluid.Program() - with fluid.program_guard(main_program): - values, pred = net.infer_network(vocab_size, emb_size) - for epoch in range(start_index, last_index + 1): - copy_program = main_program.clone() - model_path = model_dir + "/pass-" + str(epoch) - fluid.load(copy_program, model_path, exe) - accum_num = 0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - b_size = len([dat[0] for dat in data]) - wa = np.array([dat[0] for dat in data]).astype( - "int64").reshape(b_size) - wb = np.array([dat[1] for dat in data]).astype( - "int64").reshape(b_size) - wc = np.array([dat[2] for dat in data]).astype( - "int64").reshape(b_size) - - label = [dat[3] for dat in data] - input_word = [dat[4] for dat in data] - para = exe.run(copy_program, - feed={ - "analogy_a": wa, - "analogy_b": wb, - "analogy_c": wc, - "all_label": np.arange(vocab_size) - .reshape(vocab_size).astype("int64"), - }, - fetch_list=[pred.name, values], - return_numpy=False) - pre = np.array(para[0]) - val = np.array(para[1]) - for ii in range(len(label)): - top4 = pre[ii] - accum_num_sum += 1 - for idx in top4: - if int(idx) in input_word[ii]: - continue - if int(idx) == int(label[ii][0]): - accum_num += 1 - break - if step_id % 1 == 0: - print("step:%d %d " % (step_id, accum_num)) - - print("epoch:%d \t acc:%.3f " % - (epoch, 1.0 * accum_num / accum_num_sum)) - - -def infer_step(args, vocab_size, test_reader, use_cuda, i2w): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - emb_size = args.emb_size - batch_size = args.batch_size - with fluid.scope_guard(fluid.Scope()): - main_program = fluid.Program() - with fluid.program_guard(main_program): - values, pred = net.infer_network(vocab_size, emb_size) - for epoch in range(start_index, last_index + 1): - for batchid in range(args.start_batch, args.end_batch): - copy_program = main_program.clone() - model_path = model_dir + "/pass-" + str(epoch) + ( - '/batch-' + str(batchid * args.print_step)) - fluid.load(copy_program, model_path, exe) - accum_num = 0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - b_size = len([dat[0] for dat in data]) - wa = np.array([dat[0] for dat in data]).astype( - "int64").reshape(b_size) - wb = np.array([dat[1] for dat in data]).astype( - "int64").reshape(b_size) - wc = np.array([dat[2] for dat in data]).astype( - "int64").reshape(b_size) - - label = [dat[3] for dat in data] - input_word = [dat[4] for dat in data] - para = exe.run( - copy_program, - feed={ - "analogy_a": wa, - "analogy_b": wb, - "analogy_c": wc, - "all_label": - np.arange(vocab_size).reshape(vocab_size), - }, - fetch_list=[pred.name, values], - return_numpy=False) - pre = np.array(para[0]) - val = np.array(para[1]) - for ii in range(len(label)): - top4 = pre[ii] - accum_num_sum += 1 - for idx in top4: - if int(idx) in input_word[ii]: - continue - if int(idx) == int(label[ii][0]): - accum_num += 1 - break - if step_id % 1 == 0: - print("step:%d %d " % (step_id, accum_num)) - print("epoch:%d \t acc:%.3f " % - (epoch, 1.0 * accum_num / accum_num_sum)) - t1 = time.time() - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - dict_path = args.dict_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_size, test_reader, id2word = utils.prepare_data( - test_dir, dict_path, batch_size=batch_size) - print("vocab_size:", vocab_size) - if args.infer_step: - infer_step( - args, - vocab_size, - test_reader=test_reader, - use_cuda=use_cuda, - i2w=id2word) - else: - infer_epoch( - args, - vocab_size, - test_reader=test_reader, - use_cuda=use_cuda, - i2w=id2word) diff --git a/PaddleRec/word2vec/train.py b/PaddleRec/word2vec/train.py deleted file mode 100644 index ae630ba25f..0000000000 --- a/PaddleRec/word2vec/train.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import print_function -import argparse -import logging -import os -import time -import math -import random -import numpy as np -import paddle -import paddle.fluid as fluid -import six -import reader -from net import skip_gram_word2vec, skip_gram_word2vec_shuffle_batch - -import utils -import sys -if six.PY2: - reload(sys) - sys.setdefaultencoding('utf-8') - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser( - description="PaddlePaddle Word2vec example") - parser.add_argument( - '--train_data_dir', - type=str, - default='./data/text', - help="The path of taining dataset") - parser.add_argument( - '--base_lr', - type=float, - default=0.01, - help="The number of learing rate (default: 0.01)") - parser.add_argument( - '--save_step', - type=int, - default=500000, - help="The number of step to save (default: 500000)") - parser.add_argument( - '--print_batch', - type=int, - default=10, - help="The number of print_batch (default: 10)") - parser.add_argument( - '--dict_path', - type=str, - default='./data/1-billion_dict', - help="The path of data dict") - parser.add_argument( - '--batch_size', - type=int, - default=500, - help="The size of mini-batch (default:500)") - parser.add_argument( - '--num_passes', - type=int, - default=10, - help="The number of passes to train (default: 10)") - parser.add_argument( - '--model_output_dir', - type=str, - default='models', - help='The path for model to store (default: models)') - parser.add_argument('--nce_num', type=int, default=5, help='nce_num') - parser.add_argument( - '--embedding_size', - type=int, - default=64, - help='sparse feature hashing space for index processing') - parser.add_argument( - '--is_sparse', - action='store_true', - required=False, - default=False, - help='embedding and nce will use sparse or not, (default: False)') - parser.add_argument( - '--with_speed', - action='store_true', - required=False, - default=False, - help='print speed or not , (default: False)') - parser.add_argument( - '--with_shuffle_batch', - action='store_true', - required=False, - default=False, - help='negative samples come from shuffle_batch op or not , (default: False)') - parser.add_argument( - '--enable_ce', action='store_true', help='If set, run the task with continuous evaluation logs.') - - return parser.parse_args() - - -def convert_python_to_tensor(weight, batch_size, sample_reader): - def __reader__(): - cs = np.array(weight).cumsum() - result = [[], []] - for sample in sample_reader(): - for i, fea in enumerate(sample): - result[i].append(fea) - if len(result[0]) == batch_size: - tensor_result = [] - for tensor in result: - t = fluid.Tensor() - dat = np.array(tensor, dtype='int64') - if len(dat.shape) > 2: - dat = dat.reshape((dat.shape[0], dat.shape[2])) - elif len(dat.shape) == 1: - dat = dat.reshape((-1, 1)) - t.set(dat, fluid.CPUPlace()) - tensor_result.append(t) - tt = fluid.Tensor() - neg_array = cs.searchsorted(np.random.sample(args.nce_num)) - neg_array = np.tile(neg_array, batch_size) - tt.set( - neg_array.reshape((batch_size, args.nce_num)), - fluid.CPUPlace()) - tensor_result.append(tt) - yield tensor_result - result = [[], []] - - return __reader__ - - -def train_loop(args, train_program, data_loader, loss, trainer_id): - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - exec_strategy = fluid.ExecutionStrategy() - exec_strategy.use_experimental_executor = True - - print("CPU_NUM:" + str(os.getenv("CPU_NUM"))) - exec_strategy.num_threads = int(os.getenv("CPU_NUM")) - - build_strategy = fluid.BuildStrategy() - if int(os.getenv("CPU_NUM")) > 1: - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce - - train_exe = fluid.ParallelExecutor( - use_cuda=False, - loss_name=loss.name, - main_program=train_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - for pass_id in range(args.num_passes): - data_loader.start() - time.sleep(10) - epoch_start = time.time() - batch_id = 0 - start = time.time() - try: - while True: - - loss_val = train_exe.run(fetch_list=[loss.name]) - loss_val = np.mean(loss_val) - - if batch_id % args.print_batch == 0: - logger.info( - "TRAIN --> pass: {} batch: {} loss: {} reader queue:{}". - format(pass_id, batch_id, - loss_val.mean(), data_loader.queue.size())) - if args.with_speed: - if batch_id % 500 == 0 and batch_id != 0: - elapsed = (time.time() - start) - start = time.time() - samples = 1001 * args.batch_size * int( - os.getenv("CPU_NUM")) - logger.info("Time used: {}, Samples/Sec: {}".format( - elapsed, samples / elapsed)) - - if batch_id % args.save_step == 0 and batch_id != 0: - model_dir = args.model_output_dir + '/pass-' + str( - pass_id) + ('/batch-' + str(batch_id)) - if trainer_id == 0: - fluid.save(fluid.default_main_program(), model_path=model_dir) - print("model saved in %s" % model_dir) - batch_id += 1 - - except fluid.core.EOFException: - data_loader.reset() - epoch_end = time.time() - logger.info("Epoch: {0}, Train total expend: {1} ".format( - pass_id, epoch_end - epoch_start)) - model_dir = args.model_output_dir + '/pass-' + str(pass_id) - if trainer_id == 0: - fluid.save(fluid.default_main_program(), model_path=model_dir) - print("model saved in %s" % model_dir) - - -def GetFileList(data_path): - return os.listdir(data_path) - - -def train(args): - # add ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - filelist = GetFileList(args.train_data_dir) - word2vec_reader = reader.Word2VecReader(args.dict_path, args.train_data_dir, - filelist, 0, 1) - - logger.info("dict_size: {}".format(word2vec_reader.dict_size)) - - if args.with_shuffle_batch: - loss, data_loader = skip_gram_word2vec_shuffle_batch( - word2vec_reader.dict_size, - args.embedding_size, - is_sparse=args.is_sparse, - neg_num=args.nce_num) - data_loader.set_sample_generator(word2vec_reader.train(), batch_size=args.batch_size, drop_last=True) - else: - np_power = np.power(np.array(word2vec_reader.id_frequencys), 0.75) - id_frequencys_pow = np_power / np_power.sum() - - loss, data_loader = skip_gram_word2vec( - word2vec_reader.dict_size, - args.embedding_size, - is_sparse=args.is_sparse, - neg_num=args.nce_num) - - data_loader.set_batch_generator( - convert_python_to_tensor(id_frequencys_pow, args.batch_size, word2vec_reader.train())) - - optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay( - learning_rate=args.base_lr, - decay_steps=100000, - decay_rate=0.999, - staircase=True)) - - optimizer.minimize(loss) - - # do local training - logger.info("run local training") - main_program = fluid.default_main_program() - train_loop(args, main_program, data_loader, loss, 0) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - args = parse_args() - utils.check_version(args.with_shuffle_batch) - train(args) diff --git a/PaddleRec/youbube_dnn/infer.py b/PaddleRec/youbube_dnn/infer.py deleted file mode 100644 index 1a8cee925d..0000000000 --- a/PaddleRec/youbube_dnn/infer.py +++ /dev/null @@ -1,55 +0,0 @@ -import paddle.fluid as fluid -import numpy as np -import pandas as pd -import time -import sys -import os -import args -import logging -from youtubednn import YoutubeDNN - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def infer(args): - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - cur_model_path = os.path.join(args.model_dir, 'epoch_' + str(args.test_epoch), "checkpoint") - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model(cur_model_path, exe) - # Build a random data set. - sample_size = 100 - watch_vecs = [] - search_vecs = [] - other_feats = [] - - for i in range(sample_size): - watch_vec = np.random.rand(1, args.watch_vec_size) - search_vec = np.random.rand(1, args.search_vec_size) - other_feat = np.random.rand(1, args.other_feat_size) - watch_vecs.append(watch_vec) - search_vecs.append(search_vec) - other_feats.append(other_feat) - - for i in range(sample_size): - l3 = exe.run(infer_program, - feed={ - "watch_vec": watch_vecs[i].astype('float32'), - "search_vec": search_vecs[i].astype('float32'), - "other_feat": other_feats[i].astype('float32'), - }, - return_numpy=True, - fetch_list=fetch_vars) - - user_vec = pd.DataFrame(l3[0]) - user_vec.to_csv(args.user_vec_path, mode="a", index=False, header=0) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - if(os.path.exists(args.user_vec_path)): - os.system("rm " + args.user_vec_path) - infer(args) \ No newline at end of file diff --git a/PaddleRec/youbube_dnn/train.py b/PaddleRec/youbube_dnn/train.py deleted file mode 100644 index 446c9552af..0000000000 --- a/PaddleRec/youbube_dnn/train.py +++ /dev/null @@ -1,77 +0,0 @@ -import numpy as np -import pandas as pd -import os -import random -import paddle.fluid as fluid -from youtubednn import YoutubeDNN -import paddle -import args -import logging -import time - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args): - youtube_model = YoutubeDNN() - inputs = youtube_model.input_data(args.watch_vec_size, args.search_vec_size, args.other_feat_size) - loss, acc, l3 = youtube_model.net(inputs, args.output_size, layers=[128, 64, 32]) - - sgd = fluid.optimizer.SGD(learning_rate=args.base_lr) - sgd.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - # Build a random data set. - sample_size = 100 - watch_vecs = [] - search_vecs = [] - other_feats = [] - labels = [] - - for i in range(sample_size): - watch_vec = np.random.rand(args.batch_size, args.watch_vec_size) - search_vec = np.random.rand(args.batch_size, args.search_vec_size) - other_feat = np.random.rand(args.batch_size, args.other_feat_size) - watch_vecs.append(watch_vec) - search_vecs.append(search_vec) - other_feats.append(other_feat) - label = np.random.randint(args.output_size, size=(args.batch_size, 1)) - labels.append(label) - for epoch in range(args.epochs): - for i in range(sample_size): - begin = time.time() - loss_data, acc_val = exe.run(fluid.default_main_program(), - feed={ - "watch_vec": watch_vecs[i].astype('float32'), - "search_vec": search_vecs[i].astype('float32'), - "other_feat": other_feats[i].astype('float32'), - "label": np.array(labels[i]).reshape(args.batch_size, 1) - }, - return_numpy=True, - fetch_list=[loss.name, acc.name]) - end = time.time() - logger.info("epoch_id: {}, batch_time: {:.5f}s, loss: {:.5f}, acc: {:.5f}".format( - epoch, end-begin, float(np.array(loss_data)), np.array(acc_val)[0])) - #save model - model_dir = os.path.join(args.model_dir, 'epoch_' + str(epoch + 1), "checkpoint") - - feed_var_names = ["watch_vec", "search_vec", "other_feat"] - fetch_vars = [l3] - fluid.io.save_inference_model(model_dir, feed_var_names, fetch_vars, exe) - - #save all video vector - video_array = np.array(fluid.global_scope().find_var('l4_weight').get_tensor()) - video_vec = pd.DataFrame(video_array) - video_vec.to_csv(args.video_vec_path, mode="a", index=False, header=0) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - if(os.path.exists(args.video_vec_path)): - os.system("rm " + args.video_vec_path) - train(args)