From a3529ec7d9be019ff033bf4876802fe7c77aa2d7 Mon Sep 17 00:00:00 2001 From: frankwhzhang Date: Wed, 13 May 2020 15:41:45 +0800 Subject: [PATCH 1/3] update api 1.8 --- PaddleRec/ctr/dcn/network.py | 5 ++-- PaddleRec/ctr/din/cluster_train.py | 18 +++++++------- PaddleRec/ctr/din/train.py | 7 +++--- PaddleRec/multiview_simnet/nets.py | 11 +++++---- PaddleRec/ssr/cluster_train.py | 8 +++--- PaddleRec/ssr/nets.py | 12 ++++++--- PaddleRec/tagspace/net.py | 12 ++++++--- PaddleRec/word2vec/net.py | 39 +++++++++++++++++++++--------- 8 files changed, 69 insertions(+), 43 deletions(-) diff --git a/PaddleRec/ctr/dcn/network.py b/PaddleRec/ctr/dcn/network.py index ffa399edc6..8dd6503847 100644 --- a/PaddleRec/ctr/dcn/network.py +++ b/PaddleRec/ctr/dcn/network.py @@ -76,11 +76,10 @@ def build_network(self, is_test=False): def backward(self, lr): p_g_clip = fluid.backward.append_backward(loss=self.loss) - fluid.clip.set_gradient_clip( - fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_by_norm)) + clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=self.clip_by_norm) p_g_clip = fluid.clip.append_gradient_clip_ops(p_g_clip) - optimizer = fluid.optimizer.Adam(learning_rate=lr) + optimizer = fluid.optimizer.Adam(learning_rate=lr, grad_clip=clip) # params_grads = optimizer.backward(self.loss) optimizer.apply_gradients(p_g_clip) diff --git a/PaddleRec/ctr/din/cluster_train.py b/PaddleRec/ctr/din/cluster_train.py index 6b3272366f..8cd7e15c0b 100644 --- a/PaddleRec/ctr/din/cluster_train.py +++ b/PaddleRec/ctr/din/cluster_train.py @@ -86,7 +86,6 @@ def train(): logger.info("reading data completes") avg_cost, pred = network.network(item_count, cat_count, 433) - #fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0)) base_lr = args.base_lr boundaries = [410000] values = [base_lr, 0.2] @@ -101,12 +100,13 @@ def train_loop(main_program): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - feeder = fluid.DataFeeder( - feed_list=[ - "hist_item_seq", "hist_cat_seq", "target_item", "target_cat", - "label", "mask", "target_item_seq", "target_cat_seq" - ], - place=place) + feed_list = [ + "hist_item_seq", "hist_cat_seq", "target_item", "target_cat", + "label", "mask", "target_item_seq", "target_cat_seq" + ] + loader = fluid.io.DataLoader.from_generator( + feed_list=feed_list, capacity=10000, iterable=True) + loader.set_sample_list_generator(data_reader, places=place) if use_parallel: train_exe = fluid.ParallelExecutor( use_cuda=use_cuda, @@ -122,10 +122,10 @@ def train_loop(main_program): loss_sum = 0.0 for id in range(epoch_num): epoch = id + 1 - for data in data_reader(): + for data in loader(): global_step += 1 results = train_exe.run(main_program, - feed=feeder.feed(data), + feed=data, fetch_list=[avg_cost.name, pred.name], return_numpy=True) loss_sum += results[0].mean() diff --git a/PaddleRec/ctr/din/train.py b/PaddleRec/ctr/din/train.py index a519721101..4461eb6729 100644 --- a/PaddleRec/ctr/din/train.py +++ b/PaddleRec/ctr/din/train.py @@ -92,14 +92,15 @@ def train(): logger.info("reading data completes") avg_cost, pred, feed_list = network.network(item_count, cat_count) - fluid.clip.set_gradient_clip(clip=fluid.clip.GradientClipByGlobalNorm( - clip_norm=5.0)) + + clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0) base_lr = args.base_lr boundaries = [410000] values = [base_lr, 0.2] sgd_optimizer = fluid.optimizer.SGD( learning_rate=fluid.layers.piecewise_decay( - boundaries=boundaries, values=values)) + boundaries=boundaries, values=values), + grad_clip=clip) sgd_optimizer.minimize(avg_cost) place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() diff --git a/PaddleRec/multiview_simnet/nets.py b/PaddleRec/multiview_simnet/nets.py index 104101e2b8..6a22820317 100644 --- a/PaddleRec/multiview_simnet/nets.py +++ b/PaddleRec/multiview_simnet/nets.py @@ -190,9 +190,8 @@ def train_net(self): # pairwise hinge_loss loss_part1 = fluid.layers.elementwise_sub( - tensor.fill_constant_batch_size_like( - input=cos_pos, - shape=[-1, 1], + fluid.layers.fill_constant( + shape=[fluid.layers.shape(cos_pos)[0], 1], value=self.margin, dtype='float32'), cos_pos) @@ -200,8 +199,10 @@ def train_net(self): loss_part2 = fluid.layers.elementwise_add(loss_part1, cos_neg) loss_part3 = fluid.layers.elementwise_max( - tensor.fill_constant_batch_size_like( - input=loss_part2, shape=[-1, 1], value=0.0, dtype='float32'), + fluid.layers.fill_constant( + shape=[fluid.layers.shape(loss_part2)[0], 1], + value=0.0, + dtype='float32'), loss_part2) avg_cost = fluid.layers.mean(loss_part3) diff --git a/PaddleRec/ssr/cluster_train.py b/PaddleRec/ssr/cluster_train.py index 7cbe7ab429..0b76934f01 100644 --- a/PaddleRec/ssr/cluster_train.py +++ b/PaddleRec/ssr/cluster_train.py @@ -91,9 +91,11 @@ def get_cards(args): def train_loop(main_program, avg_cost, acc, train_input_data, place, args, train_reader): data_list = [var.name for var in train_input_data] - feeder = fluid.DataFeeder(feed_list=data_list, place=place) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) + loader = fluid.io.DataLoader.from_generator( + feed_list=train_input_data, capacity=10000, iterable=True) + loader.set_sample_list_generator(train_reader, places=place) train_exe = exe total_time = 0.0 @@ -103,10 +105,10 @@ def train_loop(main_program, avg_cost, acc, train_input_data, place, args, print("epoch_%d start" % epoch_idx) t0 = time.time() i = 0 - for batch_id, data in enumerate(train_reader()): + for batch_id, data in enumerate(loader()): i += 1 loss_val, correct_val = train_exe.run( - feed=feeder.feed(data), fetch_list=[avg_cost.name, acc.name]) + feed=data, fetch_list=[avg_cost.name, acc.name]) ce_info.append(float(np.mean(correct_val)) / args.batch_size) if i % args.print_batch == 0: logger.info( diff --git a/PaddleRec/ssr/nets.py b/PaddleRec/ssr/nets.py index 7b78adae3b..3026562d82 100644 --- a/PaddleRec/ssr/nets.py +++ b/PaddleRec/ssr/nets.py @@ -57,13 +57,17 @@ def __init__(self, margin=0.8): def forward(self, pos, neg): loss_part1 = fluid.layers.elementwise_sub( - tensor.fill_constant_batch_size_like( - input=pos, shape=[-1, 1], value=self.margin, dtype='float32'), + fluid.layers.fill_constant( + shape=[fluid.layers.shape(pos)[0], 1], + value=self.margin, + dtype='float32'), pos) loss_part2 = fluid.layers.elementwise_add(loss_part1, neg) loss_part3 = fluid.layers.elementwise_max( - tensor.fill_constant_batch_size_like( - input=loss_part2, shape=[-1, 1], value=0.0, dtype='float32'), + fluid.layers.fill_constant( + shape=[fluid.layers.shape(loss_part2)[0], 1], + value=0.0, + dtype='float32'), loss_part2) return loss_part3 diff --git a/PaddleRec/tagspace/net.py b/PaddleRec/tagspace/net.py index 479d6620aa..3a8287e723 100644 --- a/PaddleRec/tagspace/net.py +++ b/PaddleRec/tagspace/net.py @@ -46,13 +46,17 @@ def network(vocab_text_size, cos_neg = nn.reduce_max(cos_neg_all, dim=1, keep_dim=True) #calculate hinge loss loss_part1 = nn.elementwise_sub( - tensor.fill_constant_batch_size_like( - input=cos_pos, shape=[-1, 1], value=margin, dtype='float32'), + fluid.layers.fill_constant( + shape=[fluid.layers.shape(cos_pos)[0], 1], + value=margin, + dtype='float32'), cos_pos) loss_part2 = nn.elementwise_add(loss_part1, cos_neg) loss_part3 = nn.elementwise_max( - tensor.fill_constant_batch_size_like( - input=loss_part2, shape=[-1, 1], value=0.0, dtype='float32'), + fluid.layers.fill_constant( + shape=[fluid.layers.shape(loss_part2)[0], 1], + value=0.0, + dtype='float32'), loss_part2) avg_cost = nn.mean(loss_part3) less = tensor.cast(cf.less_than(cos_neg, cos_pos), dtype='float32') diff --git a/PaddleRec/word2vec/net.py b/PaddleRec/word2vec/net.py index 3e2ab33b92..c71e1bb54b 100644 --- a/PaddleRec/word2vec/net.py +++ b/PaddleRec/word2vec/net.py @@ -20,7 +20,10 @@ import paddle.fluid as fluid -def skip_gram_word2vec_shuffle_batch(dict_size, embedding_size, is_sparse=False, neg_num=5): +def skip_gram_word2vec_shuffle_batch(dict_size, + embedding_size, + is_sparse=False, + neg_num=5): words = [] input_word = fluid.data(name="input_word", shape=[None, 1], dtype='int64') @@ -61,7 +64,8 @@ def skip_gram_word2vec_shuffle_batch(dict_size, embedding_size, is_sparse=False, # add shuffle_batch after embedding. neg_emb_w_list = [] for i in range(neg_num): - neg_emb_w_list.append(fluid.contrib.layers.shuffle_batch(true_emb_w)) # shuffle true_word + neg_emb_w_list.append( + fluid.contrib.layers.shuffle_batch(true_emb_w)) # shuffle true_word neg_emb_w = fluid.layers.concat(neg_emb_w_list, axis=0) neg_emb_w_re = fluid.layers.reshape( @@ -69,7 +73,8 @@ def skip_gram_word2vec_shuffle_batch(dict_size, embedding_size, is_sparse=False, neg_emb_b_list = [] for i in range(neg_num): - neg_emb_b_list.append(fluid.contrib.layers.shuffle_batch(true_emb_b)) # shuffle true_word + neg_emb_b_list.append( + fluid.contrib.layers.shuffle_batch(true_emb_b)) # shuffle true_word neg_emb_b = fluid.layers.concat(neg_emb_b_list, axis=0) neg_emb_b_vec = fluid.layers.reshape(neg_emb_b, shape=[-1, neg_num]) @@ -81,15 +86,20 @@ def skip_gram_word2vec_shuffle_batch(dict_size, embedding_size, is_sparse=False, true_emb_b) input_emb_re = fluid.layers.reshape( input_emb, shape=[-1, 1, embedding_size]) - neg_matmul = fluid.layers.matmul(input_emb_re, neg_emb_w_re, transpose_y=True) + neg_matmul = fluid.layers.matmul( + input_emb_re, neg_emb_w_re, transpose_y=True) neg_matmul_re = fluid.layers.reshape(neg_matmul, shape=[-1, neg_num]) neg_logits = fluid.layers.elementwise_add(neg_matmul_re, neg_emb_b_vec) #nce loss - label_ones = fluid.layers.fill_constant_batch_size_like( - true_logits, shape=[-1, 1], value=1.0, dtype='float32') - label_zeros = fluid.layers.fill_constant_batch_size_like( - true_logits, shape=[-1, neg_num], value=0.0, dtype='float32') + label_ones = fluid.layers.fill_constant( + shape=[fluid.layers.shape(true_logits)[0], 1], + value=1.0, + dtype='float32') + label_zeros = fluid.layers.fill_constant( + shape=[fluid.layers.shape(true_logits)[0], neg_num], + value=0.0, + dtype='float32') true_xent = fluid.layers.sigmoid_cross_entropy_with_logits(true_logits, label_ones) @@ -103,6 +113,7 @@ def skip_gram_word2vec_shuffle_batch(dict_size, embedding_size, is_sparse=False, avg_cost = fluid.layers.reduce_mean(cost) return avg_cost, data_loader + def skip_gram_word2vec(dict_size, embedding_size, is_sparse=False, neg_num=5): words = [] @@ -171,10 +182,14 @@ def skip_gram_word2vec(dict_size, embedding_size, is_sparse=False, neg_num=5): neg_logits = fluid.layers.elementwise_add(neg_matmul_re, neg_emb_b_vec) #nce loss - label_ones = fluid.layers.fill_constant_batch_size_like( - true_logits, shape=[-1, 1], value=1.0, dtype='float32') - label_zeros = fluid.layers.fill_constant_batch_size_like( - true_logits, shape=[-1, neg_num], value=0.0, dtype='float32') + label_ones = fluid.layers.fill_constant( + shape=[fluid.layers.shape(true_logits)[0], 1], + value=1.0, + dtype='float32') + label_zeros = fluid.layers.fill_constant( + shape=[fluid.layers.shape(true_logits)[0], neg_num], + value=0.0, + dtype='float32') true_xent = fluid.layers.sigmoid_cross_entropy_with_logits(true_logits, label_ones) From 1caf0c88e5caa1707ca57524b8c2415fca2adad6 Mon Sep 17 00:00:00 2001 From: frankwhzhang Date: Tue, 23 Jun 2020 14:00:31 +0800 Subject: [PATCH 2/3] fix paddlerec readme --- PaddleRec/README.md | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/PaddleRec/README.md b/PaddleRec/README.md index 3728079e70..8970eb487c 100644 --- a/PaddleRec/README.md +++ b/PaddleRec/README.md @@ -1,36 +1,4 @@ PaddleRec ========= -个性化推荐 -------- - -推荐系统在当前的互联网服务中正在发挥越来越大的作用，目前大部分电子商务系统、社交网络，广告推荐，搜索引擎，信息流，都不同程度的使用了各种形式的个性化推荐技术，帮助用户快速找到他们想要的信息。 - -在工业可用的推荐系统中，推荐策略一般会被划分为多个模块串联执行。以新闻推荐系统为例，存在多个可以使用深度学习技术的环节，例如新闻的内容理解--标签标注，个性化新闻召回，个性化匹配与排序，融合等。PaddlePaddle对推荐算法的训练提供了完整的支持，并提供了多种模型配置供用户选择。 - -PaddleRec全景图： - -![paddlerec](./img/paddlerec.png) - - - -| 任务场景 | 模型 | 简介 | -| :------------------------:| :------------------------------------------------: | :----------------------------------------------------------: | -| 内容理解 | [TagSpace](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/tagspace) |Tagspace模型学习文本及标签的embedding表示，应用于工业级的标签推荐，具体应用场景有feed新闻标签推荐。| -| 内容理解 | [TextClassification](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/text_classification) |文本分类，具体应用场景有feed新闻标签分类。| -| 召回 | [Word2Vec](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/word2vec) |训练得到词的向量表示、广泛应用于NLP、推荐等任务场景。 | -| 召回 | [GNN](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/gnn) |SR-GNN，全称为Session-based Recommendations with Graph Neural Network（GNN）。使用GNN进行会话序列建模。 | -| 召回 | [TDM-Variant](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/tdm) |全称为Tree-based Deep Model for Recommender Systems。层次化建模及检索 | -| 召回 | [GRU4Rec](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/gru4rec) |Session-based 推荐, 首次将RNN（GRU）运用于session-based推荐，核心思想是在一个session中，用户点击一系列item的行为看做一个序列，用来训练RNN模型 | -| 召回 | [SSR](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ssr) |全称为SequenceSemanticRetrieval, 解决了 GRU4Rec 模型无法预测训练数据集中不存在的项目，比如新闻推荐的问题。它由两个部分组成：一个是匹配模型部分，另一个是检索部分 | -| 匹配 | [Multiview-Simnet](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/multiview_simnet) |多视角Simnet模型是可以融合用户以及推荐项目的多个视角的特征并进行个性化匹配学习的一体化模型。这类模型在很多工业化的场景中都会被使用到，比如百度的Feed产品中 | -| 匹配 | [DSSM](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/dssm) |全称：Deep Structured Semantic Model深度语义匹配模型 | -| 排序 | [DNN](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ctr/dnn) |经典的CTR预估算法。| -| 排序 | [Wide_Deep](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ctr/wide_deep) |经典的CTR预估算法。| -| 排序 | [DeepFM](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ctr/deepfm) |DeepFM，全称Factorization-Machine based Neural Network。经典的CTR预估算法，网络由DNN和FM两部分组成。 | -| 排序 | [XDeepFM](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ctr/xdeepfm) |xDeepFM，全称extreme Factorization Machine。对DeepFM和DCN的改进，提出CIN（Compressed Interaction Network），使用vector-wise等级的显示特征交叉。 | -| 排序 | [DCN](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ctr/dcn) |全称Deep & Cross Network。提出一种新的交叉网络（cross network），在每个层上明确地应用特征交叉。 | -| 排序 | [DIN](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/ctr/din) |DIN，全称为Deep Interest Network。特点为对历史序列建模的过程中结合了预估目标的信息。 | -| 融合- 多任务 | [ESMM](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/multi_task/esmm) |ESMM，全称为Entire Space Multi-task Model。提出一种新的CVR预估模型。 | -| 融合- 多任务 | [Share_bottom](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/multi_task/share_bottom) |多任务学习的基本框架，其特点是对于不同的任务，底层的参数和网络结构是共享的。 | -| 融合- 多任务 | [MMoE](https://github.com/PaddlePaddle/models/tree/develop/PaddleRec/multi_task/mmoe) |MMOE, 全称为Multi-grate Mixture-of-Experts，可以刻画任务相关性。 | +models/PaddleRec只是提供了经典推荐算法的Paddle实现，我们已经开源了功能更强大的工具组件[PaddlePaddle/PaddleRec](https://github.com/PaddlePaddle/PaddleRec) 打通了推荐算法+分布式训练全流程，并提供了高级API，在单机和分布式间可以实现无缝切换。后续我们将在[PaddlePaddle/PaddleRec](https://github.com/PaddlePaddle/PaddleRec) Repo中发布新的模型和功能，models/PaddleRec不再更新维护。 From d0f4b7ef673c07852b145293fee0d3c72cf76f33 Mon Sep 17 00:00:00 2001 From: frankwhzhang Date: Mon, 12 Oct 2020 09:01:17 +0000 Subject: [PATCH 3/3] delete some model --- PaddleRec/ctr/dcn/infer.py | 113 --------- PaddleRec/ctr/dcn/local_train.py | 95 ------- PaddleRec/ctr/deepfm/criteo_reader.py | 76 ------ PaddleRec/ctr/deepfm/infer.py | 91 ------- PaddleRec/ctr/deepfm/local_train.py | 72 ------ PaddleRec/ctr/din/infer.py | 109 --------- PaddleRec/ctr/din/train.py | 189 -------------- PaddleRec/ctr/wide_deep/infer.py | 75 ------ PaddleRec/ctr/wide_deep/train.py | 52 ---- PaddleRec/ctr/xdeepfm/criteo_reader.py | 43 ---- PaddleRec/ctr/xdeepfm/infer.py | 96 -------- PaddleRec/ctr/xdeepfm/local_train.py | 68 ------ PaddleRec/dssm/infer.py | 43 ---- PaddleRec/gnn/infer.py | 93 ------- PaddleRec/gnn/train.py | 176 ------------- PaddleRec/gru4rec/infer.py | 96 -------- PaddleRec/gru4rec/infer_sample_neg.py | 106 -------- PaddleRec/gru4rec/train.py | 174 ------------- PaddleRec/gru4rec/train_sample_neg.py | 134 ---------- PaddleRec/multi_task/esmm/infer.py | 69 ------ PaddleRec/multi_task/esmm/train.py | 45 ---- PaddleRec/multiview_simnet/infer.py | 138 ----------- PaddleRec/multiview_simnet/train.py | 194 --------------- PaddleRec/ncf/infer.py | 34 --- PaddleRec/ncf/train.py | 70 ------ PaddleRec/rerank/listwise/infer.py | 84 ------- PaddleRec/rerank/listwise/train.py | 72 ------ PaddleRec/tagspace/infer.py | 102 -------- PaddleRec/tagspace/train.py | 173 ------------- PaddleRec/tdm/tdm_demo/dataset_generator.py | 76 ------ PaddleRec/tdm/tdm_demo/distributed_train.py | 250 ------------------- PaddleRec/tdm/tdm_demo/local_infer.py | 119 --------- PaddleRec/tdm/tdm_demo/local_train.py | 135 ---------- PaddleRec/tdm/tdm_demo/predict.py | 105 -------- PaddleRec/word2vec/infer.py | 208 ---------------- PaddleRec/word2vec/train.py | 258 -------------------- PaddleRec/youbube_dnn/infer.py | 55 ----- PaddleRec/youbube_dnn/train.py | 77 ------ 38 files changed, 4165 deletions(-) delete mode 100644 PaddleRec/ctr/dcn/infer.py delete mode 100644 PaddleRec/ctr/dcn/local_train.py delete mode 100644 PaddleRec/ctr/deepfm/criteo_reader.py delete mode 100644 PaddleRec/ctr/deepfm/infer.py delete mode 100644 PaddleRec/ctr/deepfm/local_train.py delete mode 100644 PaddleRec/ctr/din/infer.py delete mode 100644 PaddleRec/ctr/din/train.py delete mode 100644 PaddleRec/ctr/wide_deep/infer.py delete mode 100644 PaddleRec/ctr/wide_deep/train.py delete mode 100644 PaddleRec/ctr/xdeepfm/criteo_reader.py delete mode 100644 PaddleRec/ctr/xdeepfm/infer.py delete mode 100644 PaddleRec/ctr/xdeepfm/local_train.py delete mode 100644 PaddleRec/dssm/infer.py delete mode 100644 PaddleRec/gnn/infer.py delete mode 100644 PaddleRec/gnn/train.py delete mode 100644 PaddleRec/gru4rec/infer.py delete mode 100644 PaddleRec/gru4rec/infer_sample_neg.py delete mode 100644 PaddleRec/gru4rec/train.py delete mode 100644 PaddleRec/gru4rec/train_sample_neg.py delete mode 100644 PaddleRec/multi_task/esmm/infer.py delete mode 100644 PaddleRec/multi_task/esmm/train.py delete mode 100644 PaddleRec/multiview_simnet/infer.py delete mode 100644 PaddleRec/multiview_simnet/train.py delete mode 100644 PaddleRec/ncf/infer.py delete mode 100644 PaddleRec/ncf/train.py delete mode 100644 PaddleRec/rerank/listwise/infer.py delete mode 100644 PaddleRec/rerank/listwise/train.py delete mode 100644 PaddleRec/tagspace/infer.py delete mode 100644 PaddleRec/tagspace/train.py delete mode 100644 PaddleRec/tdm/tdm_demo/dataset_generator.py delete mode 100644 PaddleRec/tdm/tdm_demo/distributed_train.py delete mode 100644 PaddleRec/tdm/tdm_demo/local_infer.py delete mode 100644 PaddleRec/tdm/tdm_demo/local_train.py delete mode 100644 PaddleRec/tdm/tdm_demo/predict.py delete mode 100644 PaddleRec/word2vec/infer.py delete mode 100644 PaddleRec/word2vec/train.py delete mode 100644 PaddleRec/youbube_dnn/infer.py delete mode 100644 PaddleRec/youbube_dnn/train.py diff --git a/PaddleRec/ctr/dcn/infer.py b/PaddleRec/ctr/dcn/infer.py deleted file mode 100644 index f79e36d965..0000000000 --- a/PaddleRec/ctr/dcn/infer.py +++ /dev/null @@ -1,113 +0,0 @@ -import logging -import random - -import numpy as np -import pickle - -# disable gpu training for this example -import os -os.environ['CUDA_VISIBLE_DEVICES'] = '' -import paddle -import paddle.fluid as fluid - -from config import parse_args -from reader import CriteoDataset -from network import DCN -from collections import OrderedDict -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger('fluid') -logger.setLevel(logging.INFO) - - -def infer(): - args = parse_args() - print(args) - - place = fluid.CPUPlace() - inference_scope = fluid.Scope() - - test_valid_files = [ - os.path.join(args.test_valid_data_dir, fname) - for fname in next(os.walk(args.test_valid_data_dir))[2] - ] - test_files = random.sample(test_valid_files, - int(len(test_valid_files) * 0.5)) - if not test_files: - test_files = test_valid_files - print('test files num {}'.format(len(test_files))) - - criteo_dataset = CriteoDataset() - criteo_dataset.setup(args.vocab_dir) - test_reader = criteo_dataset.test_reader(test_files, args.batch_size, 100) - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_output_dir, - 'epoch_' + args.test_epoch, "checkpoint") - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - cat_feat_dims_dict = OrderedDict() - for line in open(args.cat_feat_num): - spls = line.strip().split() - assert len(spls) == 2 - cat_feat_dims_dict[spls[0]] = int(spls[1]) - dcn_model = DCN(args.cross_num, args.dnn_hidden_units, - args.l2_reg_cross, args.use_bn, args.clip_by_norm, - cat_feat_dims_dict, args.is_sparse) - dcn_model.build_network(is_test=True) - - exe = fluid.Executor(place) - feeder = fluid.DataFeeder( - feed_list=dcn_model.data_list, place=place) - - exe.run(startup_program) - fluid.load(fluid.default_main_program(), cur_model_path) - - for var in dcn_model.auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - loss_all = 0 - num_ins = 0 - for batch_id, data_test in enumerate(test_reader()): - loss_val, auc_val = exe.run(test_program, - feed=feeder.feed(data_test), - fetch_list=[ - dcn_model.avg_logloss.name, - dcn_model.auc_var.name - ]) - # num_ins += len(data_test) - num_ins += 1 - loss_all += loss_val - logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( - batch_id, loss_all / num_ins, auc_val)) - - print( - 'The last log info is the total Logloss and AUC for all test data. ' - ) - - -def set_zero(var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - infer() diff --git a/PaddleRec/ctr/dcn/local_train.py b/PaddleRec/ctr/dcn/local_train.py deleted file mode 100644 index 411fb916bd..0000000000 --- a/PaddleRec/ctr/dcn/local_train.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import print_function, absolute_import, division -import os -import random -import sys -import time -from collections import OrderedDict - -import paddle.fluid as fluid - -from config import parse_args -from network import DCN -import utils -""" -train DCN model -""" - - -def train(args): - """train and save DCN model - - :param args: hyperparams of model - :return: - """ - # ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - cat_feat_dims_dict = OrderedDict() - for line in open(args.cat_feat_num): - spls = line.strip().split() - assert len(spls) == 2 - cat_feat_dims_dict[spls[0]] = int(spls[1]) - dcn_model = DCN(args.cross_num, args.dnn_hidden_units, args.l2_reg_cross, - args.use_bn, args.clip_by_norm, cat_feat_dims_dict, - args.is_sparse) - dcn_model.build_network() - dcn_model.backward(args.lr) - - # config dataset - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(dcn_model.data_list) - pipe_command = 'python reader.py {}'.format(args.vocab_dir) - dataset.set_pipe_command(pipe_command) - dataset.set_batch_size(args.batch_size) - dataset.set_thread(args.num_thread) - train_filelist = [ - os.path.join(args.train_data_dir, fname) - for fname in next(os.walk(args.train_data_dir))[2] - ] - dataset.set_filelist(train_filelist) - num_epoch = args.num_epoch - if args.steps: - epoch = args.steps * args.batch_size / 41000000 - full_epoch = int(epoch // 1) - last_epoch = epoch % 1 - train_filelists = [train_filelist for _ in range(full_epoch)] + [ - random.sample(train_filelist, int( - len(train_filelist) * last_epoch)) - ] - num_epoch = full_epoch + 1 - print("train epoch: {}".format(num_epoch)) - - # Executor - exe = fluid.Executor(fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - - for epoch_id in range(num_epoch): - start = time.time() - sys.stderr.write('\nepoch%d start ...\n' % (epoch_id + 1)) - dataset.set_filelist(train_filelists[epoch_id]) - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[ - dcn_model.loss, dcn_model.avg_logloss, dcn_model.auc_var - ], - fetch_info=['total_loss', 'avg_logloss', 'auc'], - debug=False, - print_period=args.print_steps) - model_dir = os.path.join(args.model_output_dir, - 'epoch_' + str(epoch_id + 1), "checkpoint") - sys.stderr.write('epoch%d is finished and takes %f s\n' % ( - (epoch_id + 1), time.time() - start)) - fluid.save(fluid.default_main_program(), model_dir) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - args = parse_args() - print(args) - utils.check_version() - train(args) diff --git a/PaddleRec/ctr/deepfm/criteo_reader.py b/PaddleRec/ctr/deepfm/criteo_reader.py deleted file mode 100644 index 037fe909bd..0000000000 --- a/PaddleRec/ctr/deepfm/criteo_reader.py +++ /dev/null @@ -1,76 +0,0 @@ -import sys -import paddle.fluid.incubate.data_generator as dg -try: - import cPickle as pickle -except ImportError: - import pickle -from collections import Counter -import os - - -class CriteoDataset(dg.MultiSlotDataGenerator): - def setup(self, feat_dict_name): - self.cont_min_ = [0, -3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - self.cont_max_ = [ - 5775, 257675, 65535, 969, 23159456, 431037, 56311, 6047, 29019, 46, - 231, 4008, 7393 - ] - self.cont_diff_ = [ - self.cont_max_[i] - self.cont_min_[i] - for i in range(len(self.cont_min_)) - ] - self.continuous_range_ = range(1, 14) - self.categorical_range_ = range(14, 40) - self.feat_dict_ = pickle.load(open(feat_dict_name, 'rb')) - - def _process_line(self, line): - features = line.rstrip('\n').split('\t') - feat_idx = [] - feat_value = [] - for idx in self.continuous_range_: - if features[idx] == '': - feat_idx.append(0) - feat_value.append(0.0) - else: - feat_idx.append(self.feat_dict_[idx]) - feat_value.append( - (float(features[idx]) - self.cont_min_[idx - 1]) / - self.cont_diff_[idx - 1]) - for idx in self.categorical_range_: - if features[idx] == '' or features[idx] not in self.feat_dict_: - feat_idx.append(0) - feat_value.append(0.0) - else: - feat_idx.append(self.feat_dict_[features[idx]]) - feat_value.append(1.0) - label = [int(features[0])] - return feat_idx, feat_value, label - - def test(self, filelist): - def local_iter(): - for fname in filelist: - with open(fname.strip(), 'r') as fin: - for line in fin: - feat_idx, feat_value, label = self._process_line(line) - yield [feat_idx, feat_value, label] - - return local_iter - - def generate_sample(self, line): - def data_iter(): - feat_idx, feat_value, label = self._process_line(line) - yield [('feat_idx', feat_idx), ('feat_value', feat_value), ('label', - label)] - - return data_iter - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - criteo_dataset = CriteoDataset() - if len(sys.argv) <= 1: - sys.stderr.write("feat_dict needed for criteo reader.") - exit(1) - criteo_dataset.setup(sys.argv[1]) - criteo_dataset.run_from_stdin() diff --git a/PaddleRec/ctr/deepfm/infer.py b/PaddleRec/ctr/deepfm/infer.py deleted file mode 100644 index 35f18a6efd..0000000000 --- a/PaddleRec/ctr/deepfm/infer.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging -import numpy as np -import pickle - -# disable gpu training for this example -import os -os.environ['CUDA_VISIBLE_DEVICES'] = '' -import paddle -import paddle.fluid as fluid - -from args import parse_args -from criteo_reader import CriteoDataset -from network_conf import ctr_deepfm_model -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger('fluid') -logger.setLevel(logging.INFO) - - -def infer(): - args = parse_args() - - place = fluid.CPUPlace() - inference_scope = fluid.Scope() - - test_files = [ - os.path.join(args.test_data_dir, x) - for x in os.listdir(args.test_data_dir) - ] - criteo_dataset = CriteoDataset() - criteo_dataset.setup(args.feat_dict) - test_reader = fluid.io.batch( - criteo_dataset.test(test_files), batch_size=args.batch_size) - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_output_dir, - 'epoch_' + args.test_epoch) - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - loss, auc, data_list, auc_states = ctr_deepfm_model( - args.embedding_size, args.num_field, args.num_feat, - args.layer_sizes, args.act, args.reg) - - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=data_list, place=place) - main_program = fluid.default_main_program() - fluid.load(main_program, cur_model_path, exe) - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - loss_all = 0 - num_ins = 0 - for batch_id, data_test in enumerate(test_reader()): - loss_val, auc_val = exe.run(test_program, - feed=feeder.feed(data_test), - fetch_list=[loss.name, auc.name]) - num_ins += len(data_test) - loss_all += loss_val - logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( - batch_id, loss_all / num_ins, auc_val)) - - print( - 'The last log info is the total Logloss and AUC for all test data. ' - ) - - -def set_zero(var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - infer() diff --git a/PaddleRec/ctr/deepfm/local_train.py b/PaddleRec/ctr/deepfm/local_train.py deleted file mode 100644 index 5b4d724af1..0000000000 --- a/PaddleRec/ctr/deepfm/local_train.py +++ /dev/null @@ -1,72 +0,0 @@ -from args import parse_args -import os -import paddle.fluid as fluid -import sys -from network_conf import ctr_deepfm_model -import time -import numpy -import pickle -import utils - - -def train(): - args = parse_args() - # add ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - print('---------- Configuration Arguments ----------') - for key, value in args.__dict__.items(): - print(key + ':' + str(value)) - - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - loss, auc, data_list, auc_states = ctr_deepfm_model( - args.embedding_size, args.num_field, args.num_feat, args.layer_sizes, - args.act, args.reg) - optimizer = fluid.optimizer.SGD( - learning_rate=args.lr, - regularization=fluid.regularizer.L2DecayRegularizer(args.reg)) - optimizer.minimize(loss) - - exe = fluid.Executor(fluid.CPUPlace()) - exe.run(fluid.default_startup_program()) - - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(data_list) - pipe_command = 'python criteo_reader.py {}'.format(args.feat_dict) - dataset.set_pipe_command(pipe_command) - dataset.set_batch_size(args.batch_size) - dataset.set_thread(args.num_thread) - train_filelist = [ - os.path.join(args.train_data_dir, x) - for x in os.listdir(args.train_data_dir) - ] - - print('---------------------------------------------') - for epoch_id in range(args.num_epoch): - start = time.time() - dataset.set_filelist(train_filelist) - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[loss, auc], - fetch_info=['epoch %d batch loss' % (epoch_id + 1), "auc"], - print_period=1000, - debug=False) - model_dir = os.path.join(args.model_output_dir, - 'epoch_' + str(epoch_id + 1)) - sys.stderr.write('epoch%d is finished and takes %f s\n' % ( - (epoch_id + 1), time.time() - start)) - main_program = fluid.default_main_program() - fluid.io.save(main_program, model_dir) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/ctr/din/infer.py b/PaddleRec/ctr/din/infer.py deleted file mode 100644 index d6d8d39eb3..0000000000 --- a/PaddleRec/ctr/din/infer.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import argparse -import logging -import numpy as np -import os -import paddle -import paddle.fluid as fluid -import reader - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle DIN example") - parser.add_argument( - '--model_path', - type=str, - required=True, - help="path of model parameters") - parser.add_argument( - '--test_path', - type=str, - default='data/paddle_test.txt.bak', - help='dir of test file') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether to use gpu') - - return parser.parse_args() - - -def calc_auc(raw_arr): - # sort by pred value, from small to big - arr = sorted(raw_arr, key=lambda d: d[2]) - auc = 0.0 - fp1, tp1, fp2, tp2 = 0.0, 0.0, 0.0, 0.0 - for record in arr: - fp2 += record[0] # noclick - tp2 += record[1] # click - auc += (fp2 - fp1) * (tp2 + tp1) - fp1, tp1 = fp2, tp2 - # if all nonclick or click, disgard - threshold = len(arr) - 1e-3 - if tp2 > threshold or fp2 > threshold: - return -0.5 - if tp2 * fp2 > 0.0: # normal auc - return (1.0 - auc / (2.0 * tp2 * fp2)) - else: - return None - - -def infer(): - args = parse_args() - model_path = args.model_path - use_cuda = True if args.use_cuda else False - data_reader, _ = reader.prepare_reader(args.test_path, 32 * 16) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - inference_scope = fluid.Scope() - - exe = fluid.Executor(place) - - #with fluid.scope_guard(inference_scope): - [inference_program, feed_target_names, - fetch_targets] = fluid.io.load_inference_model(model_path, exe) - - loader = fluid.io.DataLoader.from_generator( - feed_list=[ - inference_program.block(0).var(e) for e in feed_target_names - ], - capacity=10000, - iterable=True) - loader.set_sample_list_generator(data_reader, places=place) - - loss_sum = 0.0 - score = [] - count = 0 - for data in loader(): - res = exe.run(inference_program, feed=data, fetch_list=fetch_targets) - loss_sum += res[0] - label_data = list(np.array(data[0]["label"])) - for i in range(len(label_data)): - if label_data[i] > 0.5: - score.append([0, 1, res[1][i]]) - else: - score.append([1, 0, res[1][i]]) - count += 1 - auc = calc_auc(score) - logger.info("TEST --> loss: {}, auc: {}".format(loss_sum / count, auc)) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - infer() diff --git a/PaddleRec/ctr/din/train.py b/PaddleRec/ctr/din/train.py deleted file mode 100644 index f309459ab8..0000000000 --- a/PaddleRec/ctr/din/train.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import os -import sys -import logging -import time -import numpy as np -import argparse -import paddle.fluid as fluid -import paddle -import time -import network -import reader -import random - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser("din") - parser.add_argument( - '--config_path', - type=str, - default='data/config.txt', - help='dir of config') - parser.add_argument( - '--train_dir', - type=str, - default='data/paddle_train.txt', - help='dir of train file') - parser.add_argument( - '--model_dir', - type=str, - default='din_amazon', - help='dir of saved model') - parser.add_argument( - '--batch_size', type=int, default=16, help='number of batch size') - parser.add_argument( - '--epoch_num', type=int, default=200, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether to use gpu') - parser.add_argument( - '--parallel', - type=int, - default=0, - help='whether to use parallel executor') - parser.add_argument( - '--base_lr', type=float, default=0.85, help='based learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - parser.add_argument('--batch_num', type=int, help="batch num for ce") - args = parser.parse_args() - return args - - -def train(): - args = parse_args() - - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - config_path = args.config_path - train_path = args.train_dir - epoch_num = args.epoch_num - use_cuda = True if args.use_cuda else False - use_parallel = True if args.parallel else False - - logger.info("reading data begins") - user_count, item_count, cat_count = reader.config_read(config_path) - data_reader, max_len = reader.prepare_reader(train_path, args.batch_size * - args.num_devices) - logger.info("reading data completes") - - avg_cost, pred, feed_list = network.network(item_count, cat_count) - - clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0) - base_lr = args.base_lr - boundaries = [410000] - values = [base_lr, 0.2] - sgd_optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.piecewise_decay( - boundaries=boundaries, values=values), - grad_clip=clip) - sgd_optimizer.minimize(avg_cost) - - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - loader = fluid.io.DataLoader.from_generator( - feed_list=feed_list, capacity=10000, iterable=True) - loader.set_sample_list_generator(data_reader, places=place) - if use_parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - logger.info("train begins") - - global_step = 0 - PRINT_STEP = 1000 - - total_time = [] - ce_info = [] - start_time = time.time() - loss_sum = 0.0 - for id in range(epoch_num): - epoch = id + 1 - for data in loader(): - global_step += 1 - results = train_exe.run(feed=data, - fetch_list=[avg_cost.name, pred.name], - return_numpy=True) - loss_sum += results[0].mean() - - if global_step % PRINT_STEP == 0: - ce_info.append(loss_sum / PRINT_STEP) - total_time.append(time.time() - start_time) - logger.info( - "epoch: %d\tglobal_step: %d\ttrain_loss: %.4f\t\ttime: %.2f" - % (epoch, global_step, loss_sum / PRINT_STEP, - time.time() - start_time)) - start_time = time.time() - loss_sum = 0.0 - - if (global_step > 400000 and global_step % PRINT_STEP == 0) or ( - global_step <= 400000 and global_step % 50000 == 0): - save_dir = os.path.join(args.model_dir, - "global_step_" + str(global_step)) - feed_var_name = [ - "hist_item_seq", "hist_cat_seq", "target_item", - "target_cat", "label", "mask", "target_item_seq", - "target_cat_seq" - ] - fetch_vars = [avg_cost, pred] - fluid.io.save_inference_model(save_dir, feed_var_name, - fetch_vars, exe) - logger.info("model saved in " + save_dir) - if args.enable_ce and global_step >= args.batch_num: - break - # only for ce - if args.enable_ce: - gpu_num = get_cards(args) - ce_loss = 0 - ce_time = 0 - try: - ce_loss = ce_info[-1] - ce_time = total_time[-1] - except: - print("ce info error") - print("kpis\teach_pass_duration_card%s\t%s" % (gpu_num, ce_time)) - print("kpis\ttrain_loss_card%s\t%s" % (gpu_num, ce_loss)) - - -def get_cards(args): - if args.enable_ce: - cards = os.environ.get('CUDA_VISIBLE_DEVICES') - num = len(cards.split(",")) - return num - else: - return args.num_devices - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - train() diff --git a/PaddleRec/ctr/wide_deep/infer.py b/PaddleRec/ctr/wide_deep/infer.py deleted file mode 100644 index 3828dce284..0000000000 --- a/PaddleRec/ctr/wide_deep/infer.py +++ /dev/null @@ -1,75 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from net import wide_deep -import logging -import paddle -import args -import utils -import time -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def set_zero(var_name,scope=fluid.global_scope(), place=fluid.CPUPlace(),param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -def run_infer(args,test_data_path): - wide_deep_model = wide_deep() - test_data_generator = utils.Dataset() - test_reader = fluid.io.batch(test_data_generator.test(test_data_path), batch_size=args.batch_size) - inference_scope = fluid.Scope() - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - - cur_model_path = os.path.join(args.model_dir, 'epoch_' + str(args.test_epoch), "checkpoint") - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - inputs = wide_deep_model.input_data() - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - loss, acc, auc, batch_auc, auc_states = wide_deep_model.model(inputs, args.hidden1_units, args.hidden2_units, args.hidden3_units) - exe = fluid.Executor(place) - - fluid.load(fluid.default_main_program(), cur_model_path,exe) - loader = fluid.io.DataLoader.from_generator(feed_list=inputs, capacity=args.batch_size, iterable=True) - loader.set_sample_list_generator(test_reader, places=place) - - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - mean_acc = [] - mean_auc = [] - for batch_id, data in enumerate(loader()): - begin = time.time() - acc_val,auc_val = exe.run(program=test_program, - feed=data, - fetch_list=[acc.name, auc.name], - return_numpy=True - ) - mean_acc.append(np.array(acc_val)[0]) - mean_auc.append(np.array(auc_val)[0]) - end = time.time() - logger.info("batch_id: {}, batch_time: {:.5f}s, acc: {:.5f}, auc: {:.5f}".format( - batch_id, end-begin, np.array(acc_val)[0], np.array(auc_val)[0])) - - logger.info("mean_acc:{:.5f}, mean_auc:{:.5f}".format(np.mean(mean_acc), np.mean(mean_auc))) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - run_infer(args, args.test_data_path) - \ No newline at end of file diff --git a/PaddleRec/ctr/wide_deep/train.py b/PaddleRec/ctr/wide_deep/train.py deleted file mode 100644 index 781feb8fd9..0000000000 --- a/PaddleRec/ctr/wide_deep/train.py +++ /dev/null @@ -1,52 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from net import wide_deep -import logging -import paddle -import args -import utils -import time - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args, train_data_path): - wide_deep_model = wide_deep() - inputs = wide_deep_model.input_data() - train_data_generator = utils.Dataset() - train_reader = fluid.io.batch(train_data_generator.train(train_data_path), batch_size=args.batch_size) - - loss, acc, auc, batch_auc, auc_states = wide_deep_model.model(inputs, args.hidden1_units, args.hidden2_units, args.hidden3_units) - optimizer = fluid.optimizer.AdagradOptimizer(learning_rate=0.01) - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - loader = fluid.io.DataLoader.from_generator( - feed_list=inputs, capacity=args.batch_size, iterable=True) - loader.set_sample_list_generator(train_reader, places=place) - - for epoch in range(args.epochs): - for batch_id, data in enumerate(loader()): - begin = time.time() - loss_val, acc_val, auc_val = exe.run(program=fluid.default_main_program(), - feed=data, - fetch_list=[loss.name, acc.name, auc.name], - return_numpy=True) - end = time.time() - logger.info("epoch:{}, batch_time:{:.5f}s, loss:{:.5f}, acc:{:.5f}, auc:{:.5f}".format(epoch, end-begin, np.array(loss_val)[0], - np.array(acc_val)[0], np.array(auc_val)[0])) - - model_dir = os.path.join(args.model_dir, 'epoch_' + str(epoch + 1), "checkpoint") - main_program = fluid.default_main_program() - fluid.io.save(main_program,model_dir) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - train(args, args.train_data_path) diff --git a/PaddleRec/ctr/xdeepfm/criteo_reader.py b/PaddleRec/ctr/xdeepfm/criteo_reader.py deleted file mode 100644 index a6d32f4a47..0000000000 --- a/PaddleRec/ctr/xdeepfm/criteo_reader.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys -import paddle.fluid.incubate.data_generator as dg -import pickle -from collections import Counter -import os - - -class CriteoDataset(dg.MultiSlotDataGenerator): - def _process_line(self, line): - features = line.strip('\n').split('\t') - feat_idx = [] - feat_value = [] - for idx in range(1, 40): - feat_idx.append(int(features[idx])) - feat_value.append(1.0) - label = [int(features[0])] - return feat_idx, feat_value, label - - def test(self, filelist): - def local_iter(): - for fname in filelist: - with open(fname.strip(), 'r') as fin: - for line in fin: - feat_idx, feat_value, label = self._process_line(line) - yield [feat_idx, feat_value, label] - - return local_iter - - def generate_sample(self, line): - def data_iter(): - feat_idx, feat_value, label = self._process_line(line) - feature_name = ['feat_idx', 'feat_value', 'label'] - yield [('feat_idx', feat_idx), ('feat_value', feat_value), ('label', - label)] - - return data_iter - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - criteo_dataset = CriteoDataset() - criteo_dataset.run_from_stdin() diff --git a/PaddleRec/ctr/xdeepfm/infer.py b/PaddleRec/ctr/xdeepfm/infer.py deleted file mode 100644 index 98210fb2d4..0000000000 --- a/PaddleRec/ctr/xdeepfm/infer.py +++ /dev/null @@ -1,96 +0,0 @@ -import logging -import numpy as np -import pickle -import os -import paddle -import paddle.fluid as fluid - -from args import parse_args -from criteo_reader import CriteoDataset -import network_conf -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger('fluid') -logger.setLevel(logging.INFO) - - -def infer(): - args = parse_args() - print(args) - - if args.use_gpu == 1: - place = fluid.CUDAPlace(0) - else: - place = fluid.CPUPlace() - inference_scope = fluid.Scope() - - test_files = [ - os.path.join(args.test_data_dir, x) - for x in os.listdir(args.test_data_dir) - ] - criteo_dataset = CriteoDataset() - test_reader = fluid.io.batch( - criteo_dataset.test(test_files), batch_size=args.batch_size) - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_output_dir, - 'epoch_' + args.test_epoch, "checkpoint") - - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - loss, auc, data_list, auc_states = eval( - 'network_conf.' + args.model_name)( - args.embedding_size, args.num_field, args.num_feat, - args.layer_sizes_dnn, args.act, args.reg, - args.layer_sizes_cin) - - exe = fluid.Executor(place) - feeder = fluid.DataFeeder(feed_list=data_list, place=place) - - exe.run(startup_program) - fluid.load(fluid.default_main_program(), cur_model_path) - - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - loss_all = 0 - num_ins = 0 - for batch_id, data_test in enumerate(test_reader()): - loss_val, auc_val = exe.run(test_program, - feed=feeder.feed(data_test), - fetch_list=[loss.name, auc.name]) - - num_ins += len(data_test) - loss_all += loss_val * len(data_test) - logger.info('TEST --> batch: {} loss: {} auc_val: {}'.format( - batch_id, loss_all / num_ins, auc_val)) - - print( - 'The last log info is the total Logloss and AUC for all test data. ' - ) - - -def set_zero(var_name, - scope=fluid.global_scope(), - place=fluid.CPUPlace(), - param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - infer() diff --git a/PaddleRec/ctr/xdeepfm/local_train.py b/PaddleRec/ctr/xdeepfm/local_train.py deleted file mode 100644 index 5743503340..0000000000 --- a/PaddleRec/ctr/xdeepfm/local_train.py +++ /dev/null @@ -1,68 +0,0 @@ -from args import parse_args -import os -import paddle.fluid as fluid -import sys -import network_conf -import time -import utils - - -def train(): - args = parse_args() - # add ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - print(args) - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - loss, auc, data_list, auc_states = eval('network_conf.' + args.model_name)( - args.embedding_size, args.num_field, args.num_feat, - args.layer_sizes_dnn, args.act, args.reg, args.layer_sizes_cin) - optimizer = fluid.optimizer.SGD( - learning_rate=args.lr, - regularization=fluid.regularizer.L2DecayRegularizer(args.reg)) - optimizer.minimize(loss) - - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(data_list) - dataset.set_pipe_command('python criteo_reader.py') - dataset.set_batch_size(args.batch_size) - dataset.set_filelist([ - os.path.join(args.train_data_dir, x) - for x in os.listdir(args.train_data_dir) - ]) - - if args.use_gpu == 1: - exe = fluid.Executor(fluid.CUDAPlace(0)) - dataset.set_thread(1) - else: - exe = fluid.Executor(fluid.CPUPlace()) - dataset.set_thread(args.num_thread) - exe.run(fluid.default_startup_program()) - - for epoch_id in range(args.num_epoch): - start = time.time() - sys.stderr.write('\nepoch%d start ...\n' % (epoch_id + 1)) - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[loss, auc], - fetch_info=['loss', 'auc'], - debug=False, - print_period=args.print_steps) - model_dir = os.path.join(args.model_output_dir, - 'epoch_' + str(epoch_id + 1), "checkpoint") - sys.stderr.write('epoch%d is finished and takes %f s\n' % ( - (epoch_id + 1), time.time() - start)) - fluid.save(fluid.default_main_program(), model_dir) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/dssm/infer.py b/PaddleRec/dssm/infer.py deleted file mode 100644 index 06eb0ba5d5..0000000000 --- a/PaddleRec/dssm/infer.py +++ /dev/null @@ -1,43 +0,0 @@ -import paddle.fluid as fluid -import numpy as np -import sys -import args -import logging - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def infer(args): - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model(args.model_dir, exe) - - #构造测试数据 - sample_size = 100 - l_Qs = [] - pos_l_Ds = [] - - for i in range(sample_size): - l_Q = np.random.rand(1, args.TRIGRAM_D) - l_Qs.append(l_Q) - - l_D = np.random.rand(1, args.TRIGRAM_D) - pos_l_Ds.append(l_D) - - res = [] - for i in range(sample_size): - con_sim = exe.run(infer_program, - feed={"query": l_Qs[i].astype('float32').reshape(1,args.TRIGRAM_D), - "doc_pos": pos_l_Ds[i].astype('float32').reshape(1,args.TRIGRAM_D)}, - fetch_list=fetch_vars, - return_numpy=True) - - logger.info("query_doc_sim: {:.5f}".format(np.array(con_sim).reshape(-1,1)[0][0])) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - infer(args) \ No newline at end of file diff --git a/PaddleRec/gnn/infer.py b/PaddleRec/gnn/infer.py deleted file mode 100644 index 2bd58b0fcb..0000000000 --- a/PaddleRec/gnn/infer.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import argparse -import logging -import numpy as np -import os -import paddle -import paddle.fluid as fluid -import reader -import network - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser(description="PaddlePaddle DIN example") - parser.add_argument( - '--model_path', type=str, default='./saved_model/', help="path of model parameters") - parser.add_argument( - '--test_path', type=str, default='./data/diginetica/test.txt', help='dir of test file') - parser.add_argument( - '--config_path', type=str, default='./data/diginetica/config.txt', help='dir of config') - parser.add_argument( - '--use_cuda', type=int, default=1, help='whether to use gpu') - parser.add_argument( - '--batch_size', type=int, default=100, help='input batch size') - parser.add_argument( - '--start_index', type=int, default='0', help='start index') - parser.add_argument( - '--last_index', type=int, default='10', help='end index') - parser.add_argument( - '--hidden_size', type=int, default=100, help='hidden state size') - parser.add_argument( - '--step', type=int, default=1, help='gnn propogation steps') - return parser.parse_args() - - -def infer(args): - batch_size = args.batch_size - items_num = reader.read_config(args.config_path) - test_data = reader.Data(args.test_path, False) - place = fluid.CUDAPlace(0) if args.use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - loss, acc, py_reader, feed_datas = network.network(items_num, args.hidden_size, args.step, batch_size) - exe.run(fluid.default_startup_program()) - infer_program = fluid.default_main_program().clone(for_test=True) - - for epoch_num in range(args.start_index, args.last_index + 1): - model_path = os.path.join(args.model_path, "epoch_" + str(epoch_num)) - try: - if not os.path.exists(model_path + ".pdmodel"): - raise ValueError() - fluid.io.load(infer_program, model_path+".pdmodel", exe) - - loss_sum = 0.0 - acc_sum = 0.0 - count = 0 - py_reader.set_sample_list_generator(test_data.reader(batch_size, batch_size*20, False)) - py_reader.start() - try: - while True: - res = exe.run(infer_program, - fetch_list=[loss.name, acc.name], use_program_cache=True) - loss_sum += res[0] - acc_sum += res[1] - count += 1 - except fluid.core.EOFException: - py_reader.reset() - logger.info("TEST --> loss: %.4lf, Recall@20: %.4lf" % - (loss_sum / count, acc_sum / count)) - except ValueError as e: - logger.info("TEST --> error: there is no model in " + model_path) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = parse_args() - infer(args) diff --git a/PaddleRec/gnn/train.py b/PaddleRec/gnn/train.py deleted file mode 100644 index ae58ca3ea0..0000000000 --- a/PaddleRec/gnn/train.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. -# -#Licensed under the Apache License, Version 2.0 (the "License"); -#you may not use this file except in compliance with the License. -#You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -#Unless required by applicable law or agreed to in writing, software -#distributed under the License is distributed on an "AS IS" BASIS, -#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -#See the License for the specific language governing permissions and -#limitations under the License. - -import numpy as np -import os -from functools import partial -import logging -import time -import paddle -import paddle.fluid as fluid -import argparse -import network -import reader - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser("gnn") - parser.add_argument( - '--train_path', type=str, default='./data/diginetica/train.txt', help='dir of training data') - parser.add_argument( - '--config_path', type=str, default='./data/diginetica/config.txt', help='dir of config') - parser.add_argument( - '--model_path', type=str, default='./saved_model', help="path of model parameters") - parser.add_argument( - '--epoch_num', type=int, default=30, help='number of epochs to train for') - parser.add_argument( - '--batch_size', type=int, default=100, help='input batch size') - parser.add_argument( - '--hidden_size', type=int, default=100, help='hidden state size') - parser.add_argument( - '--l2', type=float, default=1e-5, help='l2 penalty') - parser.add_argument( - '--lr', type=float, default=0.001, help='learning rate') - parser.add_argument( - '--step', type=int, default=1, help='gnn propogation steps') - parser.add_argument( - '--lr_dc', type=float, default=0.1, help='learning rate decay rate') - parser.add_argument( - '--lr_dc_step', type=int, default=3, help='the number of steps after which the learning rate decay') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether to use gpu') - parser.add_argument( - '--use_parallel', type=int, default=1, help='whether to use parallel executor') - parser.add_argument( - '--enable_ce', action='store_true', help='If set, run the task with continuous evaluation logs.') - return parser.parse_args() - - -def train(): - args = parse_args() - - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - batch_size = args.batch_size - items_num = reader.read_config(args.config_path) - loss, acc, py_reader, feed_datas = network.network(items_num, args.hidden_size, - args.step, batch_size) - - data_reader = reader.Data(args.train_path, True) - logger.info("load data complete") - - use_cuda = True if args.use_cuda else False - use_parallel = True if args.use_parallel else False - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - - exe = fluid.Executor(place) - step_per_epoch = data_reader.length // batch_size - optimizer = fluid.optimizer.Adam( - learning_rate=fluid.layers.exponential_decay( - learning_rate=args.lr, - decay_steps=step_per_epoch * args.lr_dc_step, - decay_rate=args.lr_dc), - regularization=fluid.regularizer.L2DecayRegularizer( - regularization_coeff=args.l2)) - optimizer.minimize(loss) - - exe.run(fluid.default_startup_program()) - - all_vocab = fluid.global_scope().var("all_vocab").get_tensor() - all_vocab.set( - np.arange(1, items_num).astype("int64").reshape((-1)), place) - - feed_list = [e.name for e in feed_datas] - - if use_parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=loss.name) - else: - train_exe = exe - - logger.info("begin train") - - total_time = [] - ce_info = [] - start_time = time.time() - loss_sum = 0.0 - acc_sum = 0.0 - global_step = 0 - PRINT_STEP = 500 - #py_reader.decorate_paddle_reader(data_reader.reader(batch_size, batch_size * 20, True)) - py_reader.set_sample_list_generator(data_reader.reader(batch_size, batch_size * 20, True)) - for i in range(args.epoch_num): - epoch_sum = [] - py_reader.start() - try: - while True: - res = train_exe.run(fetch_list=[loss.name, acc.name]) - loss_sum += res[0].mean() - acc_sum += res[1].mean() - epoch_sum.append(res[0].mean()) - global_step += 1 - if global_step % PRINT_STEP == 0: - ce_info.append([loss_sum / PRINT_STEP, acc_sum / PRINT_STEP]) - total_time.append(time.time() - start_time) - logger.info("global_step: %d, loss: %.4lf, train_acc: %.4lf" % ( - global_step, loss_sum / PRINT_STEP, acc_sum / PRINT_STEP)) - loss_sum = 0.0 - acc_sum = 0.0 - start_time = time.time() - except fluid.core.EOFException: - py_reader.reset() - logger.info("epoch loss: %.4lf" % (np.mean(epoch_sum))) - save_dir = os.path.join(args.model_path, "epoch_" + str(i)) - fetch_vars = [loss, acc] - fluid.save(fluid.default_main_program(), model_path=save_dir) - logger.info("model saved in " + save_dir) - - # only for ce - if args.enable_ce: - gpu_num = get_cards(args) - ce_loss = 0 - ce_acc = 0 - ce_time = 0 - try: - ce_loss = ce_info[-1][0] - ce_acc = ce_info[-1][1] - ce_time = total_time[-1] - except: - print("ce info error") - print("kpis\teach_pass_duration_card%s\t%s" % - (gpu_num, ce_time)) - print("kpis\ttrain_loss_card%s\t%f" % - (gpu_num, ce_loss)) - print("kpis\ttrain_acc_card%s\t%f" % - (gpu_num, ce_acc)) - - -def get_cards(args): - num = 0 - cards = os.environ.get('CUDA_VISIBLE_DEVICES') - num = len(cards.split(",")) - return num - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - train() diff --git a/PaddleRec/gru4rec/infer.py b/PaddleRec/gru4rec/infer.py deleted file mode 100644 index 618bc799bd..0000000000 --- a/PaddleRec/gru4rec/infer.py +++ /dev/null @@ -1,96 +0,0 @@ -import argparse -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle - -import utils - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--start_index', type=int, default='1', help='start index') - parser.add_argument( - '--last_index', type=int, default='10', help='end index') - parser.add_argument( - '--model_dir', type=str, default='model_recall20', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - args = parser.parse_args() - return args - - -def infer(test_reader, use_cuda, model_path): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - model_path, exe) - accum_num_recall = 0.0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - src_wordseq = utils.to_lodtensor([dat[0] for dat in data], place) - label_data = [dat[1] for dat in data] - dst_wordseq = utils.to_lodtensor(label_data, place) - para = exe.run( - infer_program, - feed={"src_wordseq": src_wordseq, - "dst_wordseq": dst_wordseq}, - fetch_list=fetch_vars, - return_numpy=False) - - acc_ = para[1]._get_float_element(0) - data_length = len( - np.concatenate( - label_data, axis=0).astype("int64")) - accum_num_sum += (data_length) - accum_num_recall += (data_length * acc_) - if step_id % 1 == 0: - print("step:%d recall@20:%.4f" % - (step_id, accum_num_recall / accum_num_sum)) - t1 = time.time() - print("model:%s recall@20:%.3f time_cost(s):%.2f" % - (model_path, accum_num_recall / accum_num_sum, t1 - t0)) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_size, test_reader = utils.prepare_data( - test_dir, - vocab_path, - batch_size=batch_size, - buffer_size=1000, - word_freq_threshold=0, - is_train=False) - - for epoch in range(start_index, last_index + 1): - epoch_path = model_dir + "/epoch_" + str(epoch) - infer(test_reader=test_reader, use_cuda=use_cuda, model_path=epoch_path) diff --git a/PaddleRec/gru4rec/infer_sample_neg.py b/PaddleRec/gru4rec/infer_sample_neg.py deleted file mode 100644 index 598644e9ca..0000000000 --- a/PaddleRec/gru4rec/infer_sample_neg.py +++ /dev/null @@ -1,106 +0,0 @@ -import argparse -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle -import net -import utils - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--start_index', type=int, default='1', help='start index') - parser.add_argument( - '--last_index', type=int, default='3', help='last index') - parser.add_argument( - '--model_dir', type=str, default='model_neg_recall20', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - parser.add_argument( - '--hid_size', type=int, default='100', help='batch_size') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - args = parser.parse_args() - return args - - -def infer(args, vocab_size, test_reader, use_cuda): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - hid_size = args.hid_size - batch_size = args.batch_size - with fluid.scope_guard(fluid.Scope()): - main_program = fluid.Program() - with fluid.program_guard(main_program): - acc = net.infer_network(vocab_size, batch_size, hid_size) - for epoch in range(start_index, last_index + 1): - copy_program = main_program.clone() - model_path = model_dir + "/epoch_" + str(epoch) - fluid.load(copy_program, model_path, exe) - accum_num_recall = 0.0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - label_data = [dat[1] for dat in data] - ls, lp = utils.to_lodtensor_bpr_test(data, vocab_size, - place) - para = exe.run( - copy_program, - feed={ - "src": ls, - "all_label": - np.arange(vocab_size).reshape(vocab_size, 1), - "pos_label": lp - }, - fetch_list=[acc.name], - return_numpy=False) - - acc_ = np.array(para[0])[0] - data_length = len( - np.concatenate( - label_data, axis=0).astype("int64")) - accum_num_sum += (data_length) - accum_num_recall += (data_length * acc_) - if step_id % 1 == 0: - print("step:%d recall@20:%.4f" % - (step_id, accum_num_recall / accum_num_sum)) - t1 = time.time() - print("model:%s recall@20:%.4f time_cost(s):%.2f" % - (model_path, accum_num_recall / accum_num_sum, t1 - t0)) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_size, test_reader = utils.prepare_data( - test_dir, - vocab_path, - batch_size=batch_size, - buffer_size=1000, - word_freq_threshold=0, - is_train=False) - - infer(args, vocab_size, test_reader=test_reader, use_cuda=use_cuda) diff --git a/PaddleRec/gru4rec/train.py b/PaddleRec/gru4rec/train.py deleted file mode 100644 index c1516740b1..0000000000 --- a/PaddleRec/gru4rec/train.py +++ /dev/null @@ -1,174 +0,0 @@ -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import paddle.fluid as fluid -import paddle -import time -import utils -import net - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--train_dir', type=str, default='train_data', help='train file') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - parser.add_argument( - '--is_local', type=int, default=1, help='whether is local') - parser.add_argument( - '--hid_size', type=int, default=100, help='hidden-dim size') - parser.add_argument( - '--model_dir', type=str, default='model_recall20', help='model dir') - parser.add_argument( - '--batch_size', type=int, default=5, help='num of batch size') - parser.add_argument( - '--print_batch', type=int, default=10, help='num of print batch') - parser.add_argument( - '--pass_num', type=int, default=10, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether use gpu') - parser.add_argument( - '--parallel', type=int, default=0, help='whether parallel') - parser.add_argument( - '--base_lr', type=float, default=0.01, help='learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - parser.add_argument( - '--step_num', type=int, default=1000, help='Number of steps') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - args = parser.parse_args() - return args - - -def get_cards(args): - return args.num_devices - - -def train(): - """ do training """ - args = parse_args() - if args.enable_ce: - fluid.default_startup_program().random_seed = SEED - fluid.default_main_program().random_seed = SEED - hid_size = args.hid_size - train_dir = args.train_dir - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - parallel = True if args.parallel else False - print("use_cuda:", use_cuda, "parallel:", parallel) - batch_size = args.batch_size - vocab_size, train_reader = utils.prepare_data( - train_dir, vocab_path, batch_size=batch_size * get_cards(args),\ - buffer_size=1000, word_freq_threshold=0, is_train=True) - - # Train program - src_wordseq, dst_wordseq, avg_cost, acc = net.all_vocab_network( - vocab_size=vocab_size, hid_size=hid_size) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - if parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - pass_num = args.pass_num - model_dir = args.model_dir - fetch_list = [avg_cost.name] - - ce_info = [] - total_time = 0.0 - for pass_idx in six.moves.xrange(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - - t0 = time.time() - i = 0 - newest_ppl = 0 - for data in train_reader(): - i += 1 - lod_src_wordseq = utils.to_lodtensor([dat[0] for dat in data], - place) - lod_dst_wordseq = utils.to_lodtensor([dat[1] for dat in data], - place) - ret_avg_cost = train_exe.run(feed={ - "src_wordseq": lod_src_wordseq, - "dst_wordseq": lod_dst_wordseq - }, - fetch_list=fetch_list) - avg_ppl = np.exp(ret_avg_cost[0]) - newest_ppl = np.mean(avg_ppl) - ce_info.append(newest_ppl) - if i % args.print_batch == 0: - print("step:%d ppl:%.3f" % (i, newest_ppl)) - if args.enable_ce and i > args.step_num: - break - - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, i, total_time / epoch_idx)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - feed_var_names = ["src_wordseq", "dst_wordseq"] - fetch_vars = [avg_cost, acc] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - print("model saved in %s" % save_dir) - - # only for ce - if args.enable_ce: - ce_ppl = 0 - try: - ce_ppl = ce_info[-2] - except: - print("ce info error") - epoch_idx = args.pass_num - device = get_device(args) - if args.use_cuda: - gpu_num = device[1] - print("kpis\teach_pass_duration_gpu%s\t%s" % - (gpu_num, total_time / epoch_idx)) - print("kpis\ttrain_ppl_gpu%s\t%s" % (gpu_num, ce_ppl)) - else: - cpu_num = device[1] - threads_num = device[2] - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_ppl_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_ppl)) - - print("finish training") - - -def get_device(args): - if args.use_cuda: - gpus = os.environ.get("CUDA_VISIBLE_DEVICES", 1) - gpu_num = len(gpus.split(',')) - return "gpu", gpu_num - else: - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return "cpu", int(cpu_num), int(threads_num) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/gru4rec/train_sample_neg.py b/PaddleRec/gru4rec/train_sample_neg.py deleted file mode 100644 index f45663bf60..0000000000 --- a/PaddleRec/gru4rec/train_sample_neg.py +++ /dev/null @@ -1,134 +0,0 @@ -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import paddle.fluid as fluid -import paddle -import time -import utils -import net - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("gru4rec benchmark.") - parser.add_argument( - '--train_dir', type=str, default='train_data', help='train file') - parser.add_argument( - '--vocab_path', type=str, default='vocab.txt', help='vocab file') - parser.add_argument( - '--is_local', type=int, default=1, help='whether is local') - parser.add_argument( - '--hid_size', type=int, default=100, help='hidden-dim size') - parser.add_argument( - '--neg_size', type=int, default=10, help='neg item size') - parser.add_argument( - '--loss', type=str, default="bpr", help='loss: bpr/cross_entropy') - parser.add_argument( - '--model_dir', type=str, default='model_neg_recall20', help='model dir') - parser.add_argument( - '--batch_size', type=int, default=5, help='num of batch size') - parser.add_argument( - '--print_batch', type=int, default=10, help='num of print batch') - parser.add_argument( - '--pass_num', type=int, default=10, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether use gpu') - parser.add_argument( - '--parallel', type=int, default=0, help='whether parallel') - parser.add_argument( - '--base_lr', type=float, default=0.01, help='learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - args = parser.parse_args() - return args - - -def get_cards(args): - return args.num_devices - - -def train(): - """ do training """ - args = parse_args() - hid_size = args.hid_size - train_dir = args.train_dir - vocab_path = args.vocab_path - use_cuda = True if args.use_cuda else False - parallel = True if args.parallel else False - print("use_cuda:", use_cuda, "parallel:", parallel) - batch_size = args.batch_size - vocab_size, train_reader = utils.prepare_data( - train_dir, vocab_path, batch_size=batch_size * get_cards(args),\ - buffer_size=1000, word_freq_threshold=0, is_train=True) - - # Train program - if args.loss == 'bpr': - print('bpr loss') - src, pos_label, label, avg_cost = net.train_bpr_network( - neg_size=args.neg_size, vocab_size=vocab_size, hid_size=hid_size) - else: - print('cross-entory loss') - src, pos_label, label, avg_cost = net.train_cross_entropy_network( - neg_size=args.neg_size, vocab_size=vocab_size, hid_size=hid_size) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - if parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - pass_num = args.pass_num - model_dir = args.model_dir - fetch_list = [avg_cost.name] - - total_time = 0.0 - for pass_idx in six.moves.xrange(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - - t0 = time.time() - i = 0 - newest_ppl = 0 - for data in train_reader(): - i += 1 - ls, lp, ll = utils.to_lodtensor_bpr(data, args.neg_size, vocab_size, - place) - ret_avg_cost = train_exe.run( - feed={"src": ls, - "label": ll, - "pos_label": lp}, - fetch_list=fetch_list) - avg_ppl = np.exp(ret_avg_cost[0]) - newest_ppl = np.mean(avg_ppl) - if i % args.print_batch == 0: - print("step:%d ppl:%.3f" % (i, newest_ppl)) - - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, i, total_time / epoch_idx)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - fluid.save(fluid.default_main_program(), model_path=save_dir) - print("model saved in %s" % save_dir) - - print("finish training") - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/multi_task/esmm/infer.py b/PaddleRec/multi_task/esmm/infer.py deleted file mode 100644 index 9f85e02b76..0000000000 --- a/PaddleRec/multi_task/esmm/infer.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import numpy as np -import paddle -import paddle.fluid as fluid -from net import ESMM -import args -import logging -import utils - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def set_zero(place): - auc_states_names = [ - 'auc_1.tmp_0', 'auc_0.tmp_0' - ] - for name in auc_states_names: - param = fluid.global_scope().var(name).get_tensor() - if param: - param_array = np.zeros(param._get_dims()).astype("int64") - param.set(param_array, place) - -def run_infer(args,model_path,test_data_path,vocab_size): - place = fluid.CPUPlace() - esmm_model = ESMM() - - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - - with fluid.framework.program_guard(test_program, startup_program): - with fluid.unique_name.guard(): - inputs = esmm_model.input_data() - avg_cost,auc_ctr,auc_ctcvr= esmm_model.net(inputs, vocab_size, args.embed_size) - - dataset, file_list = utils.get_dataset(inputs, test_data_path,args.batch_size,args.cpu_num) - - exe = fluid.Executor(place) - fluid.load(fluid.default_main_program(),os.path.join(model_path, "checkpoint"), exe) - - set_zero(place) - - dataset.set_filelist(file_list) - exe.infer_from_dataset(program=test_program, - dataset=dataset, - fetch_list=[auc_ctr,auc_ctcvr], - fetch_info=["auc_ctr","auc_ctcvr"], - print_period=20, - debug=False) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - model_list = [] - for _, dir, _ in os.walk(args.model_dir): - for model in dir: - if "epoch" in model: - path = os.path.join(args.model_dir, model) - model_list.append(path) - - vocab_size =utils.get_vocab_size(args.vocab_path) - - for model in model_list: - logger.info("Test model {}".format(model)) - run_infer(args, model,args.test_data_path) - - \ No newline at end of file diff --git a/PaddleRec/multi_task/esmm/train.py b/PaddleRec/multi_task/esmm/train.py deleted file mode 100644 index 57abc81e13..0000000000 --- a/PaddleRec/multi_task/esmm/train.py +++ /dev/null @@ -1,45 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from net import ESMM -import paddle -import utils -import args - -def train(args, vocab_size, train_data_path): - esmm_model = ESMM() - inputs = esmm_model.input_data() - - dataset, file_list = utils.get_dataset(inputs, train_data_path,args.batch_size,args.cpu_num) - - avg_cost,auc_ctr,auc_ctcvr= esmm_model.net(inputs, vocab_size, args.embed_size) - optimizer = fluid.optimizer.Adam() - optimizer.minimize(avg_cost) - - if args.use_gpu == True: - exe = fluid.Executor(fluid.CUDAPlace(0)) - dataset.set_thread(1) - else: - exe = fluid.Executor(fluid.CPUPlace()) - dataset.set_thread(args.cpu_num) - - exe.run(fluid.default_startup_program()) - - for epoch in range(args.epochs): - dataset.set_filelist(file_list) - exe.train_from_dataset(program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[avg_cost,auc_ctr,auc_ctcvr], - fetch_info=['epoch %d batch loss' % (epoch), "auc_ctr","auc_ctcvr"], - print_period=20, - debug=False) - model_dir = os.path.join(args.model_dir,'epoch_' + str(epoch + 1), "checkpoint") - main_program = fluid.default_main_program() - fluid.io.save(main_program,model_dir) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - vocab_size =utils.get_vocab_size(args.vocab_path) - train(args, vocab_size, args.train_data_path) diff --git a/PaddleRec/multiview_simnet/infer.py b/PaddleRec/multiview_simnet/infer.py deleted file mode 100644 index 9dea35b0d7..0000000000 --- a/PaddleRec/multiview_simnet/infer.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import logging -import paddle.fluid as fluid -import paddle -import time -import reader as reader -from nets import MultiviewSimnet, SimpleEncoderFactory - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def check_version(): - """ - Log error and exit when the installed version of paddlepaddle is - not satisfied. - """ - err = "PaddlePaddle version 1.6 or higher is required, " \ - "or a suitable develop version is satisfied as well. \n" \ - "Please make sure the version is good with your code." \ - - try: - fluid.require_version('1.6.0') - except Exception as e: - logger.error(err) - sys.exit(1) - - -def parse_args(): - parser = argparse.ArgumentParser("multi-view simnet") - parser.add_argument("--train_file", type=str, help="Training file") - parser.add_argument("--valid_file", type=str, help="Validation file") - parser.add_argument( - "--epochs", type=int, default=10, help="Number of epochs for training") - parser.add_argument( - "--model_dir", - type=str, - default='model_output', - help="Model output folder") - parser.add_argument( - "--query_slots", type=int, default=1, help="Number of query slots") - parser.add_argument( - "--title_slots", type=int, default=1, help="Number of title slots") - parser.add_argument( - "--query_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--title_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--query_encode_dim", - type=int, - default=128, - help="Dimension of query encoder output") - parser.add_argument( - "--title_encode_dim", - type=int, - default=128, - help="Dimension of title encoder output") - parser.add_argument( - "--batch_size", type=int, default=128, help="Batch size for training") - parser.add_argument( - "--embedding_dim", - type=int, - default=128, - help="Default Dimension of Embedding") - parser.add_argument( - "--sparse_feature_dim", - type=int, - default=1000001, - help="Sparse feature hashing space for index processing") - parser.add_argument( - "--hidden_size", type=int, default=128, help="Hidden dim") - return parser.parse_args() - - -def start_infer(args, model_path): - dataset = reader.SyntheticDataset(args.sparse_feature_dim, args.query_slots, - args.title_slots) - test_reader = fluid.io.batch( - fluid.io.shuffle( - dataset.valid(), buf_size=args.batch_size * 100), - batch_size=args.batch_size) - place = fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - args.model_dir, exe) - t0 = time.time() - step_id = 0 - feeder = fluid.DataFeeder( - program=infer_program, feed_list=feed_target_names, place=place) - for batch_id, data in enumerate(test_reader()): - step_id += 1 - loss_val, correct_val = exe.run(infer_program, - feed=feeder.feed(data), - fetch_list=fetch_vars) - logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}" - .format(step_id, batch_id, loss_val, - float(correct_val) / args.batch_size)) - - -def main(): - args = parse_args() - start_infer(args, args.model_dir) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - check_version() - main() diff --git a/PaddleRec/multiview_simnet/train.py b/PaddleRec/multiview_simnet/train.py deleted file mode 100644 index 5bbd169c53..0000000000 --- a/PaddleRec/multiview_simnet/train.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import logging -import paddle.fluid as fluid -import paddle -import time -import reader as reader -from nets import MultiviewSimnet, SimpleEncoderFactory - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser("multi-view simnet") - parser.add_argument("--train_file", type=str, help="Training file") - parser.add_argument("--valid_file", type=str, help="Validation file") - parser.add_argument( - "--epochs", type=int, default=10, help="Number of epochs for training") - parser.add_argument( - "--model_output_dir", - type=str, - default='model_output', - help="Model output folder") - parser.add_argument( - "--query_slots", type=int, default=1, help="Number of query slots") - parser.add_argument( - "--title_slots", type=int, default=1, help="Number of title slots") - parser.add_argument( - "--query_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--title_encoder", - type=str, - default="bow", - help="Encoder module for slot encoding") - parser.add_argument( - "--query_encode_dim", - type=int, - default=128, - help="Dimension of query encoder output") - parser.add_argument( - "--title_encode_dim", - type=int, - default=128, - help="Dimension of title encoder output") - parser.add_argument( - "--batch_size", type=int, default=128, help="Batch size for training") - parser.add_argument( - "--embedding_dim", - type=int, - default=128, - help="Default Dimension of Embedding") - parser.add_argument( - "--sparse_feature_dim", - type=int, - default=1000001, - help="Sparse feature hashing space" - "for index processing") - parser.add_argument( - "--hidden_size", type=int, default=128, help="Hidden dim") - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - return parser.parse_args() - - -def check_version(): - """ - Log error and exit when the installed version of paddlepaddle is - not satisfied. - """ - err = "PaddlePaddle version 1.6 or higher is required, " \ - "or a suitable develop version is satisfied as well. \n" \ - "Please make sure the version is good with your code." \ - - try: - fluid.require_version('1.6.0') - except Exception as e: - logger.error(err) - sys.exit(1) - - -def start_train(args): - if args.enable_ce: - SEED = 102 - fluid.default_startup_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - dataset = reader.SyntheticDataset(args.sparse_feature_dim, args.query_slots, - args.title_slots) - train_reader = fluid.io.batch( - fluid.io.shuffle( - dataset.train(), buf_size=args.batch_size * 100), - batch_size=args.batch_size) - place = fluid.CPUPlace() - factory = SimpleEncoderFactory() - query_encoders = [ - factory.create(args.query_encoder, args.query_encode_dim) - for i in range(args.query_slots) - ] - title_encoders = [ - factory.create(args.title_encoder, args.title_encode_dim) - for i in range(args.title_slots) - ] - m_simnet = MultiviewSimnet(args.sparse_feature_dim, args.embedding_dim, - args.hidden_size) - m_simnet.set_query_encoder(query_encoders) - m_simnet.set_title_encoder(title_encoders) - all_slots, avg_cost, correct = m_simnet.train_net() - optimizer = fluid.optimizer.Adam(learning_rate=1e-4) - optimizer.minimize(avg_cost) - startup_program = fluid.default_startup_program() - loop_program = fluid.default_main_program() - - exe = fluid.Executor(place) - exe.run(startup_program) - loader = fluid.io.DataLoader.from_generator( - feed_list=all_slots, capacity=10000, iterable=True) - loader.set_sample_list_generator(train_reader, places=place) - - total_time = 0 - ce_info = [] - for pass_id in range(args.epochs): - start_time = time.time() - for batch_id, data in enumerate(loader()): - loss_val, correct_val = exe.run(loop_program, - feed=data, - fetch_list=[avg_cost, correct]) - logger.info("TRAIN --> pass: {} batch_id: {} avg_cost: {}, acc: {}" - .format(pass_id, batch_id, loss_val, - float(correct_val) / args.batch_size)) - ce_info.append(loss_val[0]) - end_time = time.time() - total_time += end_time - start_time - fluid.io.save_inference_model(args.model_output_dir, - [val.name for val in all_slots], - [avg_cost, correct], exe) - - # only for ce - if args.enable_ce: - threads_num, cpu_num = get_cards(args) - epoch_idx = args.epochs - ce_loss = 0 - try: - ce_loss = ce_info[-2] - except: - logger.error("ce info error") - - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_loss_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_loss)) - - -def get_cards(args): - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return int(threads_num), int(cpu_num) - - -def main(): - args = parse_args() - start_train(args) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - check_version() - main() diff --git a/PaddleRec/ncf/infer.py b/PaddleRec/ncf/infer.py deleted file mode 100644 index a9b628ab6b..0000000000 --- a/PaddleRec/ncf/infer.py +++ /dev/null @@ -1,34 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from gmf import GMF -from mlp import MLP -from neumf import NeuMF -from Dataset import Dataset -from evaluate import evaluate_model -import logging -import paddle -import args -import utils -import time -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - dataset = Dataset(args.path + args.dataset) - testRatings, testNegatives = dataset.testRatings, dataset.testNegatives - topK = 10 - - begin = time.time() - model_path = args.model_dir + "/epoch_" + str(12) - (hits, ndcgs) = evaluate_model(args, testRatings, testNegatives, topK, model_path) - hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() - end = time.time() - - logger.info("epoch: {}, epoch_time: {:.5f}s, HR: {:.5f}, NDCG: {:.5f}".format(args.epochs, end - begin, hr, ndcg)) - \ No newline at end of file diff --git a/PaddleRec/ncf/train.py b/PaddleRec/ncf/train.py deleted file mode 100644 index d1d79e59cf..0000000000 --- a/PaddleRec/ncf/train.py +++ /dev/null @@ -1,70 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -from gmf import GMF -from mlp import MLP -from neumf import NeuMF -from Dataset import Dataset -import logging -import paddle -import args -import utils -import time -from evaluate import evaluate_model - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args, train_data_path): - print("use_gpu:{}, NeuMF:{}, epochs:{}, batch_size:{}, num_factors:{}, num_neg:{}, lr:{}, model_dir:{}, layers:{}".format( - args.use_gpu, args.NeuMF, args.epochs, args.batch_size, args.num_factors, args.num_neg, args.lr, args.model_dir, args.layers)) - dataset = Dataset(args.path + args.dataset) - testRatings, testNegatives = dataset.testRatings, dataset.testNegatives - - train_data_generator = utils.Dataset() - train_reader = fluid.io.batch(train_data_generator.train(train_data_path, True), batch_size=args.batch_size) - - inputs = utils.input_data(True) - if args.GMF: - model = GMF() - loss, pred = model.net(inputs, args.num_users, args.num_items, args.num_factors) - elif args.MLP: - model = MLP() - loss, pred = model.net(inputs, args.num_users, args.num_items, args.layers) - elif args.NeuMF: - model = NeuMF() - loss, pred = model.net(inputs, args.num_users, args.num_items, args.num_factors, args.layers) - - optimizer = fluid.optimizer.AdamOptimizer(args.lr) - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - loader = fluid.io.DataLoader.from_generator( - feed_list=inputs, capacity=args.batch_size, iterable=True) - loader.set_sample_list_generator(train_reader, places=place) - - for epoch in range(args.epochs): - - for batch_id, data in enumerate(loader()): - begin = time.time() - loss_val = exe.run(program=fluid.default_main_program(), - feed=data, - fetch_list=[loss.name], - return_numpy=True) - end = time.time() - logger.info("epoch: {}, batch_id: {}, batch_time: {:.5f}s, loss: {:.5f}".format(epoch, batch_id, end - begin, np.array(loss_val)[0][0])) - - save_dir = "%s/epoch_%d" % (args.model_dir, epoch) - feed_var_names = ["user_input", "item_input"] - fetch_vars = [pred] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - train(args, args.train_data_path) diff --git a/PaddleRec/rerank/listwise/infer.py b/PaddleRec/rerank/listwise/infer.py deleted file mode 100644 index f024d625c4..0000000000 --- a/PaddleRec/rerank/listwise/infer.py +++ /dev/null @@ -1,84 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -import logging -import args -import random -import time -from evaluator import BiRNN - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def set_zero(var_name, scope=fluid.global_scope(), place=fluid.CPUPlace(), param_type="int64"): - """ - Set tensor of a Variable to zero. - Args: - var_name(str): name of Variable - scope(Scope): Scope object, default is fluid.global_scope() - place(Place): Place object, default is fluid.CPUPlace() - param_type(str): param data type, default is int64 - """ - param = scope.var(var_name).get_tensor() - param_array = np.zeros(param._get_dims()).astype(param_type) - param.set(param_array, place) - -def run_infer(args): - model = BiRNN() - inference_scope = fluid.Scope() - startup_program = fluid.framework.Program() - test_program = fluid.framework.Program() - cur_model_path = os.path.join(args.model_dir, 'epoch_' + str(args.test_epoch), "checkpoint") - with fluid.scope_guard(inference_scope): - with fluid.framework.program_guard(test_program, startup_program): - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - inputs = model.input_data(args.item_len) - loss, auc_val, batch_auc, auc_states = model.net(inputs, args.hidden_size, args.batch_size*args.sample_size, args.item_vocab, args.embd_dim) - exe = fluid.Executor(place) - - fluid.load(fluid.default_main_program(), cur_model_path, exe) - for var in auc_states: # reset auc states - set_zero(var.name, scope=inference_scope, place=place) - - # Build a random data set. - user_slot_names = [] - item_slot_names = [] - lens = [] - labels = [] - user_id = 0 - for i in range(args.sample_size): - user_slot_name = [] - for j in range(args.batch_size): - user_slot_name.append(user_id) - user_id += 1 - user_slot_names.append(user_slot_name) - - item_slot_name = np.random.randint(args.item_vocab, size=(args.batch_size, args.item_len)) - item_slot_names.append(item_slot_name) - lenght = np.array([args.item_len]*args.batch_size) - lens.append(lenght) - label = np.random.randint(2, size=(args.batch_size, args.item_len)) - labels.append(label) - - for i in range(args.sample_size): - begin = time.time() - loss_val, auc = exe.run(test_program, - feed={ - "user_slot_names": np.array(user_slot_names[i]).reshape(args.batch_size, 1), - "item_slot_names": item_slot_names[i].astype('int64'), - "lens": lens[i].astype('int64'), - "labels": labels[i].astype('int64') - }, - return_numpy=True, - fetch_list=[loss.name, auc_val]) - end = time.time() - logger.info("batch_time: {:.5f}s, loss: {:.5f}, auc: {:.5f}".format( - end-begin, float(np.array(loss_val)), float(np.array(auc)))) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - - args = args.parse_args() - run_infer(args) \ No newline at end of file diff --git a/PaddleRec/rerank/listwise/train.py b/PaddleRec/rerank/listwise/train.py deleted file mode 100644 index 7bc1e58e55..0000000000 --- a/PaddleRec/rerank/listwise/train.py +++ /dev/null @@ -1,72 +0,0 @@ -import numpy as np -import os -import paddle.fluid as fluid -import logging -import args -import random -import time -from evaluator import BiRNN - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args): - - model = BiRNN() - inputs = model.input_data(args.item_len) - loss, auc_val, batch_auc, auc_states = model.net(inputs, args.hidden_size, args.batch_size*args.sample_size, args.item_vocab, args.embd_dim) - - optimizer = fluid.optimizer.Adam(learning_rate=args.base_lr, epsilon=1e-4) - optimizer.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - # Build a random data set. - user_slot_names = [] - item_slot_names = [] - lens = [] - labels = [] - user_id = 0 - for i in range(args.sample_size): - user_slot_name = [] - for j in range(args.batch_size): - user_slot_name.append(user_id) - user_id += 1 - user_slot_names.append(user_slot_name) - - item_slot_name = np.random.randint(args.item_vocab, size=(args.batch_size, args.item_len)) - item_slot_names.append(item_slot_name) - lenght = np.array([args.item_len]*args.batch_size) - lens.append(lenght) - label = np.random.randint(2, size=(args.batch_size, args.item_len)) - labels.append(label) - - for epoch in range(args.epochs): - for i in range(args.sample_size): - begin = time.time() - loss_val, auc = exe.run(fluid.default_main_program(), - feed={ - "user_slot_names": np.array(user_slot_names[i]).reshape(args.batch_size, 1), - "item_slot_names": item_slot_names[i].astype('int64'), - "lens": lens[i].astype('int64'), - "labels": labels[i].astype('int64') - }, - return_numpy=True, - fetch_list=[loss.name, auc_val]) - end = time.time() - logger.info("epoch_id: {}, batch_time: {:.5f}s, loss: {:.5f}, auc: {:.5f}".format( - epoch, end-begin, float(np.array(loss_val)), float(np.array(auc)))) - - #save model - model_dir = os.path.join(args.model_dir, 'epoch_' + str(epoch + 1), "checkpoint") - main_program = fluid.default_main_program() - fluid.save(main_program, model_dir) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - train(args) \ No newline at end of file diff --git a/PaddleRec/tagspace/infer.py b/PaddleRec/tagspace/infer.py deleted file mode 100644 index 58fe933cc3..0000000000 --- a/PaddleRec/tagspace/infer.py +++ /dev/null @@ -1,102 +0,0 @@ -import sys -import argparse -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle -import utils - - -def parse_args(): - parser = argparse.ArgumentParser("tagspace benchmark.") - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--vocab_tag_path', - type=str, - default='vocab_tag.txt', - help='vocab path') - parser.add_argument( - '--start_index', type=int, default='1', help='start index') - parser.add_argument( - '--last_index', type=int, default='10', help='end index') - parser.add_argument( - '--model_dir', type=str, default='model_', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - args = parser.parse_args() - return args - - -def infer(test_reader, vocab_tag, use_cuda, model_path, epoch): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model( - model_path, exe) - t0 = time.time() - step_id = 0 - true_num = 0 - all_num = 0 - size = vocab_tag - value = [] - print("epoch " + str(epoch) + " start") - for data in test_reader(): - step_id += 1 - lod_text_seq = utils.to_lodtensor([dat[0] for dat in data], place) - lod_tag = utils.to_lodtensor([dat[1] for dat in data], place) - lod_pos_tag = utils.to_lodtensor([dat[2] for dat in data], place) - para = exe.run(infer_program, - feed={"text": lod_text_seq, - "pos_tag": lod_tag}, - fetch_list=fetch_vars, - return_numpy=False) - value.append(para[0]._get_float_element(0)) - if step_id % size == 0 and step_id > 1: - all_num += 1 - true_pos = [dat[2] for dat in data][0][0] - if value.index(max(value)) == int(true_pos): - true_num += 1 - value = [] - print("epoch:" + str(epoch) + "\tacc:" + str(1.0 * true_num / all_num)) - t1 = time.time() - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - vocab_tag_path = args.vocab_tag_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_text, vocab_tag, test_reader = utils.prepare_data( - test_dir, - "", - vocab_tag_path, - batch_size=1, - neg_size=0, - buffer_size=1000, - is_train=False) - - for epoch in range(start_index, last_index + 1): - epoch_path = model_dir + "/epoch_" + str(epoch) - infer( - test_reader=test_reader, - vocab_tag=vocab_tag, - use_cuda=False, - model_path=epoch_path, - epoch=epoch) diff --git a/PaddleRec/tagspace/train.py b/PaddleRec/tagspace/train.py deleted file mode 100644 index ee2054a7b1..0000000000 --- a/PaddleRec/tagspace/train.py +++ /dev/null @@ -1,173 +0,0 @@ -import os -import sys -import time -import six -import numpy as np -import math -import argparse -import paddle -import paddle.fluid as fluid -import time -import utils -import net - -SEED = 102 - - -def parse_args(): - parser = argparse.ArgumentParser("TagSpace benchmark.") - parser.add_argument( - '--neg_size', type=int, default=3, help='number of neg item') - parser.add_argument( - '--train_dir', type=str, default='train_data', help='train file') - parser.add_argument( - '--vocab_text_path', type=str, default='vocab_text.txt', help='text') - parser.add_argument( - '--vocab_tag_path', type=str, default='vocab_tag.txt', help='tag') - parser.add_argument( - '--model_dir', type=str, default='model_', help='model dir') - parser.add_argument( - '--batch_size', type=int, default=5, help='num of batch size') - parser.add_argument( - '--print_batch', type=int, default=10, help='num of print batch') - parser.add_argument( - '--pass_num', type=int, default=10, help='number of epoch') - parser.add_argument( - '--use_cuda', type=int, default=0, help='whether use gpu') - parser.add_argument( - '--parallel', type=int, default=0, help='whether parallel') - parser.add_argument( - '--base_lr', type=float, default=0.01, help='learning rate') - parser.add_argument( - '--num_devices', type=int, default=1, help='Number of GPU devices') - parser.add_argument( - '--enable_ce', - action='store_true', - help='If set, run the task with continuous evaluation logs.') - args = parser.parse_args() - return args - - -def get_cards(args): - return args.num_devices - - -def train(): - """ do training """ - args = parse_args() - if args.enable_ce: - fluid.default_startup_program().random_seed = SEED - fluid.default_main_program().random_seed = SEED - train_dir = args.train_dir - vocab_text_path = args.vocab_text_path - vocab_tag_path = args.vocab_tag_path - use_cuda = True if args.use_cuda else False - parallel = True if args.parallel else False - batch_size = args.batch_size - neg_size = args.neg_size - print("use_cuda: {}, parallel: {}, batch_size: {}, neg_size: {} " - .format(use_cuda, parallel, batch_size, neg_size)) - vocab_text_size, vocab_tag_size, train_reader = utils.prepare_data( - file_dir=train_dir, - vocab_text_path=vocab_text_path, - vocab_tag_path=vocab_tag_path, - neg_size=neg_size, - batch_size=batch_size * get_cards(args), - buffer_size=batch_size * 100, - is_train=True) - """ train network """ - # Train program - avg_cost, correct, cos_pos = net.network( - vocab_text_size, vocab_tag_size, neg_size=neg_size) - - # Optimization to minimize lost - sgd_optimizer = fluid.optimizer.Adagrad(learning_rate=args.base_lr) - sgd_optimizer.minimize(avg_cost) - - # Initialize executor - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - if parallel: - train_exe = fluid.ParallelExecutor( - use_cuda=use_cuda, loss_name=avg_cost.name) - else: - train_exe = exe - - pass_num = args.pass_num - model_dir = args.model_dir - fetch_list = [avg_cost.name] - total_time = 0.0 - ce_info = [] - for pass_idx in range(pass_num): - epoch_idx = pass_idx + 1 - print("epoch_%d start" % epoch_idx) - t0 = time.time() - for batch_id, data in enumerate(train_reader()): - lod_text_seq = utils.to_lodtensor([dat[0] for dat in data], place) - lod_pos_tag = utils.to_lodtensor([dat[1] for dat in data], place) - lod_neg_tag = utils.to_lodtensor([dat[2] for dat in data], place) - loss_val, correct_val = train_exe.run( - feed={ - "text": lod_text_seq, - "pos_tag": lod_pos_tag, - "neg_tag": lod_neg_tag - }, - fetch_list=[avg_cost.name, correct.name]) - ce_info.append( - float(np.sum(correct_val)) / (args.num_devices * batch_size)) - if batch_id % args.print_batch == 0: - print("TRAIN --> pass: {} batch_num: {} avg_cost: {}, acc: {}" - .format(pass_idx, (batch_id + 10) * batch_size, - np.mean(loss_val), - float(np.sum(correct_val)) / (args.num_devices * - batch_size))) - t1 = time.time() - total_time += t1 - t0 - print("epoch:%d num_steps:%d time_cost(s):%f" % - (epoch_idx, batch_id, total_time / epoch_idx)) - save_dir = "%s/epoch_%d" % (model_dir, epoch_idx) - feed_var_names = ["text", "pos_tag"] - fetch_vars = [cos_pos] - fluid.io.save_inference_model(save_dir, feed_var_names, fetch_vars, exe) - # only for ce - if args.enable_ce: - ce_acc = 0 - try: - ce_acc = ce_info[-2] - except: - logger.error("ce info error") - epoch_idx = args.pass_num - device = get_device(args) - if args.use_cuda: - gpu_num = device[1] - print("kpis\teach_pass_duration_gpu%s\t%s" % - (gpu_num, total_time / epoch_idx)) - print("kpis\ttrain_acc_gpu%s\t%s" % (gpu_num, ce_acc)) - else: - cpu_num = device[1] - threads_num = device[2] - print("kpis\teach_pass_duration_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, total_time / epoch_idx)) - print("kpis\ttrain_acc_cpu%s_thread%s\t%s" % - (cpu_num, threads_num, ce_acc)) - - print("finish training") - - -def get_device(args): - if args.use_cuda: - gpus = os.environ.get("CUDA_VISIBLE_DEVICES", 1) - gpu_num = len(gpus.split(',')) - return "gpu", gpu_num - else: - threads_num = os.environ.get('NUM_THREADS', 1) - cpu_num = os.environ.get('CPU_NUM', 1) - return "cpu", int(cpu_num), int(threads_num) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - train() diff --git a/PaddleRec/tdm/tdm_demo/dataset_generator.py b/PaddleRec/tdm/tdm_demo/dataset_generator.py deleted file mode 100644 index fccd37b805..0000000000 --- a/PaddleRec/tdm/tdm_demo/dataset_generator.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- coding=utf8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import json -import pickle -import time -import random -import os -import numpy as np -import sys -import paddle.fluid.incubate.data_generator as dg - - -class TDMDataset(dg.MultiSlotStringDataGenerator): - """ - DacDataset: inheritance MultiSlotDataGeneratior, Implement data reading - Help document: http://wiki.baidu.com/pages/viewpage.action?pageId=728820675 - """ - - def infer_reader(self, infer_file_list, batch): - """ - Read test_data line by line & yield batch - """ - - def local_iter(): - """Read file line by line""" - for fname in infer_file_list: - with open(fname, "r") as fin: - for line in fin: - one_data = (line.strip('\n')).split('\t') - input_emb = one_data[0].split(' ') - - yield [input_emb] - - import paddle - batch_iter = fluid.io.batch(local_iter, batch) - return batch_iter - - def generate_sample(self, line): - """ - Read the data line by line and process it as a dictionary - """ - - def iterator(): - """ - This function needs to be implemented by the user, based on data format - """ - features = (line.strip('\n')).split('\t') - input_emb = features[0].split(' ') - item_label = [features[1]] - - feature_name = ["input_emb", "item_label"] - yield zip(feature_name, [input_emb] + [item_label]) - - return iterator - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - d = TDMDataset() - d.run_from_stdin() diff --git a/PaddleRec/tdm/tdm_demo/distributed_train.py b/PaddleRec/tdm/tdm_demo/distributed_train.py deleted file mode 100644 index e82e5f74ce..0000000000 --- a/PaddleRec/tdm/tdm_demo/distributed_train.py +++ /dev/null @@ -1,250 +0,0 @@ -# -*- coding=utf-8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -from __future__ import print_function -import os -import time -import numpy as np -import logging -import random -from shutil import copyfile -import paddle.fluid as fluid -import paddle.fluid.incubate.fleet.base.role_maker as role_maker - -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler import fleet -from paddle.fluid.incubate.fleet.utils.hdfs import HDFSClient -from paddle.fluid.incubate.fleet.parameter_server.distribute_transpiler.distributed_strategy import StrategyFactory - -from args import print_arguments, parse_args -from utils import tdm_sampler_prepare, tdm_child_prepare, tdm_emb_prepare - -from train_network import TdmTrainNet - - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -hadoop_home = os.getenv("HADOOP_HOME") -configs = { - "fs.default.name": os.getenv("FS_NAME"), - "hadoop.job.ugi": os.getenv("FS_UGI") -} -client = HDFSClient(hadoop_home, configs) - - -def get_dataset(inputs, args): - """get dataset""" - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(inputs) - dataset.set_pipe_command("python ./dataset_generator.py") - dataset.set_batch_size(args.batch_size) - dataset.set_thread(int(args.cpu_num)) - file_list = [ - str(args.train_files_path) + "/%s" % x - for x in os.listdir(args.train_files_path) - ] - - # 请确保每一个训练节点都持有不同的训练文件 - # 当我们用本地多进程模拟分布式时，每个进程需要拿到不同的文件 - # 使用 fleet.split_files 可以便捷的以文件为单位分配训练样本 - if not int(args.is_cloud): - file_list = fleet.split_files(file_list) - logger.info("file list: {}".format(file_list)) - total_example_num = get_example_num(file_list) - return dataset, file_list, total_example_num - - -def train(args): - """run train""" - # set random - program = fluid.default_main_program() - program.random_seed = args.random_seed - - # 根据环境变量确定当前机器/进程在分布式训练中扮演的角色 - # 然后使用 fleet api的 init()方法初始化这个节点 - role = role_maker.PaddleCloudRoleMaker() - fleet.init(role) - - # 我们还可以进一步指定分布式的运行模式，通过 DistributeTranspilerConfig进行配置 - # 如下，我们设置分布式运行模式为异步(async)，同时将参数进行切分，以分配到不同的节点 - if args.sync_mode == "sync": - strategy = StrategyFactory.create_sync_strategy() - elif args.sync_mode == "half_async": - strategy = StrategyFactory.create_half_async_strategy() - elif args.sync_mode == "async": - strategy = StrategyFactory.create_async_strategy() - - # set model - logger.info("TDM Begin build network.") - tdm_model = TdmTrainNet(args) - inputs = tdm_model.input_data() - - logger.info("TDM Begin load tree travel & layer.") - avg_cost, acc = tdm_model.tdm(inputs) - logger.info("TDM End build network.") - # 配置分布式的optimizer，传入我们指定的strategy，构建program - optimizer = fluid.optimizer.AdamOptimizer( - learning_rate=args.learning_rate, lazy_mode=True) - - optimizer = fleet.distributed_optimizer(optimizer, strategy) - optimizer.minimize(avg_cost) - logger.info("TDM End append backward.") - - # 根据节点角色，分别运行不同的逻辑 - if fleet.is_server(): - logger.info("TDM Run server ...") - # 初始化及运行参数服务器节点 - logger.info("TDM init model path: {}".format( - args.init_model_files_path)) - # 模型中除了tdm树结构相关的变量都应该在此处初始化 - fleet.init_server(args.init_model_files_path) - lr = fluid.global_scope().find_var("learning_rate_0") - if lr: - lr.get_tensor().set(np.array(args.learning_rate).astype('float32'), - fluid.CPUPlace()) - logger.info("TDM Set learning rate {}".format(args.learning_rate)) - else: - logger.info("TDM Didn't find learning_rate_0 param") - logger.info("TDM load End") - - fleet.run_server() - logger.info("TDM Run server success!") - elif fleet.is_worker(): - logger.info("TDM Run worker ...") - # 初始化工作节点 - fleet.init_worker() - place = fluid.CPUPlace() - exe = fluid.Executor(place) - logger.info("TDM Run Startup Begin") - # 初始化含有分布式流程的fleet.startup_program - exe.run(fleet.startup_program) - - # Set Learning Rate - lr = fluid.global_scope().find_var("learning_rate_0") - if lr: - lr.get_tensor().set(np.array(args.learning_rate).astype('float32'), - place) - logger.info("TDM Set learning rate {}".format(args.learning_rate)) - - # Set TDM Variable - logger.info("TDM Begin load parameter.") - # Set TDM_Tree_Info - # 树结构相关的变量不参与网络更新，不存储于参数服务器，因此需要在本地手动Set - tdm_param_prepare_dict = tdm_sampler_prepare(args) - tdm_param_prepare_dict['info_array'] = tdm_child_prepare(args) - Numpy_model = {} - Numpy_model['TDM_Tree_Travel'] = tdm_param_prepare_dict['travel_array'] - Numpy_model['TDM_Tree_Layer'] = tdm_param_prepare_dict['layer_array'] - Numpy_model['TDM_Tree_Info'] = tdm_param_prepare_dict['info_array'] - # Numpy_model['TDM_Tree_Emb'] = tdm_emb_prepare(args) - # 分布式训练中，Emb存储与参数服务器，无需在本地set - for param_name in Numpy_model: - param_t = fluid.global_scope().find_var(param_name).get_tensor() - param_t.set(Numpy_model[str(param_name)].astype('int32'), place) - - logger.info("TDM Run Startup End") - - # Train loop - dataset, file_list, example_num = get_dataset(inputs, args) - logger.info("TDM Distributed training begin ...") - for epoch in range(args.epoch_num): - # local shuffle - random.shuffle(file_list) - dataset.set_filelist(file_list) - - # 训练节点运行的是经过分布式裁剪的fleet.mian_program - start_time = time.time() - exe.train_from_dataset(program=fleet.main_program, - dataset=dataset, - fetch_list=[acc, avg_cost], - fetch_info=["Epoch {} acc ".format( - epoch), "Epoch {} loss ".format(epoch)], - print_period=1, - debug=False) - end_time = time.time() - logger.info("Epoch {} finished, use time {} second, speed {} example/s".format( - epoch, end_time - start_time, example_num * 1.0 / (end_time - start_time))) - - # 默认使用0号节点保存模型 - if fleet.is_first_worker(): - model_path = os.path.join( - args.model_files_path, "epoch_" + str(epoch)) - fleet.save_persistables(executor=exe, dirname=model_path) - logger.info("Begin upload files") - # upload_files(model_path, warm_up=False) - # 在分布式环境下时，支持上传模型到hdfs - logger.info("TDM Before stop worker") - fleet.stop_worker() - logger.info("TDM Distributed training success!") - - -def upload_files(local_path, warm_up=False): - """ - upload files to hdfs - """ - remote = os.getenv("OUTPUT_PATH") - job_id = os.getenv("SYS_JOB_ID") - local = local_path.split('/')[-1] - remote_path = "{}/{}/{}/{}".format(remote, job_id, "model", local) - client.makedirs(remote_path) - hadoop_path = "{}/".format(remote_path) - - def is_adam_param(name): - adam_name = ['bias_beta', 'bias_moment', - 'moment1_0', 'moment2_0', 'pow_acc'] - for i in adam_name: - if i in name: - return True - return False - - if not warm_up: - infer_model_path = os.path.join(os.getcwd(), 'infer_model_'+local) - if not os.path.exists(infer_model_path): - os.makedirs(infer_model_path) - for root, _, files in os.walk(local_path): - for f in files: - if not is_adam_param(f): - copyfile(os.path.join(root, f), - os.path.join(infer_model_path, f)) - local_path = infer_model_path - - client.upload(hdfs_path=hadoop_path, local_path=local_path, - multi_processes=5, overwrite=False, - retry_times=3) - - -def get_example_num(file_list): - """ - Count the number of samples in the file - """ - count = 0 - for f in file_list: - last_count = count - for index, line in enumerate(open(f, 'r')): - count += 1 - logger.info("file : %s has %s example" % (f, count - last_count)) - logger.info("Total example : %s" % count) - return count - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - print(os.getcwd()) - args = parse_args() - print_arguments(args) - train(args) diff --git a/PaddleRec/tdm/tdm_demo/local_infer.py b/PaddleRec/tdm/tdm_demo/local_infer.py deleted file mode 100644 index aefedc1763..0000000000 --- a/PaddleRec/tdm/tdm_demo/local_infer.py +++ /dev/null @@ -1,119 +0,0 @@ -# -*- coding=utf-8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -from __future__ import print_function -import os -import time -import numpy as np -import logging -import argparse - -import paddle -import paddle.fluid as fluid -from paddle.fluid import profiler - -from args import print_arguments, parse_args -from infer_network import TdmInferNet -from dataset_generator import TDMDataset - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def to_tensor(data, place): - """ - Convert data to paddle tensor - """ - flattened_data = np.concatenate(data, axis=0).astype("float32") - flattened_data = flattened_data.reshape([-1, 768]) - res = fluid.Tensor() - res.set(flattened_data, place) - return res - - -def data2tensor(data, place): - """ - Dataset prepare - """ - input_emb = to_tensor([x[0] for x in data], place) - - return input_emb - - -def run_infer(args, model_path): - """run infer""" - logger.info("Infer Begin") - file_list = [ - str(args.test_files_path) + "/%s" % x - for x in os.listdir(args.test_files_path) - ] - - tdm_model = TdmInferNet(args) - inputs = tdm_model.input_data() - res_item = tdm_model.infer_net(inputs) - test_reader = TDMDataset().infer_reader(file_list, args.batch_size) - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - - path = os.path.join(args.model_files_path, model_path) - fluid.io.load_persistables( - executor=exe, - dirname=path, - main_program=fluid.default_main_program()) - - logger.info("Load persistables from \"{}\"".format(path)) - - if args.save_init_model: - logger.info("Begin Save infer model.") - model_path = (str(args.model_files_path) + "/" + "infer_model") - fluid.io.save_inference_model(executor=exe, dirname=model_path, - feeded_var_names=[ - 'input_emb', 'first_layer_node', 'first_layer_node_mask'], - target_vars=[res_item]) - logger.info("End Save infer model.") - - first_layer_node = tdm_model.first_layer_node - first_layer_nums = len(first_layer_node) - first_layer_node = np.array(first_layer_node) - first_layer_node = first_layer_node.reshape((1, -1)).astype('int64') - first_layer_node = first_layer_node.repeat(args.batch_size, axis=0) - # 在demo中，假设infer起始层的节点都不是叶子节点，mask=0 - # 若真实的起始层含有叶子节点，则对应位置的 mask=1 - first_layer_mask = ( - np.zeros((args.batch_size, first_layer_nums))).astype('int64') - - for batch_id, data in enumerate(test_reader()): - input_emb = data2tensor(data, place) - item_res = exe.run(fluid.default_main_program(), - feed={"input_emb": input_emb, - "first_layer_node": first_layer_node, - "first_layer_node_mask": first_layer_mask}, - fetch_list=[res_item]) - logger.info("TEST --> batch: {} infer_item {}".format( - batch_id, item_res)) - logger.info("Inference complete!") - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = parse_args() - print_arguments(args) - # 在此处指定infer模型所在的文件夹 - path = "epoch_0" - run_infer(args, path) diff --git a/PaddleRec/tdm/tdm_demo/local_train.py b/PaddleRec/tdm/tdm_demo/local_train.py deleted file mode 100644 index 9660a23e26..0000000000 --- a/PaddleRec/tdm/tdm_demo/local_train.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding=utf-8 -*- -""" -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -from __future__ import print_function -import os -import time -import numpy as np -import logging -import argparse - -import paddle -import paddle.fluid as fluid - -from args import print_arguments, parse_args -from utils import tdm_sampler_prepare, tdm_child_prepare -from train_network import TdmTrainNet - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def get_dataset(inputs, args): - """ - get dataset - """ - dataset = fluid.DatasetFactory().create_dataset() - dataset.set_use_var(inputs) - dataset.set_pipe_command("python ./dataset_generator.py") - dataset.set_batch_size(args.batch_size) - dataset.set_thread(int(args.cpu_num)) - file_list = [ - str(args.train_files_path) + "/%s" % x - for x in os.listdir(args.train_files_path) - ] - dataset.set_filelist(file_list) - logger.info("file list: {}".format(file_list)) - return dataset - - -def run_train(args): - """ - run train - """ - logger.info("TDM Begin build network.") - tdm_model = TdmTrainNet(args) - inputs = tdm_model.input_data() - avg_cost, acc = tdm_model.tdm(inputs) - logger.info("TDM End build network.") - - dataset = get_dataset(inputs, args) - - optimizer = fluid.optimizer.AdamOptimizer( - learning_rate=args.learning_rate, - lazy_mode=True) - optimizer.minimize(avg_cost) - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - if args.load_model: - # 从paddle二进制模型加载参数 - path = args.init_model_files_path - fluid.io.load_persistables( - executor=exe, - dirname=path, - main_program=fluid.default_main_program()) - lr = fluid.global_scope().find_var("learning_rate_0").get_tensor() - lr.set(np.array(args.learning_rate).astype('float32'), place) - logger.info("Load persistables from \"{}\"".format(path)) - else: - # 将明文树结构及数据，set到组网中的Variale中 - # 不使用NumpyInitialize方法是考虑到树结构相关数据size过大，有性能风险 - Numpy_model = {} - Numpy_model['TDM_Tree_Travel'] = tdm_model.tdm_param_prepare_dict['travel_array'] - Numpy_model['TDM_Tree_Layer'] = tdm_model.tdm_param_prepare_dict['layer_array'] - Numpy_model['TDM_Tree_Info'] = tdm_model.tdm_param_prepare_dict['info_array'] - Numpy_model['TDM_Tree_Emb'] = tdm_model.tdm_param_prepare_dict['emb_array'] - for param_name in Numpy_model: - param_t = fluid.global_scope().find_var(param_name).get_tensor() - if param_name == 'TDM_Tree_Emb': - param_t.set(Numpy_model[str(param_name) - ].astype('float32'), place) - else: - param_t.set(Numpy_model[str(param_name) - ].astype('int32'), place) - - if args.save_init_model or not args.load_model: - logger.info("Begin Save Init model.") - model_path = os.path.join(args.model_files_path, "init_model") - fluid.io.save_persistables(executor=exe, dirname=model_path) - logger.info("End Save Init model.") - - logger.info("TDM Local training begin ...") - for epoch in range(args.epoch_num): - start_time = time.time() - exe.train_from_dataset( - program=fluid.default_main_program(), - dataset=dataset, - fetch_list=[acc, avg_cost], - fetch_info=["Epoch {} acc".format( - epoch), "Epoch {} loss".format(epoch)], - print_period=1, - debug=False, - ) - end_time = time.time() - logger.info("Epoch %d finished, use time=%d sec\n" % - (epoch, end_time - start_time)) - - model_path = os.path.join(args.model_files_path, "epoch_" + str(epoch)) - fluid.io.save_persistables(executor=exe, dirname=model_path) - - logger.info("Local training success!") - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = parse_args() - print_arguments(args) - run_train(args) diff --git a/PaddleRec/tdm/tdm_demo/predict.py b/PaddleRec/tdm/tdm_demo/predict.py deleted file mode 100644 index 2ec6e522cf..0000000000 --- a/PaddleRec/tdm/tdm_demo/predict.py +++ /dev/null @@ -1,105 +0,0 @@ -# -*- coding: utf-8 -*- -""" -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os -import argparse -import numpy as np -import argparse -import paddle.fluid as fluid -from paddle.fluid.core import PaddleTensor -from paddle.fluid.core import AnalysisConfig -from paddle.fluid.core import create_paddle_predictor -from dataset_generator import TDMDataset -from infer_network import TdmInferNet -from args import print_arguments, parse_args -import logging - -logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def to_tensor(data): - """ - Convert data to paddle tensor - """ - flattened_data = np.concatenate(data, axis=0).astype("float32") - flattened_data = flattened_data.reshape([-1, 768]) - return flattened_data - - -def data2tensor(data): - """ - Dataset prepare - """ - input_emb = to_tensor([x[0] for x in data]) - - return input_emb - - -def tdm_input(input_emb, first_layer_node, first_layer_mask): - """ - Create input of tdm pred - """ - input_emb = PaddleTensor(input_emb) - first_layer_node = PaddleTensor(first_layer_node) - first_layer_mask = PaddleTensor(first_layer_mask) - return [input_emb, first_layer_node, first_layer_mask] - - -def main(): - """Predictor main""" - args = parse_args() - - config = AnalysisConfig(args.model_files_path) - config.disable_gpu() - config.enable_profile() - # config.enable_mkldnn() - config.set_cpu_math_library_num_threads(args.cpu_num) - - predictor = create_paddle_predictor(config) - tdm_model = TdmInferNet(args) - first_layer_node = tdm_model.first_layer_node - first_layer_nums = len(first_layer_node) - first_layer_node = np.array(first_layer_node) - first_layer_node = first_layer_node.reshape((1, -1)).astype('int64') - first_layer_node = first_layer_node.repeat(args.batch_size, axis=0) - first_layer_mask = ( - np.zeros((args.batch_size, first_layer_nums))).astype('int64') - - file_list = [ - str(args.test_files_path) + "/%s" % x - for x in os.listdir(args.test_files_path) - ] - test_reader = TDMDataset().infer_reader(file_list, args.batch_size) - - for batch_id, data in enumerate(test_reader()): - input_emb = data2tensor(data) - - inputs = tdm_input(input_emb, first_layer_node, first_layer_mask) - outputs = predictor.run(inputs) - output = outputs[0] - output_data = output.as_ndarray() - - logger.info("TEST --> batch: {} infer_item {}".format( - batch_id, output_data)) - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - main() diff --git a/PaddleRec/word2vec/infer.py b/PaddleRec/word2vec/infer.py deleted file mode 100644 index 4fcc20fc11..0000000000 --- a/PaddleRec/word2vec/infer.py +++ /dev/null @@ -1,208 +0,0 @@ -import argparse -import sys -import time -import math -import unittest -import contextlib -import numpy as np -import six -import paddle.fluid as fluid -import paddle -import net -import utils -if six.PY2: - reload(sys) - sys.setdefaultencoding('utf-8') - - -def parse_args(): - parser = argparse.ArgumentParser("PaddlePaddle Word2vec infer example") - parser.add_argument( - '--dict_path', - type=str, - default='./data/data_c/1-billion_dict_word_to_id_', - help="The path of dic") - parser.add_argument( - '--infer_epoch', - action='store_true', - required=False, - default=False, - help='infer by epoch') - parser.add_argument( - '--infer_step', - action='store_true', - required=False, - default=False, - help='infer by step') - parser.add_argument( - '--test_dir', type=str, default='test_data', help='test file address') - parser.add_argument( - '--print_step', type=int, default='500000', help='print step') - parser.add_argument( - '--start_index', type=int, default='0', help='start index') - parser.add_argument( - '--start_batch', type=int, default='1', help='start index') - parser.add_argument( - '--end_batch', type=int, default='13', help='start index') - parser.add_argument( - '--last_index', type=int, default='100', help='last index') - parser.add_argument( - '--model_dir', type=str, default='model', help='model dir') - parser.add_argument( - '--use_cuda', type=int, default='0', help='whether use cuda') - parser.add_argument( - '--batch_size', type=int, default='5', help='batch_size') - parser.add_argument('--emb_size', type=int, default='64', help='batch_size') - args = parser.parse_args() - return args - - -def infer_epoch(args, vocab_size, test_reader, use_cuda, i2w): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - emb_size = args.emb_size - batch_size = args.batch_size - with fluid.scope_guard(fluid.Scope()): - main_program = fluid.Program() - with fluid.program_guard(main_program): - values, pred = net.infer_network(vocab_size, emb_size) - for epoch in range(start_index, last_index + 1): - copy_program = main_program.clone() - model_path = model_dir + "/pass-" + str(epoch) - fluid.load(copy_program, model_path, exe) - accum_num = 0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - b_size = len([dat[0] for dat in data]) - wa = np.array([dat[0] for dat in data]).astype( - "int64").reshape(b_size) - wb = np.array([dat[1] for dat in data]).astype( - "int64").reshape(b_size) - wc = np.array([dat[2] for dat in data]).astype( - "int64").reshape(b_size) - - label = [dat[3] for dat in data] - input_word = [dat[4] for dat in data] - para = exe.run(copy_program, - feed={ - "analogy_a": wa, - "analogy_b": wb, - "analogy_c": wc, - "all_label": np.arange(vocab_size) - .reshape(vocab_size).astype("int64"), - }, - fetch_list=[pred.name, values], - return_numpy=False) - pre = np.array(para[0]) - val = np.array(para[1]) - for ii in range(len(label)): - top4 = pre[ii] - accum_num_sum += 1 - for idx in top4: - if int(idx) in input_word[ii]: - continue - if int(idx) == int(label[ii][0]): - accum_num += 1 - break - if step_id % 1 == 0: - print("step:%d %d " % (step_id, accum_num)) - - print("epoch:%d \t acc:%.3f " % - (epoch, 1.0 * accum_num / accum_num_sum)) - - -def infer_step(args, vocab_size, test_reader, use_cuda, i2w): - """ inference function """ - place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() - exe = fluid.Executor(place) - emb_size = args.emb_size - batch_size = args.batch_size - with fluid.scope_guard(fluid.Scope()): - main_program = fluid.Program() - with fluid.program_guard(main_program): - values, pred = net.infer_network(vocab_size, emb_size) - for epoch in range(start_index, last_index + 1): - for batchid in range(args.start_batch, args.end_batch): - copy_program = main_program.clone() - model_path = model_dir + "/pass-" + str(epoch) + ( - '/batch-' + str(batchid * args.print_step)) - fluid.load(copy_program, model_path, exe) - accum_num = 0 - accum_num_sum = 0.0 - t0 = time.time() - step_id = 0 - for data in test_reader(): - step_id += 1 - b_size = len([dat[0] for dat in data]) - wa = np.array([dat[0] for dat in data]).astype( - "int64").reshape(b_size) - wb = np.array([dat[1] for dat in data]).astype( - "int64").reshape(b_size) - wc = np.array([dat[2] for dat in data]).astype( - "int64").reshape(b_size) - - label = [dat[3] for dat in data] - input_word = [dat[4] for dat in data] - para = exe.run( - copy_program, - feed={ - "analogy_a": wa, - "analogy_b": wb, - "analogy_c": wc, - "all_label": - np.arange(vocab_size).reshape(vocab_size), - }, - fetch_list=[pred.name, values], - return_numpy=False) - pre = np.array(para[0]) - val = np.array(para[1]) - for ii in range(len(label)): - top4 = pre[ii] - accum_num_sum += 1 - for idx in top4: - if int(idx) in input_word[ii]: - continue - if int(idx) == int(label[ii][0]): - accum_num += 1 - break - if step_id % 1 == 0: - print("step:%d %d " % (step_id, accum_num)) - print("epoch:%d \t acc:%.3f " % - (epoch, 1.0 * accum_num / accum_num_sum)) - t1 = time.time() - - -if __name__ == "__main__": - import paddle - paddle.enable_static() - utils.check_version() - args = parse_args() - start_index = args.start_index - last_index = args.last_index - test_dir = args.test_dir - model_dir = args.model_dir - batch_size = args.batch_size - dict_path = args.dict_path - use_cuda = True if args.use_cuda else False - print("start index: ", start_index, " last_index:", last_index) - vocab_size, test_reader, id2word = utils.prepare_data( - test_dir, dict_path, batch_size=batch_size) - print("vocab_size:", vocab_size) - if args.infer_step: - infer_step( - args, - vocab_size, - test_reader=test_reader, - use_cuda=use_cuda, - i2w=id2word) - else: - infer_epoch( - args, - vocab_size, - test_reader=test_reader, - use_cuda=use_cuda, - i2w=id2word) diff --git a/PaddleRec/word2vec/train.py b/PaddleRec/word2vec/train.py deleted file mode 100644 index ae630ba25f..0000000000 --- a/PaddleRec/word2vec/train.py +++ /dev/null @@ -1,258 +0,0 @@ -from __future__ import print_function -import argparse -import logging -import os -import time -import math -import random -import numpy as np -import paddle -import paddle.fluid as fluid -import six -import reader -from net import skip_gram_word2vec, skip_gram_word2vec_shuffle_batch - -import utils -import sys -if six.PY2: - reload(sys) - sys.setdefaultencoding('utf-8') - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - - -def parse_args(): - parser = argparse.ArgumentParser( - description="PaddlePaddle Word2vec example") - parser.add_argument( - '--train_data_dir', - type=str, - default='./data/text', - help="The path of taining dataset") - parser.add_argument( - '--base_lr', - type=float, - default=0.01, - help="The number of learing rate (default: 0.01)") - parser.add_argument( - '--save_step', - type=int, - default=500000, - help="The number of step to save (default: 500000)") - parser.add_argument( - '--print_batch', - type=int, - default=10, - help="The number of print_batch (default: 10)") - parser.add_argument( - '--dict_path', - type=str, - default='./data/1-billion_dict', - help="The path of data dict") - parser.add_argument( - '--batch_size', - type=int, - default=500, - help="The size of mini-batch (default:500)") - parser.add_argument( - '--num_passes', - type=int, - default=10, - help="The number of passes to train (default: 10)") - parser.add_argument( - '--model_output_dir', - type=str, - default='models', - help='The path for model to store (default: models)') - parser.add_argument('--nce_num', type=int, default=5, help='nce_num') - parser.add_argument( - '--embedding_size', - type=int, - default=64, - help='sparse feature hashing space for index processing') - parser.add_argument( - '--is_sparse', - action='store_true', - required=False, - default=False, - help='embedding and nce will use sparse or not, (default: False)') - parser.add_argument( - '--with_speed', - action='store_true', - required=False, - default=False, - help='print speed or not , (default: False)') - parser.add_argument( - '--with_shuffle_batch', - action='store_true', - required=False, - default=False, - help='negative samples come from shuffle_batch op or not , (default: False)') - parser.add_argument( - '--enable_ce', action='store_true', help='If set, run the task with continuous evaluation logs.') - - return parser.parse_args() - - -def convert_python_to_tensor(weight, batch_size, sample_reader): - def __reader__(): - cs = np.array(weight).cumsum() - result = [[], []] - for sample in sample_reader(): - for i, fea in enumerate(sample): - result[i].append(fea) - if len(result[0]) == batch_size: - tensor_result = [] - for tensor in result: - t = fluid.Tensor() - dat = np.array(tensor, dtype='int64') - if len(dat.shape) > 2: - dat = dat.reshape((dat.shape[0], dat.shape[2])) - elif len(dat.shape) == 1: - dat = dat.reshape((-1, 1)) - t.set(dat, fluid.CPUPlace()) - tensor_result.append(t) - tt = fluid.Tensor() - neg_array = cs.searchsorted(np.random.sample(args.nce_num)) - neg_array = np.tile(neg_array, batch_size) - tt.set( - neg_array.reshape((batch_size, args.nce_num)), - fluid.CPUPlace()) - tensor_result.append(tt) - yield tensor_result - result = [[], []] - - return __reader__ - - -def train_loop(args, train_program, data_loader, loss, trainer_id): - - place = fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - exec_strategy = fluid.ExecutionStrategy() - exec_strategy.use_experimental_executor = True - - print("CPU_NUM:" + str(os.getenv("CPU_NUM"))) - exec_strategy.num_threads = int(os.getenv("CPU_NUM")) - - build_strategy = fluid.BuildStrategy() - if int(os.getenv("CPU_NUM")) > 1: - build_strategy.reduce_strategy = fluid.BuildStrategy.ReduceStrategy.Reduce - - train_exe = fluid.ParallelExecutor( - use_cuda=False, - loss_name=loss.name, - main_program=train_program, - build_strategy=build_strategy, - exec_strategy=exec_strategy) - - for pass_id in range(args.num_passes): - data_loader.start() - time.sleep(10) - epoch_start = time.time() - batch_id = 0 - start = time.time() - try: - while True: - - loss_val = train_exe.run(fetch_list=[loss.name]) - loss_val = np.mean(loss_val) - - if batch_id % args.print_batch == 0: - logger.info( - "TRAIN --> pass: {} batch: {} loss: {} reader queue:{}". - format(pass_id, batch_id, - loss_val.mean(), data_loader.queue.size())) - if args.with_speed: - if batch_id % 500 == 0 and batch_id != 0: - elapsed = (time.time() - start) - start = time.time() - samples = 1001 * args.batch_size * int( - os.getenv("CPU_NUM")) - logger.info("Time used: {}, Samples/Sec: {}".format( - elapsed, samples / elapsed)) - - if batch_id % args.save_step == 0 and batch_id != 0: - model_dir = args.model_output_dir + '/pass-' + str( - pass_id) + ('/batch-' + str(batch_id)) - if trainer_id == 0: - fluid.save(fluid.default_main_program(), model_path=model_dir) - print("model saved in %s" % model_dir) - batch_id += 1 - - except fluid.core.EOFException: - data_loader.reset() - epoch_end = time.time() - logger.info("Epoch: {0}, Train total expend: {1} ".format( - pass_id, epoch_end - epoch_start)) - model_dir = args.model_output_dir + '/pass-' + str(pass_id) - if trainer_id == 0: - fluid.save(fluid.default_main_program(), model_path=model_dir) - print("model saved in %s" % model_dir) - - -def GetFileList(data_path): - return os.listdir(data_path) - - -def train(args): - # add ce - if args.enable_ce: - SEED = 102 - fluid.default_main_program().random_seed = SEED - fluid.default_startup_program().random_seed = SEED - - if not os.path.isdir(args.model_output_dir): - os.mkdir(args.model_output_dir) - - filelist = GetFileList(args.train_data_dir) - word2vec_reader = reader.Word2VecReader(args.dict_path, args.train_data_dir, - filelist, 0, 1) - - logger.info("dict_size: {}".format(word2vec_reader.dict_size)) - - if args.with_shuffle_batch: - loss, data_loader = skip_gram_word2vec_shuffle_batch( - word2vec_reader.dict_size, - args.embedding_size, - is_sparse=args.is_sparse, - neg_num=args.nce_num) - data_loader.set_sample_generator(word2vec_reader.train(), batch_size=args.batch_size, drop_last=True) - else: - np_power = np.power(np.array(word2vec_reader.id_frequencys), 0.75) - id_frequencys_pow = np_power / np_power.sum() - - loss, data_loader = skip_gram_word2vec( - word2vec_reader.dict_size, - args.embedding_size, - is_sparse=args.is_sparse, - neg_num=args.nce_num) - - data_loader.set_batch_generator( - convert_python_to_tensor(id_frequencys_pow, args.batch_size, word2vec_reader.train())) - - optimizer = fluid.optimizer.SGD( - learning_rate=fluid.layers.exponential_decay( - learning_rate=args.base_lr, - decay_steps=100000, - decay_rate=0.999, - staircase=True)) - - optimizer.minimize(loss) - - # do local training - logger.info("run local training") - main_program = fluid.default_main_program() - train_loop(args, main_program, data_loader, loss, 0) - - -if __name__ == '__main__': - import paddle - paddle.enable_static() - args = parse_args() - utils.check_version(args.with_shuffle_batch) - train(args) diff --git a/PaddleRec/youbube_dnn/infer.py b/PaddleRec/youbube_dnn/infer.py deleted file mode 100644 index 1a8cee925d..0000000000 --- a/PaddleRec/youbube_dnn/infer.py +++ /dev/null @@ -1,55 +0,0 @@ -import paddle.fluid as fluid -import numpy as np -import pandas as pd -import time -import sys -import os -import args -import logging -from youtubednn import YoutubeDNN - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def infer(args): - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - cur_model_path = os.path.join(args.model_dir, 'epoch_' + str(args.test_epoch), "checkpoint") - - with fluid.scope_guard(fluid.Scope()): - infer_program, feed_target_names, fetch_vars = fluid.io.load_inference_model(cur_model_path, exe) - # Build a random data set. - sample_size = 100 - watch_vecs = [] - search_vecs = [] - other_feats = [] - - for i in range(sample_size): - watch_vec = np.random.rand(1, args.watch_vec_size) - search_vec = np.random.rand(1, args.search_vec_size) - other_feat = np.random.rand(1, args.other_feat_size) - watch_vecs.append(watch_vec) - search_vecs.append(search_vec) - other_feats.append(other_feat) - - for i in range(sample_size): - l3 = exe.run(infer_program, - feed={ - "watch_vec": watch_vecs[i].astype('float32'), - "search_vec": search_vecs[i].astype('float32'), - "other_feat": other_feats[i].astype('float32'), - }, - return_numpy=True, - fetch_list=fetch_vars) - - user_vec = pd.DataFrame(l3[0]) - user_vec.to_csv(args.user_vec_path, mode="a", index=False, header=0) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - if(os.path.exists(args.user_vec_path)): - os.system("rm " + args.user_vec_path) - infer(args) \ No newline at end of file diff --git a/PaddleRec/youbube_dnn/train.py b/PaddleRec/youbube_dnn/train.py deleted file mode 100644 index 446c9552af..0000000000 --- a/PaddleRec/youbube_dnn/train.py +++ /dev/null @@ -1,77 +0,0 @@ -import numpy as np -import pandas as pd -import os -import random -import paddle.fluid as fluid -from youtubednn import YoutubeDNN -import paddle -import args -import logging -import time - -logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s') -logger = logging.getLogger("fluid") -logger.setLevel(logging.INFO) - -def train(args): - youtube_model = YoutubeDNN() - inputs = youtube_model.input_data(args.watch_vec_size, args.search_vec_size, args.other_feat_size) - loss, acc, l3 = youtube_model.net(inputs, args.output_size, layers=[128, 64, 32]) - - sgd = fluid.optimizer.SGD(learning_rate=args.base_lr) - sgd.minimize(loss) - - place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace() - exe = fluid.Executor(place) - exe.run(fluid.default_startup_program()) - - # Build a random data set. - sample_size = 100 - watch_vecs = [] - search_vecs = [] - other_feats = [] - labels = [] - - for i in range(sample_size): - watch_vec = np.random.rand(args.batch_size, args.watch_vec_size) - search_vec = np.random.rand(args.batch_size, args.search_vec_size) - other_feat = np.random.rand(args.batch_size, args.other_feat_size) - watch_vecs.append(watch_vec) - search_vecs.append(search_vec) - other_feats.append(other_feat) - label = np.random.randint(args.output_size, size=(args.batch_size, 1)) - labels.append(label) - for epoch in range(args.epochs): - for i in range(sample_size): - begin = time.time() - loss_data, acc_val = exe.run(fluid.default_main_program(), - feed={ - "watch_vec": watch_vecs[i].astype('float32'), - "search_vec": search_vecs[i].astype('float32'), - "other_feat": other_feats[i].astype('float32'), - "label": np.array(labels[i]).reshape(args.batch_size, 1) - }, - return_numpy=True, - fetch_list=[loss.name, acc.name]) - end = time.time() - logger.info("epoch_id: {}, batch_time: {:.5f}s, loss: {:.5f}, acc: {:.5f}".format( - epoch, end-begin, float(np.array(loss_data)), np.array(acc_val)[0])) - #save model - model_dir = os.path.join(args.model_dir, 'epoch_' + str(epoch + 1), "checkpoint") - - feed_var_names = ["watch_vec", "search_vec", "other_feat"] - fetch_vars = [l3] - fluid.io.save_inference_model(model_dir, feed_var_names, fetch_vars, exe) - - #save all video vector - video_array = np.array(fluid.global_scope().find_var('l4_weight').get_tensor()) - video_vec = pd.DataFrame(video_array) - video_vec.to_csv(args.video_vec_path, mode="a", index=False, header=0) - -if __name__ == "__main__": - import paddle - paddle.enable_static() - args = args.parse_args() - if(os.path.exists(args.video_vec_path)): - os.system("rm " + args.video_vec_path) - train(args)