convert_new_weight_with_kv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import struct\n",
    "import numpy as np\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "def load_dtype(fp):\n",
    "    v = struct.unpack(\"B\", fp.read(1))[0]\n",
    "    if v == 0:\n",
    "        return np.int8\n",
    "    elif v == 1:\n",
    "        return np.float16\n",
    "    elif v == 2:\n",
    "        return np.float32\n",
    "    else:\n",
    "        raise TypeError(\"Unknown dtype %d\" % v)\n",
    "\n",
    "def load_string(fp):\n",
    "    size = struct.unpack(\"I\", fp.read(4))[0]\n",
    "    v = fp.read(size)\n",
    "    return v.decode(\"utf-8\")\n",
    "\n",
    "def load_tuple(fp):\n",
    "    dim_tuple = struct.unpack(\"B\", fp.read(1))[0]\n",
    "    ret = []\n",
    "    for _ in range(dim_tuple):\n",
    "        ret.append(struct.unpack(\"I\", fp.read(4))[0]) \n",
    "    return tuple(ret)\n",
    "\n",
    "def load_parameter(fp):    \n",
    "    shape = load_tuple(fp)\n",
    "    value_size = struct.unpack(\"I\", fp.read(4))[0]\n",
    "    dtype = load_dtype(fp)\n",
    "    value = fp.read(value_size)\n",
    "    return shape, value, dtype\n",
    "\n",
    "def load(fp, parent_name=''):\n",
    "    num_parameters, num_sub_layers = struct.unpack(\"II\", fp.read(8))\n",
    "    parameters = []\n",
    "\n",
    "    for _ in range(num_parameters):\n",
    "        name = load_string(fp)\n",
    "        shape, value, dtype = load_parameter(fp)\n",
    "        parameters.append((parent_name + '.' + name, np.frombuffer(value, dtype).reshape(shape)))\n",
    "    for _ in range(num_sub_layers):\n",
    "        name = load_string(fp)\n",
    "        parameters += load(fp, parent_name + '.' + name)\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../zhiyuan/cpm/checkpoint.pt', 'rb') as fp:\n",
    "    parameters = load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pindex = {x[0]: x[1] for x in parameters}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 516/516 [00:45<00:00, 11.39it/s]\n"
     ]
    }
   ],
   "source": [
    "npara = {}\n",
    "for name, value in tqdm(parameters):\n",
    "    if '_scale' not in name:\n",
    "        has_scale = name + '_scale'\n",
    "        if has_scale in pindex:\n",
    "            scale = pindex[has_scale]\n",
    "            value = value.astype(np.float16) * scale\n",
    "        npara[name] = value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".input_embedding.weight (30000, 2560)\n",
      ".position_embedding.weight (1024, 2560)\n",
      ".layers.0.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.0.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.0.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.0.self_attention.w_project_bias (3, 2560)\n",
      ".layers.0.self_attention.w_out (2560, 2560)\n",
      ".layers.0.self_attention.w_out_bias (2560,)\n",
      ".layers.0.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.0.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.0.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.0.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.0.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.0.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.1.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.1.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.1.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.1.self_attention.w_project_bias (3, 2560)\n",
      ".layers.1.self_attention.w_out (2560, 2560)\n",
      ".layers.1.self_attention.w_out_bias (2560,)\n",
      ".layers.1.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.1.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.1.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.1.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.1.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.1.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.2.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.2.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.2.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.2.self_attention.w_project_bias (3, 2560)\n",
      ".layers.2.self_attention.w_out (2560, 2560)\n",
      ".layers.2.self_attention.w_out_bias (2560,)\n",
      ".layers.2.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.2.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.2.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.2.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.2.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.2.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.3.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.3.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.3.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.3.self_attention.w_project_bias (3, 2560)\n",
      ".layers.3.self_attention.w_out (2560, 2560)\n",
      ".layers.3.self_attention.w_out_bias (2560,)\n",
      ".layers.3.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.3.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.3.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.3.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.3.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.3.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.4.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.4.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.4.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.4.self_attention.w_project_bias (3, 2560)\n",
      ".layers.4.self_attention.w_out (2560, 2560)\n",
      ".layers.4.self_attention.w_out_bias (2560,)\n",
      ".layers.4.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.4.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.4.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.4.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.4.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.4.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.5.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.5.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.5.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.5.self_attention.w_project_bias (3, 2560)\n",
      ".layers.5.self_attention.w_out (2560, 2560)\n",
      ".layers.5.self_attention.w_out_bias (2560,)\n",
      ".layers.5.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.5.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.5.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.5.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.5.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.5.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.6.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.6.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.6.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.6.self_attention.w_project_bias (3, 2560)\n",
      ".layers.6.self_attention.w_out (2560, 2560)\n",
      ".layers.6.self_attention.w_out_bias (2560,)\n",
      ".layers.6.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.6.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.6.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.6.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.6.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.6.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.7.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.7.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.7.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.7.self_attention.w_project_bias (3, 2560)\n",
      ".layers.7.self_attention.w_out (2560, 2560)\n",
      ".layers.7.self_attention.w_out_bias (2560,)\n",
      ".layers.7.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.7.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.7.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.7.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.7.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.7.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.8.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.8.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.8.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.8.self_attention.w_project_bias (3, 2560)\n",
      ".layers.8.self_attention.w_out (2560, 2560)\n",
      ".layers.8.self_attention.w_out_bias (2560,)\n",
      ".layers.8.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.8.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.8.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.8.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.8.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.8.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.9.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.9.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.9.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.9.self_attention.w_project_bias (3, 2560)\n",
      ".layers.9.self_attention.w_out (2560, 2560)\n",
      ".layers.9.self_attention.w_out_bias (2560,)\n",
      ".layers.9.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.9.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.9.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.9.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.9.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.9.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.10.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.10.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.10.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.10.self_attention.w_project_bias (3, 2560)\n",
      ".layers.10.self_attention.w_out (2560, 2560)\n",
      ".layers.10.self_attention.w_out_bias (2560,)\n",
      ".layers.10.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.10.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.10.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.10.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.10.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.10.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.11.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.11.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.11.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.11.self_attention.w_project_bias (3, 2560)\n",
      ".layers.11.self_attention.w_out (2560, 2560)\n",
      ".layers.11.self_attention.w_out_bias (2560,)\n",
      ".layers.11.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.11.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.11.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.11.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.11.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.11.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.12.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.12.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.12.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.12.self_attention.w_project_bias (3, 2560)\n",
      ".layers.12.self_attention.w_out (2560, 2560)\n",
      ".layers.12.self_attention.w_out_bias (2560,)\n",
      ".layers.12.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.12.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.12.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.12.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.12.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.12.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.13.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.13.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.13.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.13.self_attention.w_project_bias (3, 2560)\n",
      ".layers.13.self_attention.w_out (2560, 2560)\n",
      ".layers.13.self_attention.w_out_bias (2560,)\n",
      ".layers.13.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.13.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.13.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.13.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.13.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.13.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.14.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.14.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.14.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.14.self_attention.w_project_bias (3, 2560)\n",
      ".layers.14.self_attention.w_out (2560, 2560)\n",
      ".layers.14.self_attention.w_out_bias (2560,)\n",
      ".layers.14.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.14.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.14.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.14.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.14.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.14.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.15.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.15.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.15.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.15.self_attention.w_project_bias (3, 2560)\n",
      ".layers.15.self_attention.w_out (2560, 2560)\n",
      ".layers.15.self_attention.w_out_bias (2560,)\n",
      ".layers.15.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.15.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.15.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.15.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.15.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.15.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.16.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.16.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.16.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.16.self_attention.w_project_bias (3, 2560)\n",
      ".layers.16.self_attention.w_out (2560, 2560)\n",
      ".layers.16.self_attention.w_out_bias (2560,)\n",
      ".layers.16.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.16.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.16.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.16.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.16.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.16.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.17.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.17.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.17.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.17.self_attention.w_project_bias (3, 2560)\n",
      ".layers.17.self_attention.w_out (2560, 2560)\n",
      ".layers.17.self_attention.w_out_bias (2560,)\n",
      ".layers.17.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.17.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.17.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.17.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.17.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.17.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.18.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.18.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.18.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.18.self_attention.w_project_bias (3, 2560)\n",
      ".layers.18.self_attention.w_out (2560, 2560)\n",
      ".layers.18.self_attention.w_out_bias (2560,)\n",
      ".layers.18.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.18.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.18.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.18.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.18.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.18.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.19.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.19.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.19.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.19.self_attention.w_project_bias (3, 2560)\n",
      ".layers.19.self_attention.w_out (2560, 2560)\n",
      ".layers.19.self_attention.w_out_bias (2560,)\n",
      ".layers.19.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.19.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.19.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.19.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.19.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.19.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.20.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.20.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.20.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.20.self_attention.w_project_bias (3, 2560)\n",
      ".layers.20.self_attention.w_out (2560, 2560)\n",
      ".layers.20.self_attention.w_out_bias (2560,)\n",
      ".layers.20.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.20.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.20.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.20.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.20.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.20.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.21.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.21.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.21.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.21.self_attention.w_project_bias (3, 2560)\n",
      ".layers.21.self_attention.w_out (2560, 2560)\n",
      ".layers.21.self_attention.w_out_bias (2560,)\n",
      ".layers.21.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.21.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.21.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.21.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.21.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.21.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.22.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.22.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.22.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.22.self_attention.w_project_bias (3, 2560)\n",
      ".layers.22.self_attention.w_out (2560, 2560)\n",
      ".layers.22.self_attention.w_out_bias (2560,)\n",
      ".layers.22.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.22.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.22.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.22.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.22.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.22.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.23.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.23.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.23.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.23.self_attention.w_project_bias (3, 2560)\n",
      ".layers.23.self_attention.w_out (2560, 2560)\n",
      ".layers.23.self_attention.w_out_bias (2560,)\n",
      ".layers.23.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.23.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.23.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.23.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.23.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.23.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.24.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.24.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.24.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.24.self_attention.w_project_bias (3, 2560)\n",
      ".layers.24.self_attention.w_out (2560, 2560)\n",
      ".layers.24.self_attention.w_out_bias (2560,)\n",
      ".layers.24.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.24.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.24.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.24.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.24.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.24.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.25.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.25.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.25.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.25.self_attention.w_project_bias (3, 2560)\n",
      ".layers.25.self_attention.w_out (2560, 2560)\n",
      ".layers.25.self_attention.w_out_bias (2560,)\n",
      ".layers.25.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.25.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.25.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.25.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.25.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.25.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.26.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.26.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.26.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.26.self_attention.w_project_bias (3, 2560)\n",
      ".layers.26.self_attention.w_out (2560, 2560)\n",
      ".layers.26.self_attention.w_out_bias (2560,)\n",
      ".layers.26.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.26.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.26.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.26.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.26.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.26.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.27.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.27.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.27.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.27.self_attention.w_project_bias (3, 2560)\n",
      ".layers.27.self_attention.w_out (2560, 2560)\n",
      ".layers.27.self_attention.w_out_bias (2560,)\n",
      ".layers.27.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.27.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.27.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.27.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.27.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.27.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.28.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.28.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.28.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.28.self_attention.w_project_bias (3, 2560)\n",
      ".layers.28.self_attention.w_out (2560, 2560)\n",
      ".layers.28.self_attention.w_out_bias (2560,)\n",
      ".layers.28.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.28.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.28.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.28.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.28.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.28.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.29.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.29.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.29.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.29.self_attention.w_project_bias (3, 2560)\n",
      ".layers.29.self_attention.w_out (2560, 2560)\n",
      ".layers.29.self_attention.w_out_bias (2560,)\n",
      ".layers.29.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.29.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.29.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.29.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.29.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.29.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.30.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.30.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.30.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.30.self_attention.w_project_bias (3, 2560)\n",
      ".layers.30.self_attention.w_out (2560, 2560)\n",
      ".layers.30.self_attention.w_out_bias (2560,)\n",
      ".layers.30.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.30.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.30.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.30.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.30.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.30.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".layers.31.layer_nrom_before_self_attn.weight (2560,)\n",
      ".layers.31.layer_nrom_before_self_attn.bias (2560,)\n",
      ".layers.31.self_attention.w_project_qkv (3, 2560, 2560)\n",
      ".layers.31.self_attention.w_project_bias (3, 2560)\n",
      ".layers.31.self_attention.w_out (2560, 2560)\n",
      ".layers.31.self_attention.w_out_bias (2560,)\n",
      ".layers.31.layer_nrom_before_ff.weight (2560,)\n",
      ".layers.31.layer_nrom_before_ff.bias (2560,)\n",
      ".layers.31.dense_gelu_dense.wi.weight (10240, 2560)\n",
      ".layers.31.dense_gelu_dense.wi.weight_bias (10240,)\n",
      ".layers.31.dense_gelu_dense.wo.weight (2560, 10240)\n",
      ".layers.31.dense_gelu_dense.wo.weight_bias (2560,)\n",
      ".encoder_final_layer_nrom.weight (2560,)\n",
      ".encoder_final_layer_nrom.bias (2560,)\n"
     ]
    }
   ],
   "source": [
    "for k, v in npara.items():\n",
    "    print(k, v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_layer(n):\n",
    "    rets = {}\n",
    "    for k, v in npara.items():\n",
    "        if f'.layers.{n}.' in k:\n",
    "            rets[k] = v\n",
    "    weights = [\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_project_qkv'][0]),\n",
    "        rets[f'.layers.{n}.self_attention.w_project_bias'][0],\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_project_qkv'][1]),\n",
    "        rets[f'.layers.{n}.self_attention.w_project_bias'][1],\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_project_qkv'][2]),\n",
    "        rets[f'.layers.{n}.self_attention.w_project_bias'][2],\n",
    "        np.transpose(rets[f'.layers.{n}.self_attention.w_out']),\n",
    "        rets[f'.layers.{n}.self_attention.w_out_bias'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_self_attn.weight'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_self_attn.bias'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_ff.weight'],\n",
    "        rets[f'.layers.{n}.layer_nrom_before_ff.bias'],\n",
    "        np.transpose(rets[f'.layers.{n}.dense_gelu_dense.wi.weight']),\n",
    "        rets[f'.layers.{n}.dense_gelu_dense.wi.weight_bias'],\n",
    "        np.transpose(rets[f'.layers.{n}.dense_gelu_dense.wo.weight']),\n",
    "        rets[f'.layers.{n}.dense_gelu_dense.wo.weight_bias'],\n",
    "    ]\n",
    "    \n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = [\n",
    "    npara['.input_embedding.weight'],\n",
    "    npara['.position_embedding.weight'],\n",
    "]\n",
    "for i in range(32):\n",
    "    weights += get_layer(i)\n",
    "weights += [\n",
    "    npara['.encoder_final_layer_nrom.weight'],\n",
    "    npara['.encoder_final_layer_nrom.bias'],\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "516"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "from tf2gpt.model import GPT\n",
    "\n",
    "from gpt2_tokenizer import GPT2Tokenizer\n",
    "cbpe = GPT2Tokenizer(\n",
    "    'CPM-Generate/bpe_3w_new/vocab.json',\n",
    "    'CPM-Generate/bpe_3w_new/merges.txt',\n",
    "    model_file='CPM-Generate/bpe_3w_new/chinese_vocab.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ids = cbpe.encode('今天天气不错')\n",
    "\n",
    "# kv_cache = None\n",
    "# for i in range(20):\n",
    "#     output = gpt(tf.constant([ids]), kv_cache)\n",
    "#     output, new_kv_cache = output\n",
    "#     if kv_cache is None:\n",
    "#         kv_cache = new_kv_cache\n",
    "#     else:\n",
    "#         kv_cache = tf.concat([\n",
    "#             kv_cache,\n",
    "#             new_kv_cache\n",
    "#         ], axis=-2)\n",
    "    \n",
    "#     nid = np.argmax(output[0, -1])\n",
    "#     ids = [nid]\n",
    "#     print(i, cbpe.decode(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# ids = cbpe.encode('今天天气不错')\n",
    "\n",
    "# for i in range(20):\n",
    "#     output = gpt(tf.constant([ids]))[0]\n",
    "\n",
    "#     nid = np.argmax(output[0, -1])\n",
    "#     ids += [nid]\n",
    "#     print(i, cbpe.decode(ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as attention_layer_call_fn, attention_layer_call_and_return_conditional_losses, LayerNorm_mlp_ln0_layer_call_fn, LayerNorm_mlp_ln0_layer_call_and_return_conditional_losses, LayerNorm_mlp_ln1_layer_call_fn while saving (showing 5 of 2080). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: cpm_saved_model/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: cpm_saved_model/assets\n"
     ]
    }
   ],
   "source": [
    "gpt = GPT(\n",
    "    vocab_size=30_000,\n",
    "    layer_size=32,\n",
    "    block_size=1024,\n",
    "    embedding_dropout=0.0,\n",
    "    embedding_size=2560,\n",
    "    num_attention_heads=32,\n",
    "    attention_dropout=0.0,\n",
    "    residual_dropout=0.0,\n",
    "    use_cache=True\n",
    ")\n",
    "gpt._set_inputs(\n",
    "    tf.TensorSpec(shape=[None, None],\n",
    "        dtype=tf.int64,\n",
    "        name=\"input_ids\"\n",
    "    ))\n",
    "out = gpt(tf.constant([[1]]))\n",
    "gpt.set_weights(weights)\n",
    "gpt.save('cpm_saved_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(32, 1, 2, 32, 1, 80), dtype=float32, numpy=\n",
       "array([[[[[[ 3.22329879e-01,  4.90205526e-01,  3.10286999e-01, ...,\n",
       "             3.39013398e-01,  2.28684947e-01, -6.23727083e-01]],\n",
       "\n",
       "          [[ 1.67696393e+00,  8.82031769e-03, -5.94259560e-01, ...,\n",
       "             6.17827654e-01,  7.00745225e-01,  8.16019654e-01]],\n",
       "\n",
       "          [[-1.44008726e-01,  1.03477716e+00,  6.53821044e-03, ...,\n",
       "            -9.18062627e-01, -5.55618048e-01, -1.20327711e+00]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[ 5.26708841e-01, -1.57522810e+00,  1.21947922e-01, ...,\n",
       "            -6.22768939e-01,  1.04231811e+00, -1.65548638e-01]],\n",
       "\n",
       "          [[ 8.74828935e-01,  1.49547327e+00, -7.50782013e-01, ...,\n",
       "             2.77377510e+00,  1.05143869e+00, -1.11437336e-01]],\n",
       "\n",
       "          [[-2.21316171e+00, -7.64421582e-01,  8.60675752e-01, ...,\n",
       "             8.10929596e-01,  2.09386677e-01,  1.83064258e+00]]],\n",
       "\n",
       "\n",
       "         [[[ 9.31456268e-01,  1.34661651e+00,  1.99885219e-01, ...,\n",
       "            -7.19597816e-01, -2.24874735e+00,  1.34290040e-01]],\n",
       "\n",
       "          [[-8.75719041e-02, -1.67915642e-01,  3.37845266e-01, ...,\n",
       "             1.06191826e+00, -2.21919632e+00,  8.60908449e-01]],\n",
       "\n",
       "          [[ 5.78081071e-01,  1.01041412e+00,  1.36565685e+00, ...,\n",
       "            -3.25912356e-01, -7.39134669e-01,  5.11757314e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-2.05350295e-03,  2.14169815e-01, -1.01498938e+00, ...,\n",
       "             2.07951307e+00,  1.72649634e+00,  8.94717872e-03]],\n",
       "\n",
       "          [[ 3.22518229e-01,  2.28943610e+00,  5.43536961e-01, ...,\n",
       "            -9.87891033e-02,  2.02799782e-01,  7.56027997e-01]],\n",
       "\n",
       "          [[-1.22968388e+00,  6.64164960e-01, -3.32443327e-01, ...,\n",
       "            -4.04198408e-01,  6.21706009e-01, -1.00847185e+00]]]]],\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "       [[[[[ 1.73820770e+00,  1.50632337e-01,  7.35397339e-02, ...,\n",
       "            -1.54178232e-01,  1.92152709e-03, -1.49210423e-01]],\n",
       "\n",
       "          [[-3.68342668e-01,  1.64720893e-01, -2.67111659e+00, ...,\n",
       "            -2.65741587e-01, -1.28640544e+00,  1.01999659e-02]],\n",
       "\n",
       "          [[-1.63103923e-01,  7.83875763e-01, -2.75351048e-01, ...,\n",
       "             5.24110496e-01,  6.23714328e-01, -1.40276837e+00]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-8.83650184e-01,  6.60457850e-01, -2.26725531e+00, ...,\n",
       "             1.57002616e+00,  2.05882546e-02, -5.09074152e-01]],\n",
       "\n",
       "          [[-1.18495870e+00, -4.70875680e-01, -1.73016563e-02, ...,\n",
       "             3.25473666e-01,  3.11374843e-01, -3.59579772e-01]],\n",
       "\n",
       "          [[-4.14780937e-02,  2.19555068e+00,  7.18660593e-01, ...,\n",
       "            -8.22637752e-02, -2.53793001e-01,  2.12567121e-01]]],\n",
       "\n",
       "\n",
       "         [[[-4.70792264e-01,  1.29994065e-01, -2.21419144e+00, ...,\n",
       "            -1.99012649e+00, -1.89947233e-01,  1.88376892e+00]],\n",
       "\n",
       "          [[-1.07669389e+00,  7.52805173e-01, -5.92212260e-01, ...,\n",
       "             1.73298550e+00, -2.45536625e-01, -6.19674742e-01]],\n",
       "\n",
       "          [[-4.60734636e-01, -7.14826584e-01,  8.62953901e-01, ...,\n",
       "             2.54058868e-01, -3.82272363e-01, -1.27255797e+00]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-1.54729462e+00,  2.17697114e-01,  1.60115197e-01, ...,\n",
       "            -1.11998677e+00,  3.06574792e-01, -1.04165900e+00]],\n",
       "\n",
       "          [[ 9.96252656e-01, -3.16010863e-02, -1.51933646e+00, ...,\n",
       "             6.66637063e-01,  7.11074829e-01,  8.24389875e-01]],\n",
       "\n",
       "          [[-1.22088909e+00, -2.16930971e-01, -1.16537064e-02, ...,\n",
       "            -1.13024306e+00, -1.35767698e+00, -1.17327476e+00]]]]],\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "       [[[[[ 9.13210511e-01, -1.07037619e-01, -1.25519097e-01, ...,\n",
       "             5.15714824e-01,  1.17628443e+00, -2.00322121e-01]],\n",
       "\n",
       "          [[-8.50008607e-01, -1.74955380e+00, -3.73585492e-01, ...,\n",
       "             9.27161098e-01,  1.59789693e+00,  5.39352536e-01]],\n",
       "\n",
       "          [[ 3.14684838e-01,  5.89320183e-01,  2.12690473e+00, ...,\n",
       "             3.15057456e-01,  2.41737098e-01, -3.60853106e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-3.59122962e-01, -5.64107835e-01, -1.61498845e+00, ...,\n",
       "             6.89835608e-01, -5.21410465e-01, -1.44597745e+00]],\n",
       "\n",
       "          [[-1.45254031e-01,  4.56845343e-01, -8.19837570e-01, ...,\n",
       "             2.44016543e-01, -6.26398206e-01,  6.06826425e-01]],\n",
       "\n",
       "          [[ 1.14830089e+00, -1.14792204e+00, -3.85685354e-01, ...,\n",
       "            -2.33561337e-01,  1.00946856e+00,  1.09518933e+00]]],\n",
       "\n",
       "\n",
       "         [[[ 6.15083158e-01, -9.36657012e-01,  7.77256608e-01, ...,\n",
       "             3.40014137e-03, -1.03799653e+00, -7.50586331e-01]],\n",
       "\n",
       "          [[ 2.51169533e-01, -2.94223845e-01,  2.29602844e-01, ...,\n",
       "            -5.71109712e-01, -4.34508234e-01,  1.85630822e+00]],\n",
       "\n",
       "          [[ 5.99698961e-01,  8.24246407e-01,  1.01800168e+00, ...,\n",
       "             3.19733173e-01, -1.39687455e+00,  4.06162173e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[ 6.45173788e-01,  1.30736518e+00, -7.36819446e-01, ...,\n",
       "             9.01237190e-01,  1.53807357e-01,  7.69136548e-01]],\n",
       "\n",
       "          [[ 7.33903944e-01,  1.99823117e+00, -3.35341632e-01, ...,\n",
       "             9.73800540e-01, -1.44853306e+00, -4.63856608e-01]],\n",
       "\n",
       "          [[-1.80993223e+00,  1.81272224e-01, -9.86257792e-01, ...,\n",
       "            -2.56233700e-02,  1.36052346e+00, -9.40330446e-01]]]]],\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "       ...,\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "       [[[[[-8.24323744e-02, -7.45639443e-01,  1.04987514e+00, ...,\n",
       "             7.04677045e-01, -1.12998694e-01,  3.50479335e-01]],\n",
       "\n",
       "          [[ 7.62556270e-02, -1.81006861e+00, -6.27372384e-01, ...,\n",
       "             1.26786351e+00,  4.10329282e-01,  1.36261404e+00]],\n",
       "\n",
       "          [[-1.05238235e+00,  8.11158866e-02, -1.51939237e+00, ...,\n",
       "            -3.91674459e-01, -2.34639144e+00,  6.62017286e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-2.85234332e-01,  8.94392848e-01,  2.00599432e-02, ...,\n",
       "             3.87495905e-01, -6.52880669e-01,  7.09899962e-01]],\n",
       "\n",
       "          [[-1.14350565e-01, -1.05799630e-01,  5.55256128e-01, ...,\n",
       "             1.77494049e-01,  1.34487748e+00,  2.91698426e-01]],\n",
       "\n",
       "          [[ 8.62566233e-01,  3.22201885e-02, -3.44924808e-01, ...,\n",
       "            -1.04101157e+00, -6.35591030e-01, -3.44720274e-01]]],\n",
       "\n",
       "\n",
       "         [[[-1.11729562e+00,  2.09963843e-02, -4.77428347e-01, ...,\n",
       "             1.16444421e+00,  1.04261589e+00,  1.04918748e-01]],\n",
       "\n",
       "          [[ 2.12219730e-01,  1.05561569e-01,  5.48663020e-01, ...,\n",
       "            -3.49172622e-01,  1.65565372e-01,  2.86336184e-01]],\n",
       "\n",
       "          [[-6.90511614e-02, -5.48548400e-01,  1.17509916e-01, ...,\n",
       "            -1.54736006e+00, -1.41904962e+00,  1.70765483e+00]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-1.55198157e-01, -2.70058274e+00,  6.15349293e-01, ...,\n",
       "             1.68149221e+00,  1.04507232e+00,  1.53326035e+00]],\n",
       "\n",
       "          [[ 9.41792428e-01,  7.58580208e-01, -3.86097357e-02, ...,\n",
       "            -5.34276903e-01,  7.64408469e-01,  4.13730264e-01]],\n",
       "\n",
       "          [[-1.86356258e+00, -8.94777477e-01, -1.66904330e+00, ...,\n",
       "             7.95985520e-01,  4.23207015e-01,  1.71827972e+00]]]]],\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "       [[[[[-7.66342878e-01,  1.02164388e+00, -6.66726947e-01, ...,\n",
       "            -4.34976757e-01, -1.63264304e-01, -2.77321994e-01]],\n",
       "\n",
       "          [[ 2.47023535e+00, -1.08986747e+00, -6.85690224e-01, ...,\n",
       "             5.76272070e-01,  2.21029305e+00, -1.63800502e+00]],\n",
       "\n",
       "          [[-2.93470651e-01, -2.24589825e-01,  8.02205145e-01, ...,\n",
       "            -9.53749180e-01,  1.46018684e-01,  5.83231390e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-1.37322104e+00,  1.36866403e+00,  6.20977998e-01, ...,\n",
       "             2.10617089e+00, -4.06487942e-01, -8.70501280e-01]],\n",
       "\n",
       "          [[ 2.35766470e-01, -1.11202158e-01, -1.56978464e+00, ...,\n",
       "             7.62885392e-01,  5.54216921e-01,  4.19806153e-01]],\n",
       "\n",
       "          [[-6.26038194e-01, -2.63530195e-01, -3.03866327e-01, ...,\n",
       "            -6.82293832e-01,  4.73313183e-02, -1.61877096e-01]]],\n",
       "\n",
       "\n",
       "         [[[ 6.75016344e-01, -8.29556406e-01,  8.69932055e-01, ...,\n",
       "             1.07980335e+00,  1.81871027e-01, -7.84478545e-01]],\n",
       "\n",
       "          [[-6.81311190e-02,  4.96122092e-02, -8.13903630e-01, ...,\n",
       "            -2.12371752e-01, -4.35017467e-01, -2.10506737e-01]],\n",
       "\n",
       "          [[-8.33067000e-01,  1.88613188e+00, -4.27313775e-01, ...,\n",
       "            -1.00887728e+00, -4.89156634e-01, -5.75050712e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[ 8.12679529e-01, -1.34024620e+00, -3.14018071e-01, ...,\n",
       "            -7.91911542e-01,  6.56977296e-01, -1.02370751e+00]],\n",
       "\n",
       "          [[ 9.82600093e-01,  1.32899845e+00,  8.11604708e-02, ...,\n",
       "            -5.39835691e-01,  1.09849155e+00,  9.17476892e-01]],\n",
       "\n",
       "          [[-6.47296429e-01, -6.79838717e-01,  1.12815760e-03, ...,\n",
       "            -6.46757782e-01,  4.21619356e-01,  7.40314722e-02]]]]],\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "       [[[[[-2.43989182e+00,  8.85468245e-01, -2.10955083e-01, ...,\n",
       "             5.59785962e-01, -1.60738790e+00,  7.88756013e-02]],\n",
       "\n",
       "          [[ 2.73991048e-01, -2.04174352e+00, -7.99755573e-01, ...,\n",
       "            -1.00569499e+00,  1.39195228e+00, -9.10991371e-01]],\n",
       "\n",
       "          [[ 1.79972386e+00,  9.20113206e-01, -1.23288786e+00, ...,\n",
       "             3.37761581e-01,  2.81929135e-01, -1.41302359e+00]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[ 8.83657157e-01, -3.86107892e-01, -3.95598382e-01, ...,\n",
       "             1.73809600e+00,  7.09435940e-01,  1.23070896e+00]],\n",
       "\n",
       "          [[-1.69519830e+00, -4.65559393e-01, -1.86477393e-01, ...,\n",
       "            -2.46793211e-01, -5.52916169e-01,  5.73561192e-01]],\n",
       "\n",
       "          [[-7.61439681e-01,  9.18184519e-01, -1.38135600e+00, ...,\n",
       "             6.40236437e-01, -1.30649436e+00, -1.97641104e-02]]],\n",
       "\n",
       "\n",
       "         [[[-4.97602373e-01,  1.75028145e-01,  1.06478207e-01, ...,\n",
       "            -5.34823716e-01,  5.92995286e-01,  1.08287942e+00]],\n",
       "\n",
       "          [[-2.55429983e-01, -1.12559319e-01,  4.99477148e-01, ...,\n",
       "             1.40087247e+00, -2.36363009e-01, -1.05499372e-01]],\n",
       "\n",
       "          [[ 5.36373496e-01,  2.07094699e-01, -3.97235423e-01, ...,\n",
       "            -1.02363691e-01, -7.84065008e-01, -6.39425039e-01]],\n",
       "\n",
       "          ...,\n",
       "\n",
       "          [[-1.29592061e+00, -6.17428422e-01, -5.20015121e-01, ...,\n",
       "             5.80753624e-01, -2.86628294e+00,  1.42611754e+00]],\n",
       "\n",
       "          [[-1.13320164e-01, -3.47490817e-01,  1.28366148e+00, ...,\n",
       "             5.17388582e-01,  5.84510088e-01,  1.17389357e+00]],\n",
       "\n",
       "          [[-7.70380616e-01, -1.98972380e+00, -2.85918593e-01, ...,\n",
       "            -1.42394257e+00, -7.93658346e-02, -1.31458056e+00]]]]]],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "out[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:absl:Found untraced functions such as attention_layer_call_fn, attention_layer_call_and_return_conditional_losses, LayerNorm_mlp_ln0_layer_call_fn, LayerNorm_mlp_ln0_layer_call_and_return_conditional_losses, LayerNorm_mlp_ln1_layer_call_fn while saving (showing 5 of 2080). These functions will not be directly callable after loading.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./cpm_saved_model_with_kv_cache/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./cpm_saved_model_with_kv_cache/assets\n"
     ]
    }
   ],
   "source": [
    "gpt = GPT(\n",
    "    vocab_size=30_000,\n",
    "    layer_size=32,\n",
    "    block_size=1024,\n",
    "    embedding_dropout=0.0,\n",
    "    embedding_size=2560,\n",
    "    num_attention_heads=32,\n",
    "    attention_dropout=0.0,\n",
    "    residual_dropout=0.0,\n",
    "    use_cache=True\n",
    ")\n",
    "\n",
    "embedding_size = 2560\n",
    "attention_head = 32\n",
    "layer_size = 32\n",
    "gpt._set_inputs(\n",
    "    tf.TensorSpec(shape=[None, None],\n",
    "        dtype=tf.int64,\n",
    "        name=\"input_ids\"\n",
    "    ),\n",
    "    tf.TensorSpec(shape=[\n",
    "        layer_size, None, 2, attention_head,\n",
    "        None, embedding_size // attention_head\n",
    "    ], dtype=tf.float32, name=\"kv_cache\")\n",
    ")\n",
    "out2 = gpt(tf.constant([[1]]), out[1])\n",
    "gpt.set_weights(weights)\n",
    "\n",
    "@tf.function\n",
    "def sample(input_ids, kv_cache):\n",
    "    return gpt(input_ids, kv_cache)\n",
    "\n",
    "gpt.save('./cpm_saved_model_with_kv_cache', include_optimizer=False, signatures={\n",
    "    'serving_default': sample.get_concrete_function(\n",
    "        tf.TensorSpec(shape=[None, None],\n",
    "            dtype=tf.int64,\n",
    "            name=\"input_ids\"),\n",
    "        tf.TensorSpec(shape=[\n",
    "                layer_size, None, 2, attention_head,\n",
    "                None, embedding_size // attention_head\n",
    "            ], dtype=tf.float32, name=\"kv_cache\")\n",
    "    )\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:35:10.305733: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2021-10-13 16:35:10.305761: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "/opt/conda/lib/python3.8/runpy.py:127: RuntimeWarning: 'tf2onnx.convert' found in sys.modules after import of package 'tf2onnx', but prior to execution of 'tf2onnx.convert'; this may result in unpredictable behaviour\n",
      "  warn(RuntimeWarning(msg))\n",
      "2021-10-13 16:35:11.733973: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2021-10-13 16:35:11.734000: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-10-13 16:35:11.734022: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (iZuf6fokcl2k1pwfopz0n4Z): /proc/driver/nvidia/version does not exist\n",
      "2021-10-13 16:35:11.734267: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-10-13 16:35:11,744 - WARNING - '--tag' not specified for saved_model. Using --tag serve\n",
      "2021-10-13 16:35:55,987 - INFO - Signatures found in model: [serving_default].\n",
      "2021-10-13 16:35:55,987 - WARNING - '--signature_def' not specified, using first signature: serving_default\n",
      "2021-10-13 16:35:55,989 - INFO - Output names: ['output_1', 'output_2']\n",
      "2021-10-13 16:35:56.138088: I tensorflow/core/grappler/devices.cc:66] Number of eligible GPUs (core count >= 8, compute capability >= 0.0): 0\n",
      "2021-10-13 16:35:56.138317: I tensorflow/core/grappler/clusters/single_machine.cc:357] Starting new session\n",
      "2021-10-13 16:35:56.629990: I tensorflow/core/grappler/optimizers/meta_optimizer.cc:1137] Optimization results for grappler item: graph_to_optimize\n",
      "  function_optimizer: Graph size after: 9081 nodes (8560), 11817 edges (11295), time = 302.617ms.\n",
      "  function_optimizer: function_optimizer did nothing. time = 9.491ms.\n",
      "\n",
      "2021-10-13 16:36:58,759 - INFO - Using tensorflow=2.6.0, onnx=1.9.0, tf2onnx=1.9.2/0f28b7\n",
      "2021-10-13 16:36:58,759 - INFO - Using opset <onnx, 13>\n",
      "2021-10-13 16:38:27,754 - INFO - Computed 741 values for constant folding\n",
      "2021-10-13 16:38:58,122 - INFO - folding node using tf type=Identity, name=Func/StatefulPartitionedCall/input/_1\n",
      "2021-10-13 16:38:58,932 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer00/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,932 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer00/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer00/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer00/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer00/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer00/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer01/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer01/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,934 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer01/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer01/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer01/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer01/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer02/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer02/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer02/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer02/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer02/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,935 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer02/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer03/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer03/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer03/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer03/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer03/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer03/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer04/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,936 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer04/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer04/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer04/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer04/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer04/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer05/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer05/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer05/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer05/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,937 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer05/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer05/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer06/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer06/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer06/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer06/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer06/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer06/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer07/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,938 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer07/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer07/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer07/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer07/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer07/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer08/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer08/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer08/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer08/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,939 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer08/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer08/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer09/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer09/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer09/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer09/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer09/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer09/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,940 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer10/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer10/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer10/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer10/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer10/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer10/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer11/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer11/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer11/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,941 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer11/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer11/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer11/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer12/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer12/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer12/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer12/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer12/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer12/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,942 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer13/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer13/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer13/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer13/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer13/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer13/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer14/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer14/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer14/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,943 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer14/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer14/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer14/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer15/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer15/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer15/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer15/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer15/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer15/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,944 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer16/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer16/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer16/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer16/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer16/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer16/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer17/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer17/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,945 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer17/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer17/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer17/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer17/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer18/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer18/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer18/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer18/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer18/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,946 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer18/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer19/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer19/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer19/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer19/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer19/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer19/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer20/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer20/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,947 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer20/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer20/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer20/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer20/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer21/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer21/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer21/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer21/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer21/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,948 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer21/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer22/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer22/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer22/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer22/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer22/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer22/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer23/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer23/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,949 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer23/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer23/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer23/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer23/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer24/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer24/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer24/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer24/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer24/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,950 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer24/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer25/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer25/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer25/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer25/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer25/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer25/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer26/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,951 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer26/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer26/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer26/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer26/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer26/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer27/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer27/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer27/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer27/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,952 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer27/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer27/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer28/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer28/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer28/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer28/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer28/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer28/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer29/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,953 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer29/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer29/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer29/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer29/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer29/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer30/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer30/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer30/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer30/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,954 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer30/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer30/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer31/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer31/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer31/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer31/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer31/intermediate/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_1/layer31/output/Tensordot/concat\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer00/activation_32/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,955 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer01/activation_33/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer02/activation_34/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer03/activation_35/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer04/activation_36/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer05/activation_37/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer06/activation_38/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer07/activation_39/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer08/activation_40/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,956 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer09/activation_41/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer10/activation_42/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer11/activation_43/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer12/activation_44/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer13/activation_45/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer14/activation_46/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer15/activation_47/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer16/activation_48/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,957 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer17/activation_49/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer18/activation_50/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer19/activation_51/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer20/activation_52/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer21/activation_53/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer22/activation_54/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer23/activation_55/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer24/activation_56/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer25/activation_57/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,958 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer26/activation_58/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,959 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer27/activation_59/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,959 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer28/activation_60/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,959 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer29/activation_61/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,959 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer30/activation_62/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,959 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_1/layer31/activation_63/PartitionedCall/Sqrt\n",
      "2021-10-13 16:38:58,959 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/position_embedding_1/ReadVariableOp\n",
      "2021-10-13 16:38:58,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:38:58,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:38:58,966 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:58,987 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:58,987 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:38:59,029 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,029 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,065 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,065 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,101 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,101 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:38:59,101 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:38:59,101 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,381 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,659 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer00/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,659 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:38:59,660 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:38:59,660 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,703 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,703 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,745 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,745 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,787 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,787 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:38:59,829 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:38:59,830 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:38:59,830 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:38:59,830 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,108 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,109 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,389 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer01/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,389 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:00,389 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:00,389 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,425 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,425 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,467 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,468 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,509 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,509 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,544 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,544 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:00,544 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:00,544 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:00,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:00,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,103 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer02/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,103 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:01,103 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:01,103 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,139 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,139 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,181 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,181 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,223 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,224 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:01,266 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:01,266 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:01,545 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,545 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer03/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:01,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:01,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,865 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,865 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,902 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,902 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,943 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,944 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:01,980 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:01,980 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:01,980 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:01,980 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,260 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,260 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,538 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer04/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,538 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:02,538 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:02,538 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,575 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,576 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,617 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,618 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,660 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,660 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,696 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:02,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:02,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:02,976 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:02,976 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,254 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer05/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,254 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:03,254 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:03,254 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,290 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,290 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,326 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,326 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,370 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,370 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,412 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,412 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:03,412 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:03,412 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,691 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,691 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:03,968 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer06/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:03,969 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:03,969 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:03,969 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/key_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:04,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,040 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,040 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,081 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,082 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,116 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,116 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:04,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:04,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,398 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,675 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer07/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,676 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:04,676 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:04,676 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,712 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,712 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,754 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,755 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,791 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,791 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:04,826 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:04,827 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:04,827 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:04,827 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,104 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,105 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,382 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer08/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,383 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:05,383 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:05,383 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,419 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,419 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,455 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,456 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,498 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,498 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,533 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,534 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:05,534 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:05,534 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:05,816 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:05,817 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,096 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer09/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,096 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:06,096 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:06,097 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,133 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,133 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,175 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,175 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,211 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,211 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,246 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,247 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:06,247 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:06,247 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:06,526 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,527 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,807 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer10/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,808 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:06,808 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:06,808 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,844 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,844 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,886 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,886 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,929 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,929 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:06,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:06,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:06,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:06,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,243 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,243 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,522 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer11/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,523 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:07,523 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:07,523 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,560 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,560 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,594 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,595 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,629 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,629 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,671 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,672 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:07,672 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:07,672 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:07,949 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:07,950 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,229 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer12/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,229 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:08,229 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:08,229 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,306 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,307 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,348 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,349 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,384 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,384 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:08,384 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:08,384 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,662 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,663 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,943 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer13/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,944 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:08,944 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:08,944 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:08,980 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:08,980 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:09,022 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,022 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,064 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,064 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,105 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:09,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:09,106 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,384 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,385 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,662 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer14/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,662 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:09,662 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:09,662 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,698 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,698 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,742 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,742 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,784 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,784 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:09,820 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:09,820 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:09,820 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:09,820 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,099 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,099 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer15/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:10,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:10,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,416 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,416 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,458 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,458 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,501 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,501 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:10,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:10,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:10,819 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:10,820 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,099 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer16/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,099 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:11,100 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:11,100 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,138 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,138 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,180 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,180 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,223 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,223 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:11,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:11,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:11,546 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,546 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,822 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer17/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,822 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:11,822 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:11,822 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,859 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,859 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,900 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,901 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,936 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,936 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:11,978 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:11,979 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:11,979 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:11,979 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,256 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,257 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,532 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer18/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,533 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:12,533 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:12,533 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,569 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,569 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,612 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,612 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,648 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,648 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,685 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:12,685 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:12,685 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:12,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:12,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,242 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer19/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,242 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:13,243 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:13,243 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,285 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,285 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,327 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,327 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,369 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,370 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,405 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,405 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:13,406 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:13,406 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:13,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer20/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:13,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:13,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:13,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/key_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:14,000 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,001 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,043 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,043 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,085 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,085 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,126 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,126 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:14,126 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:14,127 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,405 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,405 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer21/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:14,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:14,684 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,720 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,720 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,762 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,762 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,804 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,805 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:14,840 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:14,840 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:14,840 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:14,840 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,118 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,119 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,398 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer22/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,398 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:15,398 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:15,399 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,434 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,434 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,476 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,477 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,519 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,519 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,554 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,554 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:15,555 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:15,555 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:15,834 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:15,834 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,113 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer23/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,114 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:16,114 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:16,114 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,152 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,153 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,196 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,196 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,233 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,233 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/context_projection_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:16,275 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,275 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:16,276 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:16,276 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,555 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,555 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,833 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer24/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,833 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:16,833 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:16,833 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,869 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,870 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,912 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,913 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,954 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,955 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:16,990 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:16,990 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:16,990 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:16,991 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,269 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,269 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,547 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer25/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,547 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:17,547 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:17,547 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,583 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,584 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,619 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,619 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,654 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,654 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:17,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:17,697 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:17,975 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:17,976 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:18,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer26/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,253 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:18,253 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:18,253 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:18,295 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,295 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:18,337 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,337 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:18,373 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,373 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:18,409 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,409 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:18,409 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:18,409 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:18,687 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/output/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:18,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer27/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:18,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:18,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:18,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,001 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,001 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,043 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,043 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,085 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,085 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,120 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,121 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:19,121 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:19,121 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,398 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,398 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,675 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer28/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,675 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:19,675 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:19,676 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,711 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,712 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,754 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,754 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,796 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,796 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:19,838 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:19,838 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:19,839 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:19,839 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,116 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,117 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer29/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:20,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:20,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,433 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,434 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,470 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,470 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,513 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,513 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,556 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,556 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:20,556 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:20,556 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:20,835 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:20,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:21,118 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer30/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,119 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:21,119 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:21,119 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:21,153 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,153 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:21,188 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,188 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/query_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 16:39:21,222 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,222 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:21,255 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,255 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:21,255 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:21,256 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:21,531 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,531 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 16:39:21,808 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/layer31/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 16:39:21,809 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/LayerNorm_final_norm/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 16:39:21,809 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/LayerNorm_final_norm/batchnorm/ReadVariableOp\n",
      "2021-10-13 16:39:21,809 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_1/Identity\n",
      "2021-10-13 16:39:26,996 - INFO - Optimizing ONNX model\n",
      "2021-10-13 16:41:23,127 - INFO - After optimization: Cast -514 (901->387), Concat -320 (578->258), Const -4319 (4861->542), Gather -256 (385->129), GlobalAveragePool +130 (0->130), Identity -270 (270->0), ReduceMean -130 (130->0), ReduceProd -384 (384->0), Reshape -129 (609->480), Shape -96 (257->161), Squeeze -33 (33->0), Transpose -193 (353->160), Unsqueeze -930 (1091->161)\n",
      "2021-10-13 16:41:43,945 - INFO - \n",
      "2021-10-13 16:41:43,945 - INFO - Successfully converted TensorFlow model cpm_saved_model to ONNX\n",
      "2021-10-13 16:41:43,946 - INFO - Model inputs: ['input_ids']\n",
      "2021-10-13 16:41:43,946 - INFO - Model outputs: ['output_1', 'output_2']\n",
      "2021-10-13 16:41:43,946 - INFO - Zipped ONNX model is saved at onnx/cpm.zip. Unzip before opening in onnxruntime.\n"
     ]
    }
   ],
   "source": [
    "!rm -rf onnx\n",
    "!mkdir -p onnx\n",
    "!python -m tf2onnx.convert --saved-model cpm_saved_model --output onnx/cpm.zip --large_model --opset=13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd onnx && unzip -q cpm.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:27:27.591847: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2021-10-13 17:27:27.591878: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
      "/opt/conda/lib/python3.8/runpy.py:127: RuntimeWarning: 'tf2onnx.convert' found in sys.modules after import of package 'tf2onnx', but prior to execution of 'tf2onnx.convert'; this may result in unpredictable behaviour\n",
      "  warn(RuntimeWarning(msg))\n",
      "2021-10-13 17:27:30.162299: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
      "2021-10-13 17:27:30.162329: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303)\n",
      "2021-10-13 17:27:30.162356: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (iZuf6fokcl2k1pwfopz0n4Z): /proc/driver/nvidia/version does not exist\n",
      "2021-10-13 17:27:30.162617: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2021-10-13 17:27:30,172 - WARNING - '--tag' not specified for saved_model. Using --tag serve\n",
      "2021-10-13 17:28:42,623 - INFO - Signatures found in model: [serving_default].\n",
      "2021-10-13 17:28:42,623 - WARNING - '--signature_def' not specified, using first signature: serving_default\n",
      "2021-10-13 17:28:42,625 - INFO - Output names: ['output_0', 'output_1']\n",
      "2021-10-13 17:28:42.782236: I tensorflow/core/grappler/devices.cc:66] Number of eligible GPUs (core count >= 8, compute capability >= 0.0): 0\n",
      "2021-10-13 17:28:42.782434: I tensorflow/core/grappler/clusters/single_machine.cc:357] Starting new session\n",
      "2021-10-13 17:28:43.271573: I tensorflow/core/grappler/optimizers/meta_optimizer.cc:1137] Optimization results for grappler item: graph_to_optimize\n",
      "  function_optimizer: Graph size after: 9378 nodes (8856), 12179 edges (11656), time = 292.731ms.\n",
      "  function_optimizer: function_optimizer did nothing. time = 6.42ms.\n",
      "\n",
      "2021-10-13 17:29:41,497 - INFO - Using tensorflow=2.6.0, onnx=1.9.0, tf2onnx=1.9.2/0f28b7\n",
      "2021-10-13 17:29:41,499 - INFO - Using opset <onnx, 13>\n",
      "2021-10-13 17:31:20,008 - INFO - Computed 741 values for constant folding\n",
      "2021-10-13 17:31:46,968 - INFO - folding node using tf type=Identity, name=Func/StatefulPartitionedCall/input/_2\n",
      "2021-10-13 17:31:47,586 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer00/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,587 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer00/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,587 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer00/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,587 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer00/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,587 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer00/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,587 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer00/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,587 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer01/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer01/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer01/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer01/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer01/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer01/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer02/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer02/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,588 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer02/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer02/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer02/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer02/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer03/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer03/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer03/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer03/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,589 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer03/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer03/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer04/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer04/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer04/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer04/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer04/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer04/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,590 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer05/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer05/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer05/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer05/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer05/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer05/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer06/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer06/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,591 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer06/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer06/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer06/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer06/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer07/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer07/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer07/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer07/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,592 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer07/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer07/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer08/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer08/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer08/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer08/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer08/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer08/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,593 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer09/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,595 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer09/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer09/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer09/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer09/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer09/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer10/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer10/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer10/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,596 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer10/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer10/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer10/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer11/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer11/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer11/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer11/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer11/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,597 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer11/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer12/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer12/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer12/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer12/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer12/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer12/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer13/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,598 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer13/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer13/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer13/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer13/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer13/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer14/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer14/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer14/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,599 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer14/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer14/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer14/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer15/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer15/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer15/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer15/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer15/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,600 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer15/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer16/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer16/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer16/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer16/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer16/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer16/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer17/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,601 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer17/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer17/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer17/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer17/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer17/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer18/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer18/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer18/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer18/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,602 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer18/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer18/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer19/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer19/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer19/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer19/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer19/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer19/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,603 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer20/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer20/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer20/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer20/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer20/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer20/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer21/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer21/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,604 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer21/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer21/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer21/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer21/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer22/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer22/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer22/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer22/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer22/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,605 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer22/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer23/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer23/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer23/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer23/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer23/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer23/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer24/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,606 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer24/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer24/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer24/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer24/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer24/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer25/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer25/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer25/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,607 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer25/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer25/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer25/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer26/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer26/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer26/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer26/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer26/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,608 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer26/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer27/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer27/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer27/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer27/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer27/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer27/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer28/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,609 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer28/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer28/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer28/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer28/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer28/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer29/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer29/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer29/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer29/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,610 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer29/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer29/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer30/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer30/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer30/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer30/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer30/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer30/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,611 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer31/attention/key_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer31/attention/value_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer31/attention/query_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer31/attention/context_projection_layer/Tensordot/concat\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer31/intermediate/Tensordot/concat\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=ConcatV2, name=StatefulPartitionedCall/gpt_5/layer31/output/Tensordot/concat\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer00/activation_160/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer01/activation_161/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,612 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer02/activation_162/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer03/activation_163/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer04/activation_164/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer05/activation_165/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer06/activation_166/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer07/activation_167/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer08/activation_168/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer09/activation_169/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,613 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer10/activation_170/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer11/activation_171/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer12/activation_172/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer13/activation_173/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer14/activation_174/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer15/activation_175/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer16/activation_176/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer17/activation_177/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,614 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer18/activation_178/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,615 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer19/activation_179/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer20/activation_180/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer21/activation_181/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer22/activation_182/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer23/activation_183/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer24/activation_184/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer25/activation_185/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,619 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer26/activation_186/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,620 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer27/activation_187/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,620 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer28/activation_188/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,620 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer29/activation_189/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,620 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer30/activation_190/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,620 - INFO - folding node using tf type=Sqrt, name=StatefulPartitionedCall/gpt_5/layer31/activation_191/PartitionedCall/Sqrt\n",
      "2021-10-13 17:31:47,620 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/position_embedding_5/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:31:47,640 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:47,640 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:47,641 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:47,663 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:47,663 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:47,699 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:47,699 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:47,735 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:47,735 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:47,770 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:47,770 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:47,770 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:47,770 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,006 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,241 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer00/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,242 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:48,242 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:48,242 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,278 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,278 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,314 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,315 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,350 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,386 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,386 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:48,386 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:48,386 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,623 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,624 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,861 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer01/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:48,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:48,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,898 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,898 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,933 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,934 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:48,969 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:48,969 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,004 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:49,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:49,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,241 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,242 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,477 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer02/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,477 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:49,477 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:49,478 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,513 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,513 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:31:49,549 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,549 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,585 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,585 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,620 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,620 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:49,621 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:49,621 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:49,857 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:49,857 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,092 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer03/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,092 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:50,092 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:50,093 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,128 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,128 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,163 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,164 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,199 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,200 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,235 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,235 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:50,235 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:50,235 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,471 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,472 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,704 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer04/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,704 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:50,705 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:50,705 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,740 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,741 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,776 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,777 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,815 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,815 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:50,851 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:50,851 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:50,851 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:50,851 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:51,167 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:51,167 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:51,490 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer05/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:51,490 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:51,490 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:51,490 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:51,549 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:51,550 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:51,585 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:51,586 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:51,644 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:51,645 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:51,706 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:51,707 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:51,707 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:51,707 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:31:52,022 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,022 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:52,318 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer06/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,318 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:52,318 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:52,318 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:52,377 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,378 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:52,437 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,438 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:52,497 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,497 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:52,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:52,540 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:52,541 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:52,774 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:52,775 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer07/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,005 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:53,006 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:53,006 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,042 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,042 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,077 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,077 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,113 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,113 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,150 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,150 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:53,150 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:53,150 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,379 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,379 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,607 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer08/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,607 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:53,608 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:53,608 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,652 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,652 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,687 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,687 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,722 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,723 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,758 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,758 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:53,759 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:53,759 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:53,986 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:53,986 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,225 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer09/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,226 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:54,226 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:54,226 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/key_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:31:54,261 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,262 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,296 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,297 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,332 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,332 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,367 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,367 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:54,367 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:54,368 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,593 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,594 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer10/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:54,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:54,823 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,858 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,859 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,894 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,894 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,930 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,930 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:54,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:54,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:54,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:54,965 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,192 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,192 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,417 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer11/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,418 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:55,418 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:55,418 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,453 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,453 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,488 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,488 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,523 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,524 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,558 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,558 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:55,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:55,559 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:55,786 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:55,787 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,014 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer12/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,015 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:56,015 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:56,015 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,051 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,051 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,086 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,087 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,158 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,158 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:56,158 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:56,159 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:31:56,385 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,386 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,615 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer13/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,616 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:56,616 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:56,616 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,651 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,652 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,687 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,688 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,723 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,723 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,758 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,759 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:56,759 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:56,759 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:56,988 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:56,989 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,215 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer14/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,216 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:57,216 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:57,216 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,287 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,287 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,322 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,323 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,364 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,364 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:57,364 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:57,365 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,603 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,604 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,831 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer15/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,831 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:57,832 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:57,832 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,871 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,872 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,941 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,941 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:57,977 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:57,977 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:57,978 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:57,978 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,204 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,204 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,436 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer16/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,436 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:58,436 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:31:58,437 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,464 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,465 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,492 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,493 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,527 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,528 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,567 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,567 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:58,568 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:58,568 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:58,895 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:58,895 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer17/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:59,122 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:59,123 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,156 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,157 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,193 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,194 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,228 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,229 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:59,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:59,265 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,493 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,493 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,716 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer18/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,716 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:59,716 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:59,717 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,744 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,745 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,806 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,806 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,837 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:31:59,864 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:31:59,864 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:31:59,864 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:31:59,865 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,059 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,059 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer19/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:00,252 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:00,253 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,279 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,280 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,307 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,307 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,336 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,336 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/context_projection_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:32:00,373 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,374 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:00,374 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:00,374 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,567 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,568 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,763 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer20/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,763 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:00,764 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:00,764 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,791 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,791 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,827 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,827 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,854 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,854 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:00,882 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:00,882 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:00,882 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:00,882 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,075 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,076 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,270 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer21/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,271 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:01,271 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:01,271 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,298 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,334 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,334 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,361 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,361 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:01,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:01,397 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,592 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,592 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,785 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer22/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,785 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:01,786 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:01,786 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,814 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,814 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,841 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,841 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,877 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,877 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:01,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:01,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:01,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:01,905 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,100 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,100 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/output/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:32:02,293 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer23/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,293 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:02,294 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:02,294 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,330 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,330 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,359 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,359 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,394 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,395 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,430 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,430 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:02,431 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:02,431 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,623 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,624 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,818 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer24/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,818 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:02,818 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:02,819 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,846 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,846 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,882 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,882 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,909 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,910 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:02,937 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:02,937 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:02,938 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:02,938 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,130 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,130 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,323 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer25/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,324 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:03,324 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:03,324 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,351 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,352 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,406 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,407 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,442 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,442 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:03,443 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:03,443 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,639 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,640 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,835 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer26/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:03,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:03,836 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,864 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,864 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/value_layer/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:32:03,900 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,900 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,928 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,928 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:03,956 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:03,956 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:03,956 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:03,956 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,154 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,155 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,347 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer27/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,347 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:04,347 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:04,347 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,375 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,375 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,410 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,411 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,438 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,438 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,474 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,474 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:04,474 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:04,474 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,669 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,669 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,861 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer28/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,861 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:04,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:04,862 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,897 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,897 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,936 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,937 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,964 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:04,992 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:04,992 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:04,992 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:04,993 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,184 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,185 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,379 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer29/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:05,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:05,380 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,416 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,416 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,452 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,452 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,479 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,480 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,507 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,507 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:05,508 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:05,508 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/intermediate/Tensordot/ReadVariableOp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-10-13 17:32:05,700 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,700 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,895 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer30/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,895 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/LayerNorm_mlp_ln0/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:05,895 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/LayerNorm_mlp_ln0/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:05,896 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/key_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,922 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/key_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,923 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/value_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,958 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/value_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,958 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/query_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:05,985 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/query_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:05,985 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/context_projection_layer/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:06,021 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/attention/context_projection_layer/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:06,021 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/LayerNorm_mlp_ln1/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:06,021 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/LayerNorm_mlp_ln1/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:06,022 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/intermediate/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:06,215 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/intermediate/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:06,215 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/output/Tensordot/ReadVariableOp\n",
      "2021-10-13 17:32:06,407 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/layer31/output/BiasAdd/ReadVariableOp\n",
      "2021-10-13 17:32:06,407 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/LayerNorm_final_norm/batchnorm/mul/ReadVariableOp\n",
      "2021-10-13 17:32:06,408 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/LayerNorm_final_norm/batchnorm/ReadVariableOp\n",
      "2021-10-13 17:32:06,408 - INFO - folding node using tf type=Identity, name=StatefulPartitionedCall/gpt_5/Identity\n",
      "2021-10-13 17:32:12,127 - INFO - Optimizing ONNX model\n",
      "2021-10-13 17:34:10,530 - INFO - After optimization: Cast -514 (902->388), Concat -320 (643->323), Const -4650 (5224->574), Gather -256 (385->129), GlobalAveragePool +130 (0->130), Identity -271 (271->0), Less -1 (2->1), ReduceMean -130 (130->0), ReduceProd -384 (384->0), Reshape -129 (609->480), Shape -96 (258->162), Squeeze -32 (130->98), Transpose -193 (353->160), Unsqueeze -930 (1093->163)\n",
      "2021-10-13 17:34:33,594 - INFO - \n",
      "2021-10-13 17:34:33,594 - INFO - Successfully converted TensorFlow model cpm_saved_model_with_kv_cache to ONNX\n",
      "2021-10-13 17:34:33,595 - INFO - Model inputs: ['input_ids', 'kv_cache']\n",
      "2021-10-13 17:34:33,595 - INFO - Model outputs: ['output_0', 'output_1']\n",
      "2021-10-13 17:34:33,595 - INFO - Zipped ONNX model is saved at onnx_kv/cpm.zip. Unzip before opening in onnxruntime.\n"
     ]
    }
   ],
   "source": [
    "!rm -rf onnx_kv\n",
    "!mkdir -p onnx_kv\n",
    "!python -m tf2onnx.convert \\\n",
    "    --saved-model cpm_saved_model_with_kv_cache \\\n",
    "    --output onnx_kv/cpm.zip --large_model --opset=13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "!cd onnx_kv && unzip -q cpm.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import onnx\n",
    "from onnxruntime.quantization import quantize_dynamic, QuantType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf onnx_q && mkdir -p onnx_q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "quantized_model = quantize_dynamic(\n",
    "    './onnx/__MODEL_PROTO.onnx',\n",
    "    './onnx_q/cpm.onnx',\n",
    "    weight_type=QuantType.QUInt8,\n",
    "    use_external_data_format=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf onnx_kv_q && mkdir -p onnx_kv_q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "quantized_model = quantize_dynamic(\n",
    "    './onnx_kv/__MODEL_PROTO.onnx',\n",
    "    './onnx_kv_q/cpm.onnx',\n",
    "    weight_type=QuantType.QUInt8,\n",
    "    use_external_data_format=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}