data_process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 162 meetings in the train set.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "# read the dataset\n",
    "# please enter the path of your data\n",
    "split = 'train'\n",
    "data_path = '/path/to/data/ALL/jsonl/' + split + '.jsonl'\n",
    "data = []\n",
    "with open(data_path) as f:\n",
    "    for line in f:\n",
    "        data.append(json.loads(line))\n",
    "n_meetings = len(data)\n",
    "print('Total {} meetings in the {} set.'.format(n_meetings, split))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import word_tokenize\n",
    "# tokneize a sent\n",
    "def tokenize(sent):\n",
    "    tokens = ' '.join(word_tokenize(sent.lower()))\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter some noises caused by speech recognition\n",
    "def clean_data(text):\n",
    "    text = text.replace('{ vocalsound } ', '')\n",
    "    text = text.replace('{ disfmarker } ', '')\n",
    "    text = text.replace('a_m_i_', 'ami')\n",
    "    text = text.replace('l_c_d_', 'lcd')\n",
    "    text = text.replace('p_m_s', 'pms')\n",
    "    text = text.replace('t_v_', 'tv')\n",
    "    text = text.replace('{ pause } ', '')\n",
    "    text = text.replace('{ nonvocalsound } ', '')\n",
    "    text = text.replace('{ gap } ', '')\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1257 query-summary pairs in the train set\n",
      "{'src': \"<s> what did user interface introduce about the detailed design of the prototype ? </s> project manager: welcome back . industrial designer: i 'm sorry to be late . project manager: welcome back everybody . user interface: yeah . thanks . project manager: so this meeting agenda will be the detailed design meeting . and uh opening and uh pms of the meet minutes , uh prototype presentation from uh christine and uh agnes . industrial designer: agnes , yes . project manager: yes and uh evaluation criteria . the finance , it 's uh from my side , from the management , and uh production evaluation . then uh closing . so we have forty minutes to discuss and uh finalise and close the product and project and to move further , okay , so okay , let 's talk about uh maybe first uh for the prototype . user interface: mm , okay . project manager: so i handle to user interface: i 've done a presentation , but it pretty much covers work that we 've both done , so if i 'm missing anything , christine can just correct me . project manager: so shall i go to sorry . industrial designer: uh thank you , so you did a powerpoint presentation , good for you . project manager: yep . s okay , let 's go to ami . user interface: it 's not the biggest powerpoint presentation in the world , but industrial designer: marketing: project manager: so in two or three or industrial designer: user interface: three . um . no it 's marketing: probably . technical pa i would think . user interface: think it 's the last one . no , then this is the la yeah , that one , final design . marketing: ha . project manager: yeah . user interface: it is named appropriately , you just could n't see the name . industrial designer: marketing: user interface: um okay , can i have the mouse ? project manager: yes . user interface: thanks . alright , so from when we were discussing specifying the case in the last meeting , we decided that we wanted an ergonomic shape , the material that we chose was wood , and uh the colour would be customisable , 'cause you can stain the wood whatever colour . um , so in terms of function , you have to be able to turn the tv on and off , volume and channel control , menu control , voice recognition control , and we 've incorporated the lcd screen on the flip panel as part of the design , if we figure out it 's too expensive , well then you just take it off . um , so to unveil our lovely product . this is our remote control , with the flip panel as you can see . marketing: user interface: so if you lift up the panel , you can see the lovely yellow lcd display . um , this is actually hard to do . the yellow button you have is the on off button , so it 's really big , hard to miss . you have the the red um triangles are the toggles for changing the volume . so up volume up , down volume down . the green are the channel changing . s and it 's one of those very light , very touchable displays . and then you have the numeric pad in the dark blue at the bottom , and on the right-hand side you have the access to the menu on the tv , and on the left-hand side you have the the the ability to turn off the voice recognition . so this is pretty much what we had on the white board the last time . project manager: mm-hmm . user interface: so . industrial designer: um and uh i could user interface: oh yes . industrial designer: yeah the d user interface: additional feature on the back is that you can have your own customised backing and i suppose you could do the same thing on the flip case on the front . so that you can really make this a highly highly customisable remote control . industrial designer: we have n't um uh specified where the speaker or the microphone will be placed . that depends on the uh s design of the circuit board inside and uh what room is left um user interface: yeah . project manager: i think the microphone is on on the top , uh on the middle , the under the flip . industrial designer: yes , okay . uh-huh . project manager: so that will be the safe , so p any the chip it 's not on the chip because you need to have microphone to industrial designer: no , i mean it depends on the design of the circuit board . project manager: yes . user interface: but it should n't be under the flip either , because you can have the remote control closed , but you still might want to activate it by voice . project manager: uh it 's it 's yeah , but uh uh my opinion i think it 's better under the flip because whenever you want to uh the talk , okay , so then you can speak then you can close it . but if you put it on the on the flip , okay , then uh technical i do n't think it 's uh feasible , 'cause most of the time you speak then it will be recognised . user interface: but if you 've already got the remote control in your hand you need to open the flip to use the voice , why use the voice , why not just use your hand ? i mean the whole point of the voice is that if the remote control is sitting there and i 'm too lazy to reach over and pick it up , i can just use my voice . industrial designer: maybe i 've got my hand in the popcorn bowl and i 'm holding my cup of coca-cola in the other hand . user interface: yeah . and you do n't wan na let go of either one . marketing: i do n't wan na say . louder . yeah . user interface: i mean it does n't have to be on the flip , it can be on the side somewhere . marketing: can also be on the side . project manager: yeah , the sides maybe is good . user interface: yeah . industrial designer: mm-hmm . user interface: so , i mean i can pass this around if anyone wants to project manager: yes . so it 's maybe good idea . industrial designer: yeah , y better you pass it around with a napkin . user interface: marketing: no , because y you can easily put a microphone on the side that would have no problem would have n't been not be damaged or anything , and it 'd be accessible all the time to voice . user interface: yeah . project manager: yes . yeah . user interface: yeah , exactly . marketing: yeah . project manager: so it 's maybe good idea . s s industrial designer: it 's um it 's um marketing: compliments to the artist . industrial designer: you need to work on the weight a little bit . user interface: yes . marketing: uh . project manager: okay . s i 'm fine , i 'm satisfi user interface: and maybe the shape of the buttons , project manager: i 'm satisfied . user interface: the little egg shapes are n't the most economical , but industrial designer: we 're glad you 're satisfied . project manager: of course it 's it 's it 's looks more heo heavy , but i think when it 's completely maybe it 's a less weight . user interface: yeah . i mean this is plasticene . project manager: yes . user interface: there 's only so much you can do . we could have possibly made it a lot thinner as well . but and part of the thing is m a lot of people say that they do n't like something that 's too light , because they do n't feel like they have enough control over it . industrial designer: mm-hmm . user interface: so i mean maybe this is excessively heavy , but i think it needs to have some weight , project manager: yep . user interface: it needs to feel like you 're still holding something . so that 's pretty much it for our presentation actually . project manager: that 's your uh prototype model ? user interface: yeah . project manager: okay , that 's good , thank you very much . marketing: project manager: so any comments or uh marketing: well , the prototype is is very well within the design and ideas that we 've we 've talked about on the previous meetings . project manager: okay . marketing: now it goes into this next phase as the financial uh marketing uh project manager: yes , that uh so i 'll come back to the so evaluation criteria , i think uh that will be good , so then let 's come to the finance uh , i have some uh calculations which i made uh as for uh the budget . so here you can uh look like uh the energy and uh dynamo and uh kinetic and solar cells . uh it 's optional , somewhat optional and ed wants the chip on print , that 's what uh we were talking about that . industrial designer: project manager: so then we have sample sensor and sample speaker , then uh we have the wood material , then special colour and push button . so it 's uh actually , our budget was uh twelve point five euro , but uh it 's coming to nine point nine five euro , so we are under uh below the budget , okay , so still we are saving some money . i think it 's a good figure . industrial designer: yes , great i i 'm surprised . marketing: industrial designer: congratulations . project manager: than thank you . marketing: well we have n't come to mine yet , so industrial designer: oh , okay user interface: industrial designer: . it 's gon na cost a long way to c you know , cost a lot of money to market it , is it ? marketing: we 're gon na have a bit of difference of opinion , yes . project manager: so maybe it 's for some money we can utilise for our uh marketing , for the sales , okay , and uh marketing: well , it just depends on if we 're gon na add a o on this pr provisionary cost analysis , we do not have a l_c_ display . l_c_ display is gon na be very expensive , user interface: no we do , but it 's not filled in . marketing: it 's gon na be project manager: it 's not . industrial designer: thirty . marketing: it 's not it does n't say . user interface: it 's number thirty . project manager: yeah . yeah . marketing: we do n't have the price up there , user interface: oh , yeah , yeah , project manager: yeah . yeah . user interface: you 're right , sorry , yes . industrial designer: yeah . marketing: okay , so if we add approximately two to three euro per remote , now we 're up around about twelve , twelve and a half as to what uh the company had initially uh requested . um industrial designer: so that means we can put the uh the lcd in , yeah . marketing: display in . but as far as production um i 'm putting up a question because we 're talking about profit also , and in mine you 'll see uh the problem with uh our survey , the p the possibility that how many units can be sold , what percentage of the market , etcetera etcetera because that has to be taken in into consideration . project manager: marketing: uh this is just production cost , it is not uh advertising cost , it 's not transportation cost uh project manager: yes , so still uh we have twelve point five euro . marketing: and that will inflate quite a bit the cost of the uh the cost of the unit for the company . project manager: yes . yeah , but yes . yep . user interface: um-hmm . marketing: so to come up with what the company wants is a fifty million pound profit , we 're gon na have to go a long ways . project manager: yes . this we are talking about one unit , okay , so when it go into the quantity , okay , and the cost will come down . marketing: yes . industrial designer: although customisation , because this is being done , you know , the on on-order basis , it might be uh the the quantity wo n't m wo n't uh marketing: slightly . project manager: yeah . marketing: it 's gon na be very hard to reduce . user interface: yes . industrial designer: the circuit board will b you 're right , would be in producing quantity , but the cost of the case would uh be fixed at the uh you got some pretty cheap labour that can do this case for one euro . marketing: that 's not bad . industrial designer: that 's really that 's the cost of the material and lab wow , that 's really outstanding . project manager: yep . yeah . but anyhow , still we are under control , okay , so what uh i will do is i will try to negotiate with the vendors , okay , to get uh the production cost less , okay , so then we can save some money , okay , to put into th our marketing or uh you know the promotions , whatever , okay , so that uh i will look after . i will speak to the management and how to get uh you know some more uh cost down . marketing: if we can go to to my display . and we 'll come back to yours project manager: yes . marketing: just to give everybody an idea of the market . so now i 'm gon na scare everybody out of this project . industrial designer: user interface: marketing: if i 'm still here . project manager: you 're in four ? marketing: yep the four gives me it 's got ta be uh trendwatch . project manager: trendwatch . industrial designer: is this the same one you did before ? marketing: no . industrial designer: okay . marketing: it should n't be user interface: that 's no , i think it 's the same one . marketing: if it 's not it 's not the right one . no , no we g no , that 's the same one . you have to go back and find another one . whatever name it popped up under . project manager: user interface: functional . marketing: uh functional , try functional , it might not be it either , but we 'll see . industrial designer: user interface: it looks like it , there 's s yeah . marketing: yep , that 's it . so we 'll go screen by screen . project manager: okay . marketing: although since uh we need to have some type of idea on a market uh we had independent study that says it this this market has an availability to absorb eight mi eight million units per year . okay ? our internal company evaluation puts it between eight to nine million which is approximately the same as the independent study . project manager: yep . marketing: so if we continue , we 'll look at the findings . next screen . which means that uh if we have a target of two million would the company has to take twenty five percent of the market in the first year , project manager: mm-hmm . marketing: which is actually a tremendous amount . industrial designer: yeah , no kidding . marketing: no kidding , yeah . industrial designer: mayb maybe they already expected something . marketing: so , if we put an inflated price of fifty euro at a production cost that can not exceed twenty-five euro , okay , we 're already in that that price , okay , project manager: yes . marketing: with transport , promotion , labour , because we hav gi included the promotion in the cost , transport for the material to the stores or whatever how however we 're gon na break this down between our our retailers . user interface: um-hmm . marketing: twenty-five percent of the market to get to two million units . at two million units , we have to have a profit of twenty-five euro per unit to get to the fifty million unit eu euro profit . project manager: mm-hmm . yep . marketing: okay ? so , obviously we w w i just did a run down the evaluation of the form , the fan uh the fancy stylishness of the of the unit , the ease of use , speech recognition , cost , we 've gone through these . now , the company must evaluate the feasibility of being able to take enough of the market to justify in production . or we project this over two years , but being that the market changes very very quickly , maybe there 's no more interest in buying this thing in eighteen months from now . project manager: yes . yep . of course . marketing: so , now we have to come up with a decision . user interface: mm . marketing: can the company sell two million units ? project manager: yep . marketing: can it sell it for fifty euros ? industrial designer: could could i go to findings ? project manager: yep . industrial designer: uh uh um i would uh like to explore the possibility of using um alternative um delivery and sales channel which would be um to use the internet for promotion and ordering project manager: yes . user interface: i was thinking the same thing , yeah . industrial designer: and then to drop-ship the p product to the customer 's residence . user interface: directly . industrial designer: that way you have no storage , you have no um you do have transportation , still have the labour cost , user interface: um-hmm . industrial designer: but you do n't have the transport to the uh point of sale . project manager: yes . industrial designer: the point of sale is online . user interface: yeah . project manager: to the agents . user interface: yeah . you can do a shipping centre somewhere , or strategically place shipping centres to minimise distance costs . industrial designer: right , like amazon . in fact , we should sell through amazon , project manager: yes . or ebay , or industrial designer: do n't you think ? or ebay , yeah . project manager: yes . user interface: yeah . industrial designer: there 's an idea . marketing: project manager: yeah , that 's a good idea . industrial designer: going with um project manager: to impro more profit and uh marketing: s upscale technology . project manager: yeah , yes . industrial designer: ah , we we 're do you know , selling a unique product uh . marketing: well . user interface: that actually makes more sense if we 're gon na make it so highly customisable , 'cause on the web people can look at the different options they have , see maybe what other people have done , what the range of possibility as , industrial designer: mm . marketing: there are several companies that have gone that way . user interface: whereas if you 're in a store , you ca n't unless you 're a highly imaginative person , you may not really know what it is you want , industrial designer: mm . user interface: whereas on the web , if you have a bunch of pictures , it can sort of trigger ideas and industrial designer: mm-hmm . and you can even have an a movie that you can rotate the object and look at the di the only thing that you 're missing really is the weight . user interface: yeah . the weight and feel . marketing: weight , the feel of the product , but industrial designer: mm . user interface: yeah . industrial designer: we 're getting used to that . it 's not quite like trying on a shoe , but people are getting used to buying things online that they ca n't touch before buying . user interface: yeah . yeah . marketing: there are several that have gone through with the watches , too . you can customise a watch , you can see how it is at the f at the end of the production , industrial designer: uh-huh . user interface: mm-hmm . marketing: you can change it uh there 's a lot of online that 's that is doing this now . user interface: yes . marketing: and when you 're rotating , you 'll look behind and look this way uh it 's possible to do with this , maybe there 's a possibility of selling more than two million units in one year , which could you know , feasibili feasibility uh lower the price of the unit . project manager: yeah . user interface: mm-hmm . project manager: we can . industrial designer: great . project manager: i do n't think that 's uh not possible , it 's uh okay then , l uh let 's wait for the production , okay , then uh you can evaluate the product , so how it looks like technically and uh how it look like uh the real . user interface: what turnaround time do we have ? project manager: t user interface: 'cause i mean production evaluation can be very very quick or very very long . project manager: oh but yes it 's it 's very quick , of course . it will uh come back in two weeks , okay , it will be ready in two weeks . industrial designer: works for me . project manager: for evaluation , okay . industrial designer: prototypes , you mean . project manager: yes , the prototype uh prototype product evaluation . industrial designer: in um we probably should do some market tests uh once we have the prototypes and do some orders and things like that and test-market it . project manager: yes . yes . yes . marketing: well , obviously . user interface: yeah . project manager: yeah . marketing: mm that 'd have to be thrown out on the market for people to get an idea , project manager: so you can take a minimum two weeks to a maximum four weeks . marketing: to see get get their user interface: mm-hmm . think minimum two weeks if we 're gon na develop prototypes and then try to take them to different places and see how people use project manager: yeah . user interface: it 's not a trivial task . project manager: yeah , because we we are not going to do it in uh our factory , okay , so we can give it a product evalua user interface: no no . we definitely should n't do it in our factory . project manager: yes , yes . industrial designer: project manager: so we 'll do it in the other place , and i do n't think it 's take more than four weeks time . or uh okay , so then the real production we will start once we product evaluation , okay , then uh it 's approve from uh the technical team and uh your team , okay , uh from the management , then we can launch in the market . hm ? industrial designer: any outstanding ? project manager: s any any other uh questions or uh comments to be discuss ? industrial designer: no , i 'm user interface: what ab industrial designer: go ahead . marketing: i think we pretty much covered everything . project manager: okay , so then uh industrial designer: did you have something ? user interface: well i was just wondering about if we 're gon na do a product evaluation then what about time for redesign if the users come back and tell us no this is bad , this is bad , we want this done differently . project manager: okay uh , let 's take like this . let 's proceed with this model , okay , for the for the marketing direction , okay . industrial designer: project manager: so no more changes will be made , okay , in this the basic design . okay ? so we will introduce m this model and uh let 's introduce in the market and let 's take the feedback from the customers , then we can uh go for the industrial designer: second generation . project manager: second generation . user interface: project manager: okay . there 's no end , there 's not limit . industrial designer: user interface: the problem is there might not be a second generation if the first generation flops for some silly reason that we have n't thought of . project manager: every every custom industrial designer: marketing: well , then it may not be . project manager: okay . well , every customer , okay , they have their own ideas , they have their own test , okay , so there 's no end , there 's no limit . marketing: like people do n't like wood . user interface: no , but there 's a difference between releasing a product that has been minimally tested and fine-tuned to suit a general range of requirements versus releasing a product that we think will work but we do n't really have anything to back it up . marketing: very specific . project manager: yeah , so that 's the reason you are here for uh the design , okay , i hope you made a good design . user interface: yes , but i 'm not everybody . industrial designer: user interface: i mean the whole point of user evaluation is to see what real people need . we have our own motivations in mind , we have our own ideas in mind , but that does n't mean that that 's what 's gon na sell . project manager: yeah , but uh see , we ought to take a few considerations , okay , one is the price consideration , one is future consideration , okay , like uh you can eat uh you can all eat more chi i can eat more chilli , okay , so i it 's a depends on the individual taste , you know , industrial designer: project manager: so we have we have to balance somewhere . user interface: yeah , of course . i 'm just trying to point out that i think that your evaluation and redesign turnaround time is too short well you have no redesign not you personally , but in the project we have no redesign time and project manager: yes . industrial designer: our project does n't um project manager: yeah . industrial designer: ed , d do you know what season of the year or time of the year is the most important for tv remote control sales ? marketing: hmm . industrial designer: would it be the christmas season by any chance ? project manager: the sports time . industrial designer: sports season . which sport season ? marketing: right before the eur the world cup . world soccer . world cup soccer , project manager: football . industrial designer: so so user interface: yeah . marketing: they need those things that they have their hands g occupied and they need to be able to talk to the con remote control . project manager: football . industrial designer: maybe what user interface: industrial designer: so i think what we need to do is perhaps to synchronise the final the the launch of a user-tested device with some special event . marketing: project manager: yes . yes . industrial designer: and and then um user interface: yeah , that 's a good idea . industrial designer: so that gives us a little more time perhaps then we anticipated , because i do n't know when the world cup is , but i 'm sure there 's gon na be one . marketing: or any major sports . industrial designer: or another m major sports event . probably not the um the football games coming up the end of january . i think that might be a little too aggressive um , project manager: yes . user interface: yeah . industrial designer: but , so , i 'm just ig uh pointing out a uh a strategy to uh do some additional user testing project manager: research . industrial designer: pri and then to launch um at a a major sports event user interface: mm-hmm . project manager: yep . industrial designer: or uh perhaps to uh also user interface: that 's actually good place to advertise it too . industrial designer: and to work with motion pictures . user interface: yeah . industrial designer: there might be some motion pictures that are coming out um that are coming out on d_v_d_ that uh they need to have a m special remote control to work with it , so we could maybe work out a campaign with uh with sony pictures for example . project manager: yes . industrial designer: maybe some management has got uh relationships there we can leverage . user interface: mm-hmm . project manager: yes , the that of course uh i will convince the management to do that , okay . industrial designer: that 's great . user interface: it 's just something to to keep in mind , 'cause it 's really really important . project manager: sure , sure , yes . user interface: a lot of products have gone out there without being properly user-tested and completely flopped , when in fact it gets re-released a few years down the line with proper testing and it takes off like crazy . industrial designer: disposable diapers is an example of that in fact . user interface: really ? industrial designer: yes , it is one of the first consumer products that was launched about thirty years that was a disposable consumer product , user interface: that i did n't know . industrial designer: and uh people the market had n't really gotten on to the concept that you could use something and then throw it away , 'cause it was n't uh user interface: mm-hmm . industrial designer: but then when they re-launched them thirty years later , they were virtually the same design , project manager: . industrial designer: but people had gotten the throw-away , you know , paper cups and napki y all kinds of things that they had n't um so , you 're right , timing is very important , user interface: yeah . project manager: yeah . industrial designer: but i think we 've got a good product . project manager: that 's the reason ed is here . user interface: yeah . project manager: i think he can promote the the brand value and the product value . industrial designer: that 's right . it 's gon na be very important to the company . project manager: yes . we are behind the scene and he is the front screen , so . user interface: yep . project manager: he 's on the big screen . marketing: yeah , i 'm the one who takes the heat . user interface: exactly . industrial designer: good luck , ed . marketing: if it 's a flop , it 's the marketer . industrial designer: you look very relaxed , considering h you know , the uh the weight on your shoulders , yeah . marketing: yes . yes . stress . user interface: project manager: okay , so then uh let 's come to the closing and uh are the costs within the budget marketing: celebration . project manager: and uh is the product evaluated , okay , so that will uh come soon . okay for uh but our time being , so thanks for all your efforts and great work and uh great design and uh let 's leave it to the ed for later for once production is over and the meantime let 's celebrate . so let 's meet up uh this evening to hang up for some party . industrial designer: user interface: marketing: industrial designer: sounds good . project manager: 's good . industrial designer: thank you . user interface: okay . marketing: very good . industrial designer: nice working with you . marketing: thank you very much . project manager: thank you . thank you again for all . user interface: thanks industrial designer: user interface: project manager: and see you in the evening for drinks . marketing: bye-bye . yep , okay , see you later on . user interface: bye . </s>\", 'tgt': 'user interface suggested that the prototype of the remote control would have an ergonomic shape , be made out of wood and the colour would be customisable . user interface also introduced the general layout of the white board under the flip : the big yellow button would be charged for turning the tv on and off , the red triangle one would be used to change the volume , while the green one controlled the channel changing . there would be a dark blue numeric pad at the bottom and an lcd screen on the flip panel . the prototype would have a customised backing and front .'}\n"
     ]
    }
   ],
   "source": [
    "# process data for BART\n",
    "# the input of the model here is the entire content of the meeting\n",
    "bart_data = []\n",
    "for i in range(len(data)):\n",
    "    # get meeting content\n",
    "    src = []\n",
    "    for k in range(len(data[i]['meeting_transcripts'])):\n",
    "        cur_turn = data[i]['meeting_transcripts'][k]['speaker'].lower() + ': '\n",
    "        cur_turn = cur_turn + tokenize(data[i]['meeting_transcripts'][k]['content'])\n",
    "        src.append(cur_turn)\n",
    "    src = ' '.join(src)\n",
    "    for j in range(len(data[i]['general_query_list'])):\n",
    "        cur = {}\n",
    "        query = tokenize(data[i]['general_query_list'][j]['query'])\n",
    "        cur['src'] = clean_data('<s> ' + query + ' </s> ' + src + ' </s>')\n",
    "        target = tokenize(data[i]['general_query_list'][j]['answer'])\n",
    "        cur['tgt'] = target\n",
    "        bart_data.append(cur)\n",
    "    for j in range(len(data[i]['specific_query_list'])):\n",
    "        cur = {}\n",
    "        query = tokenize(data[i]['specific_query_list'][j]['query'])\n",
    "        cur['src'] = clean_data('<s> ' + query + ' </s> ' + src + ' </s>')\n",
    "        target = tokenize(data[i]['specific_query_list'][j]['answer'])\n",
    "        cur['tgt'] = target\n",
    "        bart_data.append(cur)\n",
    "        \n",
    "print('Total {} query-summary pairs in the {} set'.format(len(bart_data), split))\n",
    "print(bart_data[2])\n",
    "with open('/path/to/data/bart_' + split + '.jsonl', 'w') as f:\n",
    "    for i in range(len(bart_data)):\n",
    "        print(json.dumps(bart_data[i]), file=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 1257 query-summary pairs in the train set\n",
      "{'src': \"<s> what did user interface introduce about the detailed design of the prototype ? </s> industrial designer: i 'm sorry to be late . project manager: welcome back everybody . user interface: yeah . thanks . project manager: so this meeting agenda will be the detailed design meeting . and uh opening and uh pms of the meet minutes , uh prototype presentation from uh christine and uh agnes . industrial designer: agnes , yes . project manager: yes and uh evaluation criteria . the finance , it 's uh from my side , from the management , and uh production evaluation . then uh closing . so we have forty minutes to discuss and uh finalise and close the product and project and to move further , okay , so okay , let 's talk about uh maybe first uh for the prototype . user interface: mm , okay . project manager: so i handle to user interface: i 've done a presentation , but it pretty much covers work that we 've both done , so if i 'm missing anything , christine can just correct me . project manager: so shall i go to sorry . industrial designer: uh thank you , so you did a powerpoint presentation , good for you . project manager: yep . s okay , let 's go to ami . user interface: it 's not the biggest powerpoint presentation in the world , but industrial designer: marketing: project manager: so in two or three or industrial designer: user interface: three . um . no it 's marketing: probably . technical pa i would think . user interface: think it 's the last one . no , then this is the la yeah , that one , final design . marketing: ha . project manager: yeah . user interface: it is named appropriately , you just could n't see the name . industrial designer: marketing: user interface: um okay , can i have the mouse ? project manager: yes . user interface: thanks . alright , so from when we were discussing specifying the case in the last meeting , we decided that we wanted an ergonomic shape , the material that we chose was wood , and uh the colour would be customisable , 'cause you can stain the wood whatever colour . um , so in terms of function , you have to be able to turn the tv on and off , volume and channel control , menu control , voice recognition control , and we 've incorporated the lcd screen on the flip panel as part of the design , if we figure out it 's too expensive , well then you just take it off . um , so to unveil our lovely product . this is our remote control , with the flip panel as you can see . marketing: user interface: so if you lift up the panel , you can see the lovely yellow lcd display . um , this is actually hard to do . the yellow button you have is the on off button , so it 's really big , hard to miss . you have the the red um triangles are the toggles for changing the volume . so up volume up , down volume down . the green are the channel changing . s and it 's one of those very light , very touchable displays . and then you have the numeric pad in the dark blue at the bottom , and on the right-hand side you have the access to the menu on the tv , and on the left-hand side you have the the the ability to turn off the voice recognition . so this is pretty much what we had on the white board the last time . project manager: mm-hmm . user interface: so . industrial designer: um and uh i could user interface: oh yes . industrial designer: yeah the d user interface: additional feature on the back is that you can have your own customised backing and i suppose you could do the same thing on the flip case on the front . so that you can really make this a highly highly customisable remote control . </s>\", 'tgt': 'user interface suggested that the prototype of the remote control would have an ergonomic shape , be made out of wood and the colour would be customisable . user interface also introduced the general layout of the white board under the flip : the big yellow button would be charged for turning the tv on and off , the red triangle one would be used to change the volume , while the green one controlled the channel changing . there would be a dark blue numeric pad at the bottom and an lcd screen on the flip panel . the prototype would have a customised backing and front .'}\n"
     ]
    }
   ],
   "source": [
    "# process data for BART\n",
    "# the input of the model here is the gold span corresponding to each query\n",
    "bart_data_gold = []\n",
    "for i in range(len(data)):\n",
    "    # get meeting content\n",
    "    entire_src = []\n",
    "    for k in range(len(data[i]['meeting_transcripts'])):\n",
    "        cur_turn = data[i]['meeting_transcripts'][k]['speaker'].lower() + ': '\n",
    "        cur_turn = cur_turn + tokenize(data[i]['meeting_transcripts'][k]['content'])\n",
    "        entire_src.append(cur_turn)\n",
    "    entire_src = ' '.join(entire_src)\n",
    "    for j in range(len(data[i]['general_query_list'])):\n",
    "        cur = {}\n",
    "        query = tokenize(data[i]['general_query_list'][j]['query'])\n",
    "        cur['src'] = clean_data('<s> ' + query + ' </s> ' + entire_src + ' </s>')\n",
    "        target = tokenize(data[i]['general_query_list'][j]['answer'])\n",
    "        cur['tgt'] = target\n",
    "        bart_data_gold.append(cur)\n",
    "    for j in range(len(data[i]['specific_query_list'])):\n",
    "        cur = {}\n",
    "        query = tokenize(data[i]['specific_query_list'][j]['query'])\n",
    "        src = []\n",
    "        # get the content in the gold span for each query\n",
    "        for span in data[i]['specific_query_list'][j]['relevant_text_span']:\n",
    "            assert len(span) == 2\n",
    "            st, ed = int(span[0]), int(span[1])\n",
    "            for k in range(st, ed + 1):\n",
    "                cur_turn = data[i]['meeting_transcripts'][k]['speaker'].lower() + ': '\n",
    "                cur_turn = cur_turn + tokenize(data[i]['meeting_transcripts'][k]['content'])\n",
    "                src.append(cur_turn)\n",
    "        src = ' '.join(src)\n",
    "        cur['src'] = clean_data('<s> ' + query + ' </s> ' + src + ' </s>')\n",
    "        target = tokenize(data[i]['specific_query_list'][j]['answer'])\n",
    "        cur['tgt'] = target\n",
    "        bart_data_gold.append(cur)\n",
    "        \n",
    "print('Total {} query-summary pairs in the {} set'.format(len(bart_data_gold), split))\n",
    "print(bart_data_gold[2])\n",
    "with open('/path/to/data/bart_' + split + '._gold.jsonl', 'w') as f:\n",
    "    for i in range(len(bart_data_gold)):\n",
    "        print(json.dumps(bart_data_gold[i]), file=f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}