Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Scripts #1

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Scripts #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
53778f4
Initial commit of changes made
sboyle3 Nov 17, 2022
81f0132
Adding new files on scripts branch
sboyle3 Nov 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.txt
149 changes: 149 additions & 0 deletions gitSectionA/dbmaint.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/ kdb+ partitioned database maintenance
\d .os
WIN:.z.o in`w32`w64
pth:{p:$[10h=type x;x;string x];if[WIN;p[where"/"=p]:"\\"];(":"=first p)_ p}
cpy:{system$[WIN;"copy /v /z ";"cp "],pth[x]," ",pth y}
del:{system$[WIN;"del ";"rm "],pth x}
ren:{system$[WIN;"move ";"mv "],pth[x]," ",pth y}
here:{hsym`$system$[WIN;"cd";"pwd"]}
\d .

add1col:{[tabledir;colname;defaultvalue]
if[not colname in ac:allcols tabledir;
stdout"adding column ",(string colname)," (type ",(string type defaultvalue),") to `",string tabledir;
num:count get(`)sv tabledir,first ac;
.[(`)sv tabledir,colname;();:;num#defaultvalue];
@[tabledir;`.d;,;colname]]}

allcols:{[tabledir]get tabledir,`.d}

allpaths:{[dbdir;table]
files:key dbdir;
if[any files like"par.txt";:raze allpaths[;table]each hsym each`$read0(`)sv dbdir,`par.txt];
files@:where files like"[0-9]*";(`)sv'dbdir,'files,'table}

copy1col:{[tabledir;oldcol;newcol]
if[(oldcol in ac)and not newcol in ac:allcols tabledir;
stdout"copying ",(string oldcol)," to ",(string newcol)," in `",string tabledir;
.os.cpy[(`)sv tabledir,oldcol;(`)sv tabledir,newcol];@[tabledir;`.d;,;newcol]]}

delete1col:{[tabledir;col]
if[col in ac:allcols tabledir;
stdout"deleting column ",(string col)," from `",string tabledir;
.os.del[(`)sv tabledir,col];@[tabledir;`.d;:;ac except col]]}

/
enum:{[tabledir;val]
if[not 11=abs type val;:val];
.[p;();,;u@:iasc u@:where not(u:distinct enlist val)in v:$[type key p:(`)sv tabledir,`sym;get p;0#`]];`sym!(v,u)?val}
\

enum:{[tabledir;val]if[not 11=abs type val;:val];.Q.dd[tabledir;`sym]?val}


find1col:{[tabledir;col]
$[col in allcols tabledir;
stdout"column ",string[col]," (type ",(string first"i"$read1((`)sv tabledir,col;8;1)),") in `",string tabledir;
stdout"column ",string[col]," *NOT*FOUND* in `",string tabledir]}

fix1table:{[tabledir;goodpartition;goodpartitioncols]
if[count missing:goodpartitioncols except allcols tabledir;
stdout"fixing table `",string tabledir;{add1col[x;z;0#get y,z]}[tabledir;goodpartition]each missing]}

fn1col:{[tabledir;col;fn]
if[col in allcols tabledir;
oldattr:-2!oldvalue:get p:tabledir,col;
newattr:-2!newvalue:fn oldvalue;
if[$[not oldattr~newattr;1b;not oldvalue~newvalue];
stdout"resaving column ",(string col)," (type ",(string type newvalue),") in `",string tabledir;
oldvalue:0;.[(`)sv p;();:;newvalue]]]}

reordercols0:{[tabledir;neworder]
if[not((count ac)=count neworder)or all neworder in ac:allcols tabledir;'`order];
stdout"reordering columns in `",string tabledir;
@[tabledir;`.d;:;neworder]}

rename1col:{[tabledir;oldname;newname]
if[(oldname in ac)and not newname in ac:allcols tabledir;
stdout"renaming ",(string oldname)," to ",(string newname)," in `",string tabledir;
.os.ren[` sv tabledir,oldname;` sv tabledir,newname];@[tabledir;`.d;:;.[ac;where ac=oldname;:;newname]]]}

ren1table:{[old;new]stdout"renaming ",(string old)," to ",string new;.os.ren[old;new];}

add1table:{[dbdir;tablename;table]
stdout"adding ",string tablename;
@[tablename;`;:;.Q.en[dbdir]0#table];}

stdout:{-1 raze[" "sv string`date`second$.z.P]," ",x;}
validcolname:{(not x in `i,.Q.res,key`.q)and x = .Q.id x}

//////////////////////////////////////////////////////////////////////////////////////////////////////////
// * public

thisdb:`:. / if functions are to be run within the database instance then use <thisdb> (`:.) as dbdir

addcol:{[dbdir;table;colname;defaultvalue] / addcol[`:/data/taq;`trade;`noo;0h]
if[not validcolname colname;'(`)sv colname,`invalid.colname];
add1col[;colname;enum[dbdir;defaultvalue]]each allpaths[dbdir;table];}

castcol:{[dbdir;table;col;newtype] / castcol[thisdb;`trade;`size;`short]
fncol[dbdir;table;col;newtype$]}

clearattrcol:{[dbdir;table;col] / clearattr[thisdb;`trade;`sym]
setattrcol[dbdir;table;col;(`)]}

copycol:{[dbdir;table;oldcol;newcol] / copycol[`:/k4/data/taq;`trade;`size;`size2]
if[not validcolname newcol;'(`)sv newcol,`invalid.newname];
copy1col[;oldcol;newcol]each allpaths[dbdir;table];}

deletecol:{[dbdir;table;col] / deletecol[`:/k4/data/taq;`trade;`iz]
delete1col[;col]each allpaths[dbdir;table];}

findcol:{[dbdir;table;col] / findcol[`:/k4/data/taq;`trade;`iz]
find1col[;col]each allpaths[dbdir;table];}

/ adds missing columns, but DOESN'T delete extra columns - do that manually
fixtable:{[dbdir;table;goodpartition] / fixtable[`:/k4/data/taq;`trade;`:/data/taq/2005.02.19]
fix1table[;goodpartition;allcols goodpartition]each allpaths[dbdir;table]except goodpartition;}

fncol:{[dbdir;table;col;fn] / fncol[thisdb;`trade;`price;2*]
fn1col[;col;fn]each allpaths[dbdir;table];}

listcols:{[dbdir;table] / listcols[`:/k4/data/taq;`trade]
allcols first allpaths[dbdir;table]}

renamecol:{[dbdir;table;oldname;newname] / renamecol[`:/k4/data/taq;`trade;`woz;`iz]
if[not validcolname newname;'` sv newname,`invalid.newname];
rename1col[;oldname;newname]each allpaths[dbdir;table];}

reordercols:{[dbdir;table;neworder] / reordercols[`:/k4/data/taq;`trade;reverse cols trade]
reordercols0[;neworder]each allpaths[dbdir;table];}

setattrcol:{[dbdir;table;col;newattr] / setattr[thisdb;`trade;`sym;`g] / `s `p `u
fncol[dbdir;table;col;newattr#]}

addtable:{[dbdir;tablename;table] / addtable[`:.;`trade;([]price...)]
add1table[dbdir;;table]each allpaths[dbdir;tablename];}

rentable:{[dbdir;old;new] / rentable[`:.;`trade;`transactions]
ren1table'[allpaths[dbdir;old];allpaths[dbdir;new]];}

\
test with https://github.com/KxSystems/kdb/blob/master/tq.q (sample taq database)

if making changes to current database you need to reload (\l .) to make modifications visible

if the database you've been modifying is a tick database don't forget to adjust the schema (tick/???.q) to reflect your changes to the data


addcol[`:.;`trade;`num;10]
addcol[`:.;`trade;`F;`test]
delete1col[`:./2000.10.02/trade;`F]
fixtable[`:.;`trade;`:./2000.10.03/trade]
reordercols[`:.;`quote;except[2 rotate cols quote;`date]]
clearattrcol[`:.;`trade;`sym]
setattrcol[`:.;`trade;`sym;`p]
castcol[`:.;`trade;`time;`second]
renamecol[`:.;`trade;`price;`PRICE]
pxcols:{(y,())renamecol[`:.;x]'z,()]
`PRICE`size renamecol[`:.;`trade]'`p`s
71 changes: 71 additions & 0 deletions gitSectionA/quote.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
time sym bid ask bsize asize
2019-06-18D03:05:15.716040000 GS.N 4.194429 28.92601 4 5
2019-06-18D03:05:17.715982000 GS.N 11.53192 48.36199 9 4
2019-06-18D03:05:19.715528000 MSFT.O 7.836585 7.540665 1 8
2019-06-18D03:05:20.715186000 MSFT.O 24.88246 30.54409 4 2
2019-06-18D03:05:22.714657000 IBM.N 12.80329 2.876258 8 7
2019-06-18D03:05:24.714268000 IBM.N 47.99482 5.434121 7 3
2019-06-18D03:05:27.703161000 GS.N 21.64267 31.62057 8 2
2019-06-18D03:05:32.703864000 MSFT.O 24.22364 38.8941 6 6
2019-06-18D03:05:36.702931000 GS.N 38.3743 21.11112 7 8
2019-06-18D03:05:42.717138000 IBM.N 39.67756 44.86179 5 0
2019-06-18D03:05:44.716618000 GS.N 38.85652 31.78591 4 3
2019-06-18D03:05:45.716420000 MSFT.O 45.41356 32.07488 4 8
2019-06-18D03:05:48.710737000 MSFT.O 36.2474 28.09518 4 4
2019-06-18D03:05:50.704024000 IBM.N 4.991336 40.98007 5 7
2019-06-18D03:05:51.703557000 MSFT.O 15.0702 48.37882 1 2
2019-06-18D03:05:53.703353000 GS.N 15.57182 43.24631 2 8
2019-06-18D03:05:56.711853000 BA.N 32.84367 43.62513 6 9
2019-06-18D03:05:57.711573000 GS.N 18.07572 17.7519 4 8
2019-06-18D03:05:58.711376000 MSFT.O 16.69403 26.14589 5 4
2019-06-18D03:05:59.711168000 BA.N 47.58373 17.07496 4 0
2019-06-18D03:06:01.710647000 BA.N 39.79765 9.314345 0 9
2019-06-18D03:06:03.703738000 BA.N 9.363172 15.93321 5 1
2019-06-18D03:06:04.703629000 GS.N 0.1092236 35.46211 7 3
2019-06-18D03:06:06.703179000 GS.N 19.61625 17.64781 9 0
2019-06-18D03:06:07.702876000 BA.N 22.59818 46.28021 2 7
2019-06-18D03:06:09.702319000 BA.N 5.239721 0.8342128 4 1
2019-06-18D03:06:10.702161000 GS.N 15.04851 0.2208297 6 4
2019-06-18D03:06:12.717323000 MSFT.O 46.37225 38.16163 4 8
2019-06-18D03:06:14.716694000 BA.N 1.405337 32.67986 2 1
2019-06-18D03:06:16.716342000 MSFT.O 24.18711 32.5099 6 1
2019-06-18D03:06:18.715898000 IBM.N 23.07442 31.69767 1 5
2019-06-18D03:06:19.715737000 GS.N 36.08687 28.26131 1 4
2019-06-18D03:06:20.715418000 BA.N 30.32804 14.53779 5 1
2019-06-18D03:06:23.714658000 GS.N 15.82207 13.62928 2 9
2019-06-18D03:06:24.714425000 MSFT.O 38.29525 29.69318 0 4
2019-06-18D03:06:27.713770000 BA.N 31.07077 42.14983 7 4
2019-06-18D03:06:33.185300000 MSFT.O 12.00385 7.485022 5 2
2019-06-18D03:06:33.712352000 BA.N 8.324299 10.96574 6 7
2019-06-18D03:06:36.711239000 IBM.N 18.81573 44.36463 2 3
2019-06-18D03:06:37.711231000 MSFT.O 31.61489 20.45336 1 3
2019-06-18D03:06:39.706445000 MSFT.O 4.038664 23.12236 6 9
2019-06-18D03:06:40.706252000 MSFT.O 24.2475 46.81004 5 6
2019-06-18D03:06:42.705819000 BA.N 6.383121 9.113048 0 5
2019-06-18D03:06:43.705597000 BA.N 26.9158 22.50462 3 8
2019-06-18D03:06:44.705333000 GS.N 11.85489 12.08228 1 2
2019-06-18D03:06:47.704608000 BA.N 13.23763 12.97067 5 5
2019-06-18D03:06:49.704167000 IBM.N 43.99847 31.0608 5 7
2019-06-18D03:06:52.703447000 MSFT.O 39.60351 33.07996 4 5
2019-06-18D03:06:53.703062000 MSFT.O 6.65758 46.37964 3 4
2019-06-18D03:06:55.702757000 GS.N 34.01953 36.8386 9 8
2019-06-18D03:06:57.702259000 IBM.N 23.66585 27.28721 8 6
2019-06-18D03:07:00.717191000 GS.N 4.775345 35.20805 5 6
2019-06-18D03:07:01.716814000 BA.N 32.64634 46.93086 0 6
2019-06-18D03:07:02.716682000 GS.N 15.7824 9.282377 0 9
2019-06-18D03:07:03.716454000 IBM.N 28.97735 47.92627 5 1
2019-06-18D03:07:04.716242000 MSFT.O 6.41355 38.08624 3 2
2019-06-18D03:07:07.715561000 GS.N 1.062656 12.47356 9 8
2019-06-18D03:07:11.714443000 IBM.N 3.415027 14.99222 2 6
2019-06-18D03:07:12.714438000 IBM.N 34.22852 31.04312 7 4
2019-06-18D03:07:15.709614000 MSFT.O 8.238095 28.05404 7 5
2019-06-18D03:07:16.716735000 GS.N 0.9810431 21.17852 1 5
2019-06-18D03:07:17.716585000 IBM.N 12.26745 8.909183 9 9
2019-06-18D03:07:23.715263000 GS.N 39.31552 13.63137 0 1
2019-06-18D03:07:26.714542000 IBM.N 5.039161 10.74923 1 0
2019-06-18D03:07:27.714268000 BA.N 38.01613 40.07364 7 7
2019-06-18D03:07:28.714092000 BA.N 9.187076 21.18061 0 2
2019-06-18D03:07:32.712777000 GS.N 14.56379 27.17526 6 2
2019-06-18D03:07:35.711844000 MSFT.O 27.55229 3.059558 2 0
2019-06-18D03:07:37.711543000 GS.N 12.33477 40.97651 3 6
2019-06-18D03:07:38.711250000 IBM.N 21.45296 28.87573 9 4
7 changes: 7 additions & 0 deletions gitSectionA/sym.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
// schema.q
// Quote Table Schema
quote:([]time:`timestamp$();sym:`symbol$();bid:`float$();ask:`float$();bsize:`long$();asize:`long$());
// trade Table Schema
trade:([]time:`timestamp$();sym:`symbol$();price:`float$();size:`long$());
// Aggregation Table Schema
aggTrade:([sym:`symbol$()];time:`timestamp$();maxTPrice:`float$();minTPice:`float$();tVolume:`long$();bestBid:`float$();bestAsk:`float$());
Loading