Skip to content

Commit

Permalink
locli: report improvements: autogen list, pass through workload & tag…
Browse files Browse the repository at this point in the history 
… charts with runs
  • Loading branch information
@deepfire
deepfire committed Nov 14, 2022
1 parent bf311b9 commit 0fbf71b
Show file tree
Hide file tree
Showing 7 changed files with 77 additions and 63 deletions.
64 changes: 32 additions & 32 deletions bench/locli/src/Cardano/Analysis/API.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,11 +340,11 @@ data PropSubset

bpFieldSelectForger :: Field DSelect p a -> Bool
bpFieldSelectForger Field{fId} = elem fId
[ "fStarted", "fLeading", "fForged", "fAnnounced", "fAdopted", "fSendStart" ]
[ "bfStarted", "bfLeading", "bfForged", "bfAnnounced", "fSending", "bfAdopted" ]

bpFieldSelectPeers :: Field DSelect p a -> Bool
bpFieldSelectPeers Field{fId} = elem fId
[ "pNoticed", "pRequested", "pFetched", "pAnnounced", "pAdopted", "pSendStart" ]
[ "boNoticed", "boRequested", "boFetched", "boAnnounced", "boSending", "boAdopted" ]

bpFieldSelectEndToEnd :: Field DSelect p a -> Bool
bpFieldSelectEndToEnd Field{fHead2} = elem fHead2 adoptionCentilesRendered
Expand Down Expand Up @@ -449,37 +449,37 @@ instance CDFFields BlockProp p where
instance TimelineFields BlockEvents where
timelineFields _ =
-- Width LeftPad
[ Field 5 0 "block" "block" "no." (IWord64 (unBlockNo . beBlockNo)) ""
, Field 5 0 "abs.slot" "abs." "slot#" (IWord64 (unSlotNo . beSlotNo)) ""
, Field 6 0 "hash" "block" "hash" (IText (shortHash . beBlock)) ""
, Field 6 0 "hashPrev" "prev" "hash" (IText (shortHash . beBlockPrev)) ""
, Field 7 0 "forger" "forger" "host" (IText (toText . unHost . bfForger . beForge)) ""
, Field 6 0 "blockSize" "size" "bytes" (IInt (bfBlockSize . beForge)) ""
, Field 5 0 "blockGap" "block" "gap" (IDeltaT (bfBlockGap . beForge)) ""
, Field 3 0 "forks" "for" "-ks" (IInt (count bpeIsFork . beErrors)) ""
, Field 4 0 "fStarted" (f!!0) "Start" (IDeltaT (bfStarted . beForge)) ""
, Field 4 0 "fBlkCtx" (f!!1) "BlkCtx" (IText (maybe "?" show.bfBlkCtx .beForge)) ""
, Field 4 0 "fLgrState" (f!!2) "LgrSta" (IText (maybe "?" show.bfLgrState.beForge)) ""
, Field 4 0 "fLgrView" (f!!3) "LgrVie" (IText (maybe "?" show.bfLgrView .beForge)) ""
, Field 4 0 "fLeading" (f!!4) "Lead" (IDeltaT (bfLeading . beForge)) ""
, Field 4 0 "fForged" (f!!5) "Forge" (IDeltaT (bfForged . beForge)) ""
, Field 4 0 "fAnnounced" (f!!6) "Announ" (IDeltaT (bfAnnounced . beForge)) ""
, Field 4 0 "fSendStart" (f!!7) "Sendin" (IDeltaT (bfSending . beForge)) ""
, Field 4 0 "fAdopted" (f!!8) "Adopt" (IDeltaT (bfAdopted . beForge)) ""
, Field 4 0 "noticedVal" (p!!0) "Notic" (IDeltaT (af boNoticed . valids)) ""
, Field 4 0 "requestedVal" (p!!1) "Requd" (IDeltaT (af boRequested . valids)) ""
, Field 4 0 "fetchedVal" (p!!2) "Fetch" (IDeltaT (af boFetched . valids)) ""
, Field 4 0 "pAnnouncedVal" (p!!3) "Annou" (IDeltaT (af' boAnnounced . valids)) ""
, Field 4 0 "pSendStartVal" (p!!4) "Send" (IDeltaT (af' boSending . valids)) ""
, Field 4 0 "pAdoptedVal" (p!!5) "Adopt" (IDeltaT (af' boAdopted . valids)) ""
, Field 4 0 "pPropag0.5" (r!!0) "0.5" (IDeltaT (percSpec 0.5 . bePropagation)) ""
, Field 4 0 "pPropag0.8" (r!!1) "0.8" (IDeltaT (percSpec 0.8 . bePropagation)) ""
, Field 4 0 "pPropag0.96" (r!!2) "0.96" (IDeltaT (percSpec 0.96 . bePropagation)) ""
, Field 4 0 "pPropag1.0" (r!!3) "1.0" (IDeltaT (snd . cdfRange . bePropagation)) ""
, Field 3 0 "valid" "va-" "lid" (IText (bool "-" "+" . (== 0) . length
[ Field 5 0 "beBlockNo" "block" "no." (IWord64 (unBlockNo . beBlockNo)) ""
, Field 5 0 "beSlotNo" "abs." "slot#" (IWord64 (unSlotNo . beSlotNo)) ""
, Field 6 0 "beBlock" "block" "hash" (IText (shortHash . beBlock)) ""
, Field 6 0 "beBlockPrev" "prev" "hash" (IText (shortHash . beBlockPrev)) ""
, Field 7 0 "bfForger" "forger" "host" (IText (toText . unHost . bfForger . beForge)) ""
, Field 6 0 "bfBlockSize" "size" "bytes" (IInt (bfBlockSize . beForge)) ""
, Field 5 0 "bfBlockGap" "block" "gap" (IDeltaT (bfBlockGap . beForge)) ""
, Field 3 0 "bpeIsFork" "for" "-ks" (IInt (count bpeIsFork . beErrors)) ""
, Field 4 0 "bfStarted" (f!!0) "Start" (IDeltaT (bfStarted . beForge)) ""
, Field 4 0 "bfBlkCtx" (f!!1) "BlkCtx" (IText (maybe "?" show.bfBlkCtx .beForge)) ""
, Field 4 0 "bfLgrState" (f!!2) "LgrSta" (IText (maybe "?" show.bfLgrState.beForge)) ""
, Field 4 0 "bfLgrView" (f!!3) "LgrVie" (IText (maybe "?" show.bfLgrView .beForge)) ""
, Field 4 0 "bfLeading" (f!!4) "Lead" (IDeltaT (bfLeading . beForge)) ""
, Field 4 0 "bfForged" (f!!5) "Forge" (IDeltaT (bfForged . beForge)) ""
, Field 4 0 "bfAnnounced" (f!!6) "Announ" (IDeltaT (bfAnnounced . beForge)) ""
, Field 4 0 "bfSending" (f!!7) "Sendin" (IDeltaT (bfSending . beForge)) ""
, Field 4 0 "bfAdopted" (f!!8) "Adopt" (IDeltaT (bfAdopted . beForge)) ""
, Field 4 0 "boNoticed" (p!!0) "Notic" (IDeltaT (af boNoticed . valids)) ""
, Field 4 0 "boRequested" (p!!1) "Requd" (IDeltaT (af boRequested . valids)) ""
, Field 4 0 "boFetched" (p!!2) "Fetch" (IDeltaT (af boFetched . valids)) ""
, Field 4 0 "boAnnounced" (p!!3) "Annou" (IDeltaT (af' boAnnounced . valids)) ""
, Field 4 0 "boSending" (p!!4) "Send" (IDeltaT (af' boSending . valids)) ""
, Field 4 0 "pAdopted" (p!!5) "Adopt" (IDeltaT (af' boAdopted . valids)) ""
, Field 4 0 "bePropagation.5" (r!!0) "0.5" (IDeltaT (percSpec 0.5 . bePropagation)) ""
, Field 4 0 "bePropagation.8" (r!!1) "0.8" (IDeltaT (percSpec 0.8 . bePropagation)) ""
, Field 4 0 "bePropagation.96"(r!!2) "0.96" (IDeltaT (percSpec 0.96 . bePropagation)) ""
, Field 4 0 "bePropagation.1" (r!!3) "1.0" (IDeltaT (snd . cdfRange . bePropagation)) ""
, Field 3 0 "beAcceptance" "va-" "lid" (IText (bool "-" "+" . (== 0) . length
. filter (not . snd) . beAcceptance)) ""
, Field 3 0 "valid.observ" "good" "obsv" (IInt (length . valids)) ""
, Field 5 0 "errors" "all" "errs" (IInt (length . beErrors)) ""
, Field 5 0 "beErrors" "all" "errs" (IInt (length . beErrors)) ""
, Field 3 0 "missNotic" (m!!0) "ntc" (IInt (count (bpeIsMissing Notice) . beErrors)) ""
, Field 3 0 "missReque" (m!!1) "req" (IInt (count (bpeIsMissing Request) . beErrors)) ""
, Field 3 0 "missFetch" (m!!2) "fch" (IInt (count (bpeIsMissing Fetch) . beErrors)) ""
Expand Down Expand Up @@ -538,7 +538,7 @@ data PerfSubset

mtFieldsReport :: Field DSelect p a -> Bool
mtFieldsReport Field{fId} = elem fId
[ "cpuProcess", "cpuGC", "cpuMutator", "cpuSpanLenAll", "memRSS", "rtsHeap", "rtsLive", "rtsAllocation" ]
[ "rCentiCpu", "rCentiGC", "rCentiMut", "cdfSpanLensCpu", "rRSS", "rHeap", "rLive", "rAlloc" ]

perfSubsetFn :: PerfSubset -> (Field DSelect p a -> Bool)
perfSubsetFn = \case
Expand Down
12 changes: 7 additions & 5 deletions bench/locli/src/Cardano/Analysis/Context.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,13 @@ data PParams

data GeneratorProfile
= GeneratorProfile
{ add_tx_size :: Word64
, inputs_per_tx :: Word64
, outputs_per_tx :: Word64
, tps :: Double
, tx_count :: Word64
{ add_tx_size :: Word64
, inputs_per_tx :: Word64
, outputs_per_tx :: Word64
, tps :: Double
, tx_count :: Word64
, plutusMode :: Maybe Bool
, plutusLoopScript :: Maybe FilePath
}
deriving (Generic, Show, FromJSON, ToJSON)

Expand Down
36 changes: 24 additions & 12 deletions bench/locli/src/Cardano/Report.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ import Data.List (last)
import Data.Text qualified as T
import Data.Text.Lazy qualified as LT
import Data.Time.Clock
import System.FilePath as FS
import System.Posix.User

import Text.EDE
import Text.EDE hiding (Id)

import Data.CDF

Expand Down Expand Up @@ -61,18 +62,20 @@ getReport rTarget mrev = do

data Workload
= WValue
| WPlutus
| WPlutusLoopCountdown
| WPlutusLoopSECP

instance ToJSON Workload where
toJSON = \case
WValue -> "Value"
WPlutus -> "Plutus"
WValue -> "value-only"
WPlutusLoopCountdown -> "Plutus countdown loop"
WPlutusLoopSECP -> "Plutus SECP loop"

data Section where
STable :: CDFFields a p =>
{ sData :: !(a p)
, sFieldSelector :: !(Field DSelect p a -> Bool)
, sDataId :: !Text
, sDataRef :: !Text
, sOrgTableSrc :: !Text
, sTitle :: !Text
} -> Section
Expand Down Expand Up @@ -107,7 +110,7 @@ instance ToJSON TmplRun where
toJSON TmplRun{trManifest=Manifest{..},..} =
object
[ "meta" .= trMeta
, "id" .= trShortId
, "shortId" .= trShortId
, "workload" .= trWorkload
, "branch" .= mNodeBranch
, "ver" .= mNodeApproxVer
Expand All @@ -124,26 +127,33 @@ instance ToJSON TmplRun where
]

liftTmplRun :: Run -> TmplRun
liftTmplRun Run{..} =
liftTmplRun Run{generatorProfile=GeneratorProfile{..}, ..} =
TmplRun
{ trMeta = metadata
, trShortId = ShortId (batch metadata)
, trWorkload = WValue
, trManifest = manifest metadata & unsafeShortenManifest 5
, trWorkload =
case ( plutusMode & fromMaybe False
, plutusLoopScript & fromMaybe "" & FS.takeFileName & FS.dropExtension ) of
(False, _) -> WValue
(True, "loop") -> WPlutusLoopCountdown
(True, "schnorr-secp256k1-loop") -> WPlutusLoopSECP
(_, scr) ->
error $ "Unknown Plutus script: " <> scr
}

data TmplSection
= TmplTable
{ tsTitle :: !Text
, tsDataId :: !Text
, tsDataRef :: !Text
, tsOrgTableSrc :: !Text
, tsNRows :: !Int
}

instance ToJSON TmplSection where
toJSON TmplTable{..} = object
[ "title" .= tsTitle
, "dataId" .= tsDataId
, "dataRef" .= tsDataRef
, "orgFile" .= tsOrgTableSrc
-- Yes, strange as it is, this is the encoding to ease iteration in ED-E.
, "rows" .= T.replicate tsNRows "."
Expand All @@ -155,9 +165,11 @@ liftTmplSection =
STable{..} ->
TmplTable
{ tsTitle = sTitle
, tsDataId = sDataId
, tsDataRef = sDataRef
, tsOrgTableSrc = sOrgTableSrc
, tsNRows = length $ filterFields sFieldSelector
, tsNRows =
-- trace (printf "nrows: %d" $ length $ filterFields sFieldSelector :: String) $
length $ filterFields sFieldSelector
}

generate :: InputDir -> Maybe TextInputFile
Expand Down
4 changes: 2 additions & 2 deletions nix/workbench/ede/chart.ede
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ set yrange [*:*]
set ylabel "{{ args.ylabel }}"
{% endif %}
eval cdfI_{{ (runs | length) + 1 }}("{{ args.metric }}", "{{ args.title }}", \
"{{ base.ver }}/{{ base.meta.era | toTitle }}", "{{ base.meta.tag }}", \
"{{ base.meta.tag }}/{{ base.shortId }}/{{ base.meta.era | toTitle }}", "{{ base.meta.tag }}", \
{% for run in runs %}
"{{ run.value.ver }}/{{ run.value.meta.era | toTitle }}", "{{ run.value.meta.tag }}"{% if !run.last %},{% endif %} \
"{{ run.value.meta.tag }}/{{ run.value.shortId }}/{{ run.value.meta.era | toTitle }}", "{{ run.value.meta.tag }}"{% if !run.last %},{% endif %} \
{% endfor %}
)
#+end_src
Expand Down
4 changes: 2 additions & 2 deletions nix/workbench/ede/report.ede
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#+CONSTANTS:{% for sec in sections %} {{ sec.value.dataId }}={{ sec.value.orgFile }} {% endfor %}
#+CONSTANTS:{% for sec in sections %} {{ sec.value.dataRef }}={{ sec.value.orgFile }} {% endfor %}
#+CONSTANTS: base=../{{ base.meta.tag }}/analysis
{% for run in runs %}
#+CONSTANTS: run{{ run.index }}=../{{ run.value.meta.tag }}/analysis
Expand All @@ -18,7 +18,7 @@

*** Manifest

We compare ... relative to ={{ base.ver }}= /{{ base.meta.era | toTitle }}, under {{ base.workload }} workload.
We compare {% for run in runs %}{%if !run.first%}{%if !run.last%}, {%else%} and {%endif%}{%endif%}{{ run.value.ver }}/{{ run.value.meta.era | toTitle }}{% endfor %} relative to ={{ base.ver }}=/{{ base.meta.era | toTitle }}, under {{ base.workload }} workload.

{% include "manifest.ede" %}

Expand Down
4 changes: 2 additions & 2 deletions nix/workbench/ede/table.ede
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
| | {% for run in runs %} | | | {% endfor %} |
{% endfor %}
|---------------------------+-----{% for run in runs %}-+------+------+-----{% endfor %}-|
#+TBLFM: $2=remote(file:$base/${{ table.dataId }},@@#$average);p3::$1=remote(file:$base/${{ table.dataId }},@@#$metric)
#+TBLFM: $2=remote(file:$base/${{ table.dataRef }},@@#$average);p3::$1=remote(file:$base/${{ table.dataRef }},@@#$metric)
{% for run in runs %}
#+TBLFM: ${{ run.index * 3 }}=remote(file:$run{{ run.index }}/${{ table.dataId }},@@#$average);p3::${{ run.index * 3 + 1 }}=${{ run.index * 3 }}-$2;p3::${{ run.index * 3 + 2 }}=round(100*${{ run.index * 3 + 1 }}/$2)
#+TBLFM: ${{ run.index * 3 }}=remote(file:$run{{ run.index }}/${{ table.dataRef }},@@#$average);p3::${{ run.index * 3 + 1 }}=${{ run.index * 3 }}-$2;p3::${{ run.index * 3 + 2 }}=round(100*${{ run.index * 3 + 1 }}/$2)
{% endfor %}
16 changes: 8 additions & 8 deletions nix/workbench/run.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ usage_run() {
A unique name would be allocated for this run,
and a run alias $(green current) will be created for it.
$(helpcmd list-aws) List AWS runs
$(blk lsaws)
$(helpcmd fetch-aws RUN)
$(helpcmd list-remote) List AWS runs
$(blk lsaws lsr)
$(helpcmd fetch-run RUN)
Fetch an AWS run
$(blk fetch)
$(helpcmd analysis-from-aws RUN..)
$(blk fetch-analysis fa) Fetch analyses of AWS runs
$(blk fetch fr)
$(helpcmd fetch-analysis RUN..)
$(blk fa) Fetch analyses of AWS runs
$(helpcmd analyse-aws RUN..) Run analyses of AWS runs, remotely
$(helpcmd describe RUN)
Expand Down Expand Up @@ -95,7 +95,7 @@ case "$op" in
sh -c "'$(run_ls_cmd $(jq <<<$r .depl)/runs)'"
fi;;

list-remote | lsr )
list-remote | lsaws | lsr )
run "${sargs[@]}" list --on-remote;;

list-sets | lss )
Expand Down Expand Up @@ -524,7 +524,7 @@ case "$op" in

echo $dir;;

fetch-run | fr )
fetch-run | fetch | fr )
local usage="USAGE: wb run $op RUN [MACHINE] [DEPLOYMENT=bench-1] [ENV=bench]"
local run=${1:?$usage}
local mach=${2:-all-hosts}
Expand Down

0 comments on commit 0fbf71b

Please sign in to comment.