Performance improvement in streaming CESR over HTTP.

scripts/demo/basic/essr.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#kli init --name sender --salt 0ACDEyMzQ1Njc4OWxtbm9aBc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis
+#kli incept --name sender --alias sender --file ${KERI_DEMO_SCRIPT_DIR}/data/gleif-sample.json
+#
+#kli init --name recipient --salt 0ACDEyMzQ1Njc4OWxtbm9qWc --nopasscode --config-dir ${KERI_SCRIPT_DIR} --config-file demo-witness-oobis
+#kli incept --name recipient --alias recipient --file ${KERI_DEMO_SCRIPT_DIR}/data/gleif-sample.json
+#
+#kli oobi resolve --name sender --oobi-alias recipient --oobi http://127.0.0.1:5642/oobi/EFXJsTFSo10FAZGR-8_Uw1DlhU8nuRFOAN9Z8ajJ56ci/witness
+#kli oobi resolve --name recipient --oobi-alias sender --oobi http://127.0.0.1:5642/oobi/EJf7MfzNmehwY5310MUWXPSxhAA_3ifPW2bdsjwqnvae/witness
+#
+#kli vc registry incept --name sender --alias sender --registry-name vLEI
+#
+#kli vc create --name sender --alias sender --registry-name vLEI --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao --recipient ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k --data @${KERI_DEMO_SCRIPT_DIR}/data/credential-data.json
+#SAID=$(kli vc list --name sender --alias sender --issued --said --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao)
+#
+#kli ipex grant --name sender --alias sender --said "${SAID}" --recipient ELjSFdrTdCebJlmvbFNX9-TLhR2PO0_60al1kQp5_e6k
+
+kli essr send --name sender --alias sender --recipient recipient

scripts/demo/vLEI/issue-xbrl-attestation.sh

@@ -52,42 +52,66 @@ kli vc registry incept --name legal-entity --alias legal-entity --registry-name
 kli vc registry incept --name person  --passcode DoB26Fj4x9LboAFWJra17O --alias person --registry-name vLEI-person
 
 # Issue QVI credential vLEI from GLEIF External to QVI
-kli vc issue --name external --alias external --registry-name vLEI-external --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX --data @${KERI_DEMO_SCRIPT_DIR}/data/qvi-data.json
-kli vc list --name qvi --alias qvi --poll
+kli vc create --name external --alias external --registry-name vLEI-external --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX --data @${KERI_DEMO_SCRIPT_DIR}/data/qvi-data.json
+SAID=$(kli vc list --name external --alias external --issued --said --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao)
+kli ipex grant --name external --alias external --said "${SAID}" --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX
+GRANT=$(kli ipex list --name qvi --alias qvi --poll --said)
+kli ipex admit --name qvi --alias qvi --said "${GRANT}"
+kli vc list --name qvi --alias qvi
 
 # Issue LE credential from QVI to Legal Entity - have to create the edges first
 QVI_SAID=`kli vc list --name qvi --alias qvi --said --schema EBfdlu8R27Fbx-ehrqwImnK-8Cm79sqbAQ4MmvEAYqao`
 echo \"$QVI_SAID\" | jq -f ${KERI_DEMO_SCRIPT_DIR}/data/legal-entity-edges-filter.jq  > /tmp/legal-entity-edges.json
 kli saidify --file /tmp/legal-entity-edges.json
-kli vc issue --name qvi --alias qvi --registry-name vLEI-qvi --schema ENPXp1vQzRF6JwIuS-mp2U8Uf1MoADoP_GqQ62VsDZWY --recipient EIitNxxiNFXC1HDcPygyfyv3KUlBfS_Zf-ZYOvwjpTuz --data @${KERI_DEMO_SCRIPT_DIR}/data/legal-entity-data.json --edges @/tmp/legal-entity-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/rules.json
-kli vc list --name legal-entity --alias legal-entity --poll
+kli vc create --name qvi --alias qvi --registry-name vLEI-qvi --schema ENPXp1vQzRF6JwIuS-mp2U8Uf1MoADoP_GqQ62VsDZWY --recipient EIitNxxiNFXC1HDcPygyfyv3KUlBfS_Zf-ZYOvwjpTuz --data @${KERI_DEMO_SCRIPT_DIR}/data/legal-entity-data.json --edges @/tmp/legal-entity-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/rules.json
+SAID=$(kli vc list --name qvi --alias qvi --issued --said --schema ENPXp1vQzRF6JwIuS-mp2U8Uf1MoADoP_GqQ62VsDZWY)
+kli ipex grant --name qvi --alias qvi --said "${SAID}" --recipient EIitNxxiNFXC1HDcPygyfyv3KUlBfS_Zf-ZYOvwjpTuz
+GRANT=$(kli ipex list --name legal-entity --alias legal-entity --poll --said)
+kli ipex admit --name legal-entity --alias legal-entity --said "${GRANT}"
+kli vc list --name legal-entity --alias legal-entity
 
 # Issue ECR Authorization credential from Legal Entity to QVI - have to create the edges first
 LE_SAID=`kli vc list --name legal-entity --alias legal-entity --said`
 echo \"$LE_SAID\" | jq -f ${KERI_DEMO_SCRIPT_DIR}/data/ecr-auth-edges-filter.jq > /tmp/ecr-auth-edges.json
 kli saidify --file /tmp/ecr-auth-edges.json
-kli vc issue --name legal-entity --alias legal-entity --registry-name vLEI-legal-entity --schema EH6ekLjSr8V32WyFbGe1zXjTzFs9PkTYmupJ9H65O14g --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX --data @${KERI_DEMO_SCRIPT_DIR}/data/ecr-auth-data.json --edges @/tmp/ecr-auth-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/ecr-auth-rules.json
-kli vc list --name qvi --alias qvi --poll
+kli vc create --name legal-entity --alias legal-entity --registry-name vLEI-legal-entity --schema EH6ekLjSr8V32WyFbGe1zXjTzFs9PkTYmupJ9H65O14g --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX --data @${KERI_DEMO_SCRIPT_DIR}/data/ecr-auth-data.json --edges @/tmp/ecr-auth-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/ecr-auth-rules.json
+SAID=$(kli vc list --name legal-entity --alias legal-entity --issued --said --schema EH6ekLjSr8V32WyFbGe1zXjTzFs9PkTYmupJ9H65O14g)
+kli ipex grant --name legal-entity --alias legal-entity --said "${SAID}" --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX
+GRANT=$(kli ipex list --name qvi --alias qvi --poll --said | tail -1)
+kli ipex admit --name qvi --alias qvi --said "${GRANT}"
+kli vc list --name qvi --alias qvi
 
 # Issue ECR credential from QVI to Person
 AUTH_SAID=`kli vc list --name qvi --alias qvi --said --schema EH6ekLjSr8V32WyFbGe1zXjTzFs9PkTYmupJ9H65O14g`
 echo "[\"$QVI_SAID\", \"$AUTH_SAID\"]" | jq -f ${KERI_DEMO_SCRIPT_DIR}/data/ecr-edges-filter.jq > /tmp/ecr-edges.json
 kli saidify --file /tmp/ecr-edges.json
-kli vc issue --name qvi --alias qvi --private --registry-name vLEI-qvi --schema EEy9PkikFcANV1l7EHukCeXqrzT1hNZjGlUk7wuMO5jw --recipient EKE7b7owCvObR6dBTrU7w38_oATL9Tcrp_-xjPn05zYe --data @${KERI_DEMO_SCRIPT_DIR}/data/ecr-data.json --edges @/tmp/ecr-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/ecr-rules.json
-kli vc list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --poll
+kli vc create --name qvi --alias qvi --private --registry-name vLEI-qvi --schema EEy9PkikFcANV1l7EHukCeXqrzT1hNZjGlUk7wuMO5jw --recipient EKE7b7owCvObR6dBTrU7w38_oATL9Tcrp_-xjPn05zYe --data @${KERI_DEMO_SCRIPT_DIR}/data/ecr-data.json --edges @/tmp/ecr-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/ecr-rules.json
+SAID=$(kli vc list --name qvi --alias qvi --issued --said --schema EEy9PkikFcANV1l7EHukCeXqrzT1hNZjGlUk7wuMO5jw)
+kli ipex grant --name qvi --alias qvi --said "${SAID}" --recipient EKE7b7owCvObR6dBTrU7w38_oATL9Tcrp_-xjPn05zYe
+GRANT=$(kli ipex list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --poll --said)
+kli ipex admit --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --said "${GRANT}"
+kli vc list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O
 
 # Issue OOR Authorization credential from Legal Entity to QVI - have to create the edges first
 echo \"$LE_SAID\" | jq -f ${KERI_DEMO_SCRIPT_DIR}/data/oor-auth-edges-filter.jq > /tmp/oor-auth-edges.json
 kli saidify --file /tmp/oor-auth-edges.json
-kli vc issue --name legal-entity --alias legal-entity --registry-name vLEI-legal-entity --schema EKA57bKBKxr_kN7iN5i7lMUxpMG-s19dRcmov1iDxz-E --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX --data @${KERI_DEMO_SCRIPT_DIR}/data/oor-auth-data.json --edges @/tmp/oor-auth-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/rules.json
-kli vc list --name qvi --alias qvi --poll
+kli vc create --name legal-entity --alias legal-entity --registry-name vLEI-legal-entity --schema EKA57bKBKxr_kN7iN5i7lMUxpMG-s19dRcmov1iDxz-E --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX --data @${KERI_DEMO_SCRIPT_DIR}/data/oor-auth-data.json --edges @/tmp/oor-auth-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/rules.json
+SAID=$(kli vc list --name legal-entity --alias legal-entity --issued --said --schema EKA57bKBKxr_kN7iN5i7lMUxpMG-s19dRcmov1iDxz-E)
+kli ipex grant --name legal-entity --alias legal-entity --said "${SAID}" --recipient EHMnCf8_nIemuPx-cUHaDQq8zSnQIFAurdEpwHpNbnvX
+GRANT=$(kli ipex list --name qvi --alias qvi --poll --said | tail -1)
+kli ipex admit --name qvi --alias qvi --said "${GRANT}"
+kli vc list --name qvi --alias qvi
 
 # Issue OOR credential from QVI to Person
 AUTH_SAID=`kli vc list --name qvi --alias qvi --said --schema EKA57bKBKxr_kN7iN5i7lMUxpMG-s19dRcmov1iDxz-E`
 echo "[\"$QVI_SAID\", \"$AUTH_SAID\"]" | jq -f ${KERI_DEMO_SCRIPT_DIR}/data/oor-edges-filter.jq > /tmp/oor-edges.json
 kli saidify --file /tmp/oor-edges.json
-kli vc issue --name qvi --alias qvi --registry-name vLEI-qvi --schema EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy --recipient EKE7b7owCvObR6dBTrU7w38_oATL9Tcrp_-xjPn05zYe --data @${KERI_DEMO_SCRIPT_DIR}/data/oor-data.json --edges @/tmp/oor-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/rules.json
-kli vc list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --poll
+kli vc create --name qvi --alias qvi --registry-name vLEI-qvi --schema EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy --recipient EKE7b7owCvObR6dBTrU7w38_oATL9Tcrp_-xjPn05zYe --data @${KERI_DEMO_SCRIPT_DIR}/data/oor-data.json --edges @/tmp/oor-edges.json --rules @${KERI_DEMO_SCRIPT_DIR}/data/rules.json
+SAID=$(kli vc list --name qvi --alias qvi --issued --said --schema EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy)
+kli ipex grant --name qvi --alias qvi --said "${SAID}" --recipient EKE7b7owCvObR6dBTrU7w38_oATL9Tcrp_-xjPn05zYe
+GRANT=$(kli ipex list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --poll --said | tail -1)
+kli ipex admit --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --said "${GRANT}"
+kli vc list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O
 
 # Issue iXBRL data attestation from Person
 OOR_SAID=`kli vc list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --said --schema EBNaNu-M9P5cgrnfl2Fvymy4E_jvxxyjb70PRtiANlJy`
@@ -96,5 +120,5 @@ kli saidify --file /tmp/xbrl-edges.json
 NOW=`date -u +"%Y-%m-%dT%H:%M:%S+00:00"`
 echo \"$NOW\" | jq -f ${KERI_DEMO_SCRIPT_DIR}/data/xbrl-data.jq > /tmp/xbrl-data.json
 kli saidify --file /tmp/xbrl-data.json
-kli vc issue --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --registry-name vLEI-person --schema EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi --data @/tmp/xbrl-data.json --edges @/tmp/xbrl-edges.json
+kli vc create --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --registry-name vLEI-person --schema EMhvwOlyEJ9kN4PrwCpr9Jsv7TxPhiYveZ0oP3lJzdEi --data @/tmp/xbrl-data.json --edges @/tmp/xbrl-edges.json
 kli vc list --name person --alias person --passcode DoB26Fj4x9LboAFWJra17O --issued

src/keri/app/agenting.py

@@ -10,7 +10,7 @@
 from hio.base import doing
 from hio.core import http
 from hio.core.tcp import clienting
-from hio.help import decking
+from hio.help import decking, Hict
 
 from . import httping, forwarding
 from .. import help
@@ -69,7 +69,6 @@ def receipt(self, pre, sn=None):
         if ser.ked['t'] in (coring.Ilks.rot,):
             adds = ser.ked["ba"]
             for wit in adds:
-                print(f"catching up {wit}")
                 yield from self.catchup(ser.pre, wit)
 
         clients = dict()
@@ -722,6 +721,100 @@ def idle(self):
         return len(self.sent) == self.posted
 
 
+class TCPStreamMessenger(doing.DoDoer):
+    """ Send events to witnesses for receipting using TCP direct connection
+
+    """
+
+    def __init__(self, hab, wit, url, msgs=None, sent=None, doers=None, **kwa):
+        """
+        For the current event, gather the current set of witnesses, send the event,
+        gather all receipts and send them to all other witnesses
+
+        Parameters:
+            hab: Habitat of the identifier to populate witnesses
+
+        """
+        self.hab = hab
+        self.wit = wit
+        self.url = url
+        self.posted = 0
+        self.msgs = msgs if msgs is not None else decking.Deck()
+        self.sent = sent if sent is not None else decking.Deck()
+        self.parser = None
+        doers = doers if doers is not None else []
+        doers.extend([doing.doify(self.receiptDo)])
+
+        self.kevery = eventing.Kevery(db=self.hab.db,
+                                      **kwa)
+
+        super(TCPStreamMessenger, self).__init__(doers=doers)
+
+    def receiptDo(self, tymth=None, tock=0.0):
+        """
+        Returns doifiable Doist compatible generator method (doer dog)
+
+        Usage:
+            add result of doify on this method to doers list
+        """
+        self.wind(tymth)
+        self.tock = tock
+        _ = (yield self.tock)
+
+        up = urlparse(self.url)
+        if up.scheme != kering.Schemes.tcp:
+            raise ValueError(f"invalid scheme {up.scheme} for TcpWitnesser")
+
+        client = clienting.Client(host=up.hostname, port=up.port)
+        self.parser = parsing.Parser(ims=client.rxbs,
+                                     framed=True,
+                                     kvy=self.kevery)
+
+        clientDoer = clienting.ClientDoer(client=client)
+        self.extend([clientDoer, doing.doify(self.msgDo)])
+
+        while True:
+            while not self.msgs:
+                yield self.tock
+
+            msg = self.msgs.popleft()
+            self.posted += 1
+
+            client.tx(msg)  # send to connected remote
+
+            while client.txbs:
+                yield self.tock
+
+            self.sent.append(msg)
+            yield self.tock
+
+    def msgDo(self, tymth=None, tock=0.0, **opts):
+        """
+        Returns doifiable Doist compatible generator method (doer dog) to process
+            incoming message stream of .kevery
+
+        Doist Injected Attributes:
+            g.tock = tock  # default tock attributes
+            g.done = None  # default done state
+            g.opts
+
+        Parameters:
+            tymth is injected function wrapper closure returned by .tymen() of
+                Tymist instance. Calling tymth() returns associated Tymist .tyme.
+            tock is injected initial tock value
+            opts is dict of injected optional additional parameters
+
+
+        Usage:
+            add result of doify on this method to doers list
+        """
+        yield from self.parser.parsator()  # process messages continuously
+
+    @property
+    def idle(self):
+        return len(self.sent) == self.posted
+
+
 class HTTPMessenger(doing.DoDoer):
     """
     Interacts with Recipients on HTTP and SSE for sending events and receiving receipts
@@ -800,6 +893,65 @@ def idle(self):
         return len(self.msgs) == 0 and self.posted == len(self.sent)
 
 
+class HTTPStreamMessenger(doing.DoDoer):
+    """
+    Interacts with Recipients on HTTP and SSE for sending events and receiving receipts
+
+    """
+
+    def __init__(self, hab, wit, url, msg=b'', headers=None, **kwa):
+        """
+        For the current event, gather the current set of witnesses, send the event,
+        gather all receipts and send them to all other witnesses
+
+        Parameters:
+            hab: Habitat of the identifier to populate witnesses
+
+        """
+        self.hab = hab
+        self.wit = wit
+        self.rep = None
+        headers = headers if headers is not None else {}
+
+        up = urlparse(url)
+        if up.scheme != kering.Schemes.http and up.scheme != kering.Schemes.https:
+            raise ValueError(f"invalid scheme {up.scheme} for HTTPMessenger")
+
+        self.client = http.clienting.Client(scheme=up.scheme, hostname=up.hostname, port=up.port)
+        clientDoer = http.clienting.ClientDoer(client=self.client)
+
+        headers = Hict([
+            ("Content-Type", "application/cesr"),
+            ("Content-Length", len(msg)),
+            (httping.CESR_DESTINATION_HEADER, self.wit),
+        ] + list(headers.items()))
+
+        self.client.request(
+            method="PUT",
+            path="/",
+            headers=headers,
+            body=bytes(msg)
+        )
+
+        doers = [clientDoer]
+
+        super(HTTPStreamMessenger, self).__init__(doers=doers, **kwa)
+
+    def recur(self, tyme, deeds=None):
+        """
+        Returns doifiable Doist compatible generator method (doer dog)
+
+        Usage:
+            add result of doify on this method to doers list
+        """
+        if self.client.responses:
+            self.rep = self.client.respond()
+            self.remove([self.client])
+            return True
+
+        return super(HTTPStreamMessenger, self).recur(tyme, deeds)
+
+
 def mailbox(hab, cid):
     for (_, erole, eid), end in hab.db.ends.getItemIter(keys=(cid, kering.Roles.mailbox)):
         if end.allowed:
@@ -853,6 +1005,31 @@ def messengerFrom(hab, pre, urls):
     return witer
 
 
+def streamMessengerFrom(hab, pre, urls, msg, headers=None):
+    """ Create a Witnesser (tcp or http) based on provided endpoints
+
+    Parameters:
+        hab (Habitat): Environment to use to look up witness URLs
+        pre (str): qb64 identifier prefix of recipient to create a messanger for
+        urls (dict): map of schemes to urls of available endpoints
+        msg (bytes): bytes of message to send
+        headers (dict): optional headers to send with HTTP requests
+
+    Returns:
+        Optional(TcpWitnesser, HTTPMessenger): witnesser for ensuring full reciepts
+    """
+    if kering.Schemes.http in urls or kering.Schemes.https in urls:
+        url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https]
+        witer = HTTPStreamMessenger(hab=hab, wit=pre, url=url, msg=msg, headers=headers)
+    elif kering.Schemes.tcp in urls:
+        url = urls[kering.Schemes.tcp]
+        witer = TCPStreamMessenger(hab=hab, wit=pre, url=url)
+    else:
+        raise kering.ConfigurationError(f"unable to find a valid endpoint for witness {pre}")
+
+    return witer
+
+
 def httpClient(hab, wit):
     """ Create and return a http.client and http.ClientDoer for the witness
 

src/keri/app/cli/commands/ends/add.py

@@ -33,7 +33,7 @@
 parser.add_argument("--eid", "-e", help="qualified base64 of AID to authorize with new role for the AID identified "
                                         "by alias",
                     required=True)
-parser.add_argument("--time", help="timestamp for the revocation", required=False, default=None)
+parser.add_argument("--time", help="timestamp for the end auth", required=False, default=None)
 
 
 def add_end(args):