From 0f1c99e898bec75212f91deeee4bec3330bbb18d Mon Sep 17 00:00:00 2001 From: Rodolfo Miranda Date: Wed, 13 Sep 2023 12:05:52 -0300 Subject: [PATCH 1/3] support for https urls --- src/keria/app/agenting.py | 68 ++++++++++++++++++++++----------------- src/keria/app/aiding.py | 21 ++++++++---- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/src/keria/app/agenting.py b/src/keria/app/agenting.py index 466e2658..c26cc3f1 100644 --- a/src/keria/app/agenting.py +++ b/src/keria/app/agenting.py @@ -7,11 +7,12 @@ import json import os from dataclasses import asdict -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin from keri import kering from keri.app.notifying import Notifier from keri.app.storing import Mailboxer +from ordered_set import OrderedSet as oset import falcon from falcon import media @@ -19,7 +20,7 @@ from hio.core import http from hio.help import decking from keri.app import configing, keeping, habbing, storing, signaling, oobiing, agenting, delegating, \ - forwarding, querying, connecting, grouping + forwarding, querying, connecting from keri.app.grouping import Counselor from keri.app.keeping import Algos from keri.core import coring, parsing, eventing, routing @@ -36,8 +37,6 @@ from keri.app import challenging from . import aiding, notifying, indirecting, credentialing, presenting -from . import grouping as keriagrouping -from ..peer import exchanging as keriaexchanging from .specing import AgentSpecResource from ..core import authing, longrunning, httping from ..core.authing import Authenticater @@ -82,8 +81,6 @@ def setup(name, bran, adminPort, bootPort, base='', httpPort=None, configFile=No credentialing.loadEnds(app=app, identifierResource=aidEnd) presenting.loadEnds(app=app) notifying.loadEnds(app=app) - keriagrouping.loadEnds(app=app) - keriaexchanging.loadEnds(app=app) if httpPort: happ = falcon.App(middleware=falcon.CORSMiddleware( @@ -277,13 +274,11 @@ def __init__(self, hby, rgy, agentHab, agency, caid, **opts): signaler = signaling.Signaler() self.notifier = Notifier(hby=hby, signaler=signaler) - self.mux = grouping.Multiplexor(hby=hby, notifier=self.notifier) # Initialize all the credential processors self.verifier = verifying.Verifier(hby=hby, reger=rgy.reger) - self.registrar = credentialing.Registrar(agentHab=agentHab, hby=hby, rgy=rgy, counselor=self.counselor, - witPub=self.witPub, witDoer=self.witDoer, postman=self.postman, - verifier=self.verifier) + self.registrar = credentialing.Registrar(agentHab=agentHab, hby=hby, rgy=rgy, counselor=self.counselor, witPub=self.witPub, + witDoer=self.witDoer, postman=self.postman, verifier=self.verifier) self.credentialer = credentialing.Credentialer(agentHab=agentHab, hby=self.hby, rgy=self.rgy, postman=self.postman, registrar=self.registrar, verifier=self.verifier, notifier=self.notifier) @@ -301,8 +296,7 @@ def __init__(self, hby, rgy, agentHab, agency, caid, **opts): challengeHandler = challenging.ChallengeHandler(db=hby.db, signaler=signaler) handlers = [issueHandler, requestHandler, proofHandler, applyHandler, challengeHandler] - self.exc = exchanging.Exchanger(hby=hby, handlers=handlers) - grouping.loadHandlers(hby=hby, exc=self.exc, mux=self.mux) + self.exc = exchanging.Exchanger(db=hby.db, handlers=handlers) self.rvy = routing.Revery(db=hby.db, cues=self.cues) self.kvy = eventing.Kevery(db=hby.db, @@ -331,7 +325,7 @@ def __init__(self, hby, rgy, agentHab, agency, caid, **opts): Querier(hby=hby, agentHab=agentHab, kvy=self.kvy, queries=self.queries), Escrower(kvy=self.kvy, rgy=self.rgy, rvy=self.rvy, tvy=self.tvy, exc=self.exc, vry=self.verifier, registrar=self.registrar, credentialer=self.credentialer), - ParserDoer(kvy=self.kvy, parser=self.parser), + Messager(kvy=self.kvy, parser=self.parser), Witnesser(receiptor=receiptor, witners=self.witners), Delegator(agentHab=agentHab, swain=self.swain, anchors=self.anchors), GroupRequester(hby=hby, agentHab=agentHab, postman=self.postman, counselor=self.counselor, @@ -370,12 +364,12 @@ def inceptExtern(self, pre, verfers, digers, **kwargs): self.agency.incept(self.caid, pre) -class ParserDoer(doing.Doer): +class Messager(doing.Doer): def __init__(self, kvy, parser): self.kvy = kvy self.parser = parser - super(ParserDoer, self).__init__() + super(Messager, self).__init__() def recur(self, tyme=None): if self.parser.ims: @@ -476,11 +470,28 @@ def recur(self, tyme): sigers = msg["sigers"] ghab = self.hby.habs[serder.pre] + if "smids" in msg: + smids = msg['smids'] + else: + smids = ghab.db.signingMembers(pre=ghab.pre) + + if "rmids" in msg: + rmids = msg['rmids'] + else: + rmids = ghab.db.rotationMembers(pre=ghab.pre) + atc = bytearray() # attachment atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) for siger in sigers: atc.extend(siger.qb64b) + others = list(oset(smids + (rmids or []))) + others.remove(ghab.mhab.pre) # don't send to self + print(f"Sending multisig event to {len(others)} other participants") + for recpt in others: + self.postman.send(hab=self.agentHab, dest=recpt, topic="multisig", serder=serder, + attachment=atc) + prefixer = coring.Prefixer(qb64=serder.pre) seqner = coring.Seqner(sn=serder.sn) saider = coring.Saider(qb64=serder.said) @@ -497,7 +508,7 @@ def __init__(self, hby, agentHab, queries, kvy): self.queries = queries self.kvy = kvy - super(Querier, self).__init__(always=True) + super(Querier, self).__init__() def recur(self, tyme, deeds=None): """ Processes query reqests submitting any on the cue""" @@ -505,14 +516,8 @@ def recur(self, tyme, deeds=None): msg = self.queries.popleft() pre = msg["pre"] - if "sn" in msg: - seqNoDo = querying.SeqNoQuerier(hby=self.hby, hab=self.agentHab, pre=pre, sn=msg["sn"]) - self.extend([seqNoDo]) - elif "anchor" in msg: - pass - else: - qryDo = querying.QueryDoer(hby=self.hby, hab=self.agentHab, pre=pre, kvy=self.kvy) - self.extend([qryDo]) + qryDo = querying.QueryDoer(hby=self.hby, hab=self.agentHab, pre=pre, kvy=self.kvy) + self.extend([qryDo]) return super(Querier, self).recur(tyme, deeds) @@ -889,31 +894,34 @@ def on_get(req, rep, alias): if role in (kering.Roles.witness,): # Fetch URL OOBIs for all witnesses oobis = [] for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise falcon.HTTPNotFound(description=f"unable to query witness {wit}, no http endpoint") + url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(urls[kering.Schemes.http]) - oobis.append(f"http://{up.hostname}:{up.port}/oobi/{hab.pre}/witness/{wit}") + oobis.append(f"{url}oobi/{hab.pre}/witness/{wit}") res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: raise falcon.HTTPNotFound(description=f"unable to query controller {hab.pre}, no http endpoint") + url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(urls[kering.Schemes.http]) - oobis.append(f"http://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + oobis.append(f"{url}oobi/{hab.pre}/controller") res["oobis"] = oobis elif role in (kering.Roles.agent,): oobis = [] - roleUrls = hab.fetchRoleUrls(hab.pre, scheme=kering.Schemes.http, role=kering.Roles.agent) + roleUrls = hab.fetchRoleUrls(hab.pre, scheme=kering.Schemes.http, role=kering.Roles.agent) or hab.fetchRoleUrls(hab.pre, scheme=kering.Schemes.https, role=kering.Roles.agent) if not roleUrls: raise falcon.HTTPNotFound(description=f"unable to query controller {hab.pre}, no http endpoint") for eid, urls in roleUrls['agent'].items(): + url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(urls[kering.Schemes.http]) - oobis.append(f"http://{up.hostname}:{up.port}/oobi/{hab.pre}/agent/{eid}") + oobis.append(f"{url}oobi/{hab.pre}/agent/{eid}") res["oobis"] = oobis else: rep.status = falcon.HTTP_404 diff --git a/src/keria/app/aiding.py b/src/keria/app/aiding.py index 27f3e77d..3dd1eda2 100644 --- a/src/keria/app/aiding.py +++ b/src/keria/app/aiding.py @@ -641,24 +641,26 @@ def on_get(req, rep, name): if role in (kering.Roles.witness,): # Fetch URL OOBIs for all witnesses oobis = [] for wit in hab.kever.wits: - urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) + urls = hab.fetchUrls(eid=wit, scheme=kering.Schemes.http) or hab.fetchUrls(eid=wit, scheme=kering.Schemes.https) if not urls: raise falcon.HTTPNotFound(description=f"unable to query witness {wit}, no http endpoint") - up = urlparse(urls[kering.Schemes.http]) + url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] + up = urlparse(url) oobis.append(f"{kering.Schemes.http}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness/{wit}") res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] - urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) + urls = hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.http) or hab.fetchUrls(eid=hab.pre, scheme=kering.Schemes.https) if not urls: raise falcon.HTTPNotFound(description=f"unable to query controller {hab.pre}, no http endpoint") - up = urlparse(urls[kering.Schemes.http]) + url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] + up = urlparse(url) oobis.append(f"{kering.Schemes.http}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") res["oobis"] = oobis elif role in (kering.Roles.agent,): # Fetch URL OOBIs for all witnesses - roleUrls = hab.fetchRoleUrls(cid=hab.pre, role=kering.Roles.agent, scheme=kering.Schemes.http) + roleUrls = hab.fetchRoleUrls(cid=hab.pre, role=kering.Roles.agent, scheme=kering.Schemes.http) or hab.fetchRoleUrls(cid=hab.pre, role=kering.Roles.agent, scheme=kering.Schemes.https) if kering.Roles.agent not in roleUrls: raise falcon.HTTPNotFound(description=f"unable to query agent roles for {hab.pre}, no http endpoint") @@ -668,9 +670,14 @@ def on_get(req, rep, name): for agent in set(aoobis.keys()): murls = aoobis.naball(agent) for murl in murls: - for url in murl.naball(kering.Schemes.http): + urls = [] + if kering.Schemes.http in murl: + urls.extend(murl.naball(kering.Schemes.http)) + if kering.Schemes.https in murl: + urls.extend(murl.naball(kering.Schemes.https)) + for url in urls: up = urlparse(url) - oobis.append(f"{kering.Schemes.http}://{up.hostname}:{up.port}/oobi/{hab.pre}/agent/{agent}") + oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/agent/{agent}") res["oobis"] = oobis else: From 9af59b5551d77cd8efa317c98126ae7aa1f6d797 Mon Sep 17 00:00:00 2001 From: Rodolfo Miranda Date: Wed, 13 Sep 2023 12:06:23 -0300 Subject: [PATCH 2/3] urljoin --- src/keria/app/agenting.py | 12 ++++++------ src/keria/app/aiding.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/keria/app/agenting.py b/src/keria/app/agenting.py index c26cc3f1..ff7de123 100644 --- a/src/keria/app/agenting.py +++ b/src/keria/app/agenting.py @@ -899,8 +899,8 @@ def on_get(req, rep, alias): raise falcon.HTTPNotFound(description=f"unable to query witness {wit}, no http endpoint") url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(urls[kering.Schemes.http]) - oobis.append(f"{url}oobi/{hab.pre}/witness/{wit}") + up = urlparse(url) + oobis.append(urljoin(up.geturl(),f"/oobi/{hab.pre}/witness/{wit}")) res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] @@ -909,8 +909,8 @@ def on_get(req, rep, alias): raise falcon.HTTPNotFound(description=f"unable to query controller {hab.pre}, no http endpoint") url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(urls[kering.Schemes.http]) - oobis.append(f"{url}oobi/{hab.pre}/controller") + up = urlparse(url) + oobis.append(urljoin(up.geturl(),f"/oobi/{hab.pre}/controller")) res["oobis"] = oobis elif role in (kering.Roles.agent,): oobis = [] @@ -920,8 +920,8 @@ def on_get(req, rep, alias): for eid, urls in roleUrls['agent'].items(): url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] - up = urlparse(urls[kering.Schemes.http]) - oobis.append(f"{url}oobi/{hab.pre}/agent/{eid}") + up = urlparse(url) + oobis.append(urljoin(up.geturl(),f"/oobi/{hab.pre}/agent/{eid}")) res["oobis"] = oobis else: rep.status = falcon.HTTP_404 diff --git a/src/keria/app/aiding.py b/src/keria/app/aiding.py index 3dd1eda2..d1a20db1 100644 --- a/src/keria/app/aiding.py +++ b/src/keria/app/aiding.py @@ -6,7 +6,7 @@ """ import json from dataclasses import asdict -from urllib.parse import urlparse +from urllib.parse import urlparse, urljoin import falcon from keri import kering @@ -647,7 +647,7 @@ def on_get(req, rep, name): url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(url) - oobis.append(f"{kering.Schemes.http}://{up.hostname}:{up.port}/oobi/{hab.pre}/witness/{wit}") + oobis.append(urljoin(up.geturl(),f"/oobi/{hab.pre}/witness/{wit}")) res["oobis"] = oobis elif role in (kering.Roles.controller,): # Fetch any controller URL OOBIs oobis = [] @@ -657,7 +657,7 @@ def on_get(req, rep, name): url = urls[kering.Schemes.http] if kering.Schemes.http in urls else urls[kering.Schemes.https] up = urlparse(url) - oobis.append(f"{kering.Schemes.http}://{up.hostname}:{up.port}/oobi/{hab.pre}/controller") + oobis.append(urljoin(up.geturl(),f"/oobi/{hab.pre}/controller")) res["oobis"] = oobis elif role in (kering.Roles.agent,): # Fetch URL OOBIs for all witnesses roleUrls = hab.fetchRoleUrls(cid=hab.pre, role=kering.Roles.agent, scheme=kering.Schemes.http) or hab.fetchRoleUrls(cid=hab.pre, role=kering.Roles.agent, scheme=kering.Schemes.https) @@ -677,7 +677,7 @@ def on_get(req, rep, name): urls.extend(murl.naball(kering.Schemes.https)) for url in urls: up = urlparse(url) - oobis.append(f"{up.scheme}://{up.hostname}:{up.port}/oobi/{hab.pre}/agent/{agent}") + oobis.append(urljoin(up.geturl(),f"/oobi/{hab.pre}/agent/{agent}")) res["oobis"] = oobis else: From 68f5a990979cda46784070d785d25c402700ff14 Mon Sep 17 00:00:00 2001 From: Rodolfo Miranda Date: Wed, 13 Sep 2023 15:54:24 -0300 Subject: [PATCH 3/3] rebasing --- src/keria/app/agenting.py | 52 +++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/src/keria/app/agenting.py b/src/keria/app/agenting.py index ff7de123..07f1d494 100644 --- a/src/keria/app/agenting.py +++ b/src/keria/app/agenting.py @@ -12,7 +12,6 @@ from keri import kering from keri.app.notifying import Notifier from keri.app.storing import Mailboxer -from ordered_set import OrderedSet as oset import falcon from falcon import media @@ -20,7 +19,7 @@ from hio.core import http from hio.help import decking from keri.app import configing, keeping, habbing, storing, signaling, oobiing, agenting, delegating, \ - forwarding, querying, connecting + forwarding, querying, connecting, grouping from keri.app.grouping import Counselor from keri.app.keeping import Algos from keri.core import coring, parsing, eventing, routing @@ -37,6 +36,8 @@ from keri.app import challenging from . import aiding, notifying, indirecting, credentialing, presenting +from . import grouping as keriagrouping +from ..peer import exchanging as keriaexchanging from .specing import AgentSpecResource from ..core import authing, longrunning, httping from ..core.authing import Authenticater @@ -81,6 +82,8 @@ def setup(name, bran, adminPort, bootPort, base='', httpPort=None, configFile=No credentialing.loadEnds(app=app, identifierResource=aidEnd) presenting.loadEnds(app=app) notifying.loadEnds(app=app) + keriagrouping.loadEnds(app=app) + keriaexchanging.loadEnds(app=app) if httpPort: happ = falcon.App(middleware=falcon.CORSMiddleware( @@ -274,11 +277,13 @@ def __init__(self, hby, rgy, agentHab, agency, caid, **opts): signaler = signaling.Signaler() self.notifier = Notifier(hby=hby, signaler=signaler) + self.mux = grouping.Multiplexor(hby=hby, notifier=self.notifier) # Initialize all the credential processors self.verifier = verifying.Verifier(hby=hby, reger=rgy.reger) - self.registrar = credentialing.Registrar(agentHab=agentHab, hby=hby, rgy=rgy, counselor=self.counselor, witPub=self.witPub, - witDoer=self.witDoer, postman=self.postman, verifier=self.verifier) + self.registrar = credentialing.Registrar(agentHab=agentHab, hby=hby, rgy=rgy, counselor=self.counselor, + witPub=self.witPub, witDoer=self.witDoer, postman=self.postman, + verifier=self.verifier) self.credentialer = credentialing.Credentialer(agentHab=agentHab, hby=self.hby, rgy=self.rgy, postman=self.postman, registrar=self.registrar, verifier=self.verifier, notifier=self.notifier) @@ -296,7 +301,8 @@ def __init__(self, hby, rgy, agentHab, agency, caid, **opts): challengeHandler = challenging.ChallengeHandler(db=hby.db, signaler=signaler) handlers = [issueHandler, requestHandler, proofHandler, applyHandler, challengeHandler] - self.exc = exchanging.Exchanger(db=hby.db, handlers=handlers) + self.exc = exchanging.Exchanger(hby=hby, handlers=handlers) + grouping.loadHandlers(hby=hby, exc=self.exc, mux=self.mux) self.rvy = routing.Revery(db=hby.db, cues=self.cues) self.kvy = eventing.Kevery(db=hby.db, @@ -325,7 +331,7 @@ def __init__(self, hby, rgy, agentHab, agency, caid, **opts): Querier(hby=hby, agentHab=agentHab, kvy=self.kvy, queries=self.queries), Escrower(kvy=self.kvy, rgy=self.rgy, rvy=self.rvy, tvy=self.tvy, exc=self.exc, vry=self.verifier, registrar=self.registrar, credentialer=self.credentialer), - Messager(kvy=self.kvy, parser=self.parser), + ParserDoer(kvy=self.kvy, parser=self.parser), Witnesser(receiptor=receiptor, witners=self.witners), Delegator(agentHab=agentHab, swain=self.swain, anchors=self.anchors), GroupRequester(hby=hby, agentHab=agentHab, postman=self.postman, counselor=self.counselor, @@ -364,12 +370,12 @@ def inceptExtern(self, pre, verfers, digers, **kwargs): self.agency.incept(self.caid, pre) -class Messager(doing.Doer): +class ParserDoer(doing.Doer): def __init__(self, kvy, parser): self.kvy = kvy self.parser = parser - super(Messager, self).__init__() + super(ParserDoer, self).__init__() def recur(self, tyme=None): if self.parser.ims: @@ -470,28 +476,11 @@ def recur(self, tyme): sigers = msg["sigers"] ghab = self.hby.habs[serder.pre] - if "smids" in msg: - smids = msg['smids'] - else: - smids = ghab.db.signingMembers(pre=ghab.pre) - - if "rmids" in msg: - rmids = msg['rmids'] - else: - rmids = ghab.db.rotationMembers(pre=ghab.pre) - atc = bytearray() # attachment atc.extend(coring.Counter(code=coring.CtrDex.ControllerIdxSigs, count=len(sigers)).qb64b) for siger in sigers: atc.extend(siger.qb64b) - others = list(oset(smids + (rmids or []))) - others.remove(ghab.mhab.pre) # don't send to self - print(f"Sending multisig event to {len(others)} other participants") - for recpt in others: - self.postman.send(hab=self.agentHab, dest=recpt, topic="multisig", serder=serder, - attachment=atc) - prefixer = coring.Prefixer(qb64=serder.pre) seqner = coring.Seqner(sn=serder.sn) saider = coring.Saider(qb64=serder.said) @@ -508,16 +497,21 @@ def __init__(self, hby, agentHab, queries, kvy): self.queries = queries self.kvy = kvy - super(Querier, self).__init__() + super(Querier, self).__init__(always=True) def recur(self, tyme, deeds=None): """ Processes query reqests submitting any on the cue""" if self.queries: msg = self.queries.popleft() pre = msg["pre"] - - qryDo = querying.QueryDoer(hby=self.hby, hab=self.agentHab, pre=pre, kvy=self.kvy) - self.extend([qryDo]) + if "sn" in msg: + seqNoDo = querying.SeqNoQuerier(hby=self.hby, hab=self.agentHab, pre=pre, sn=msg["sn"]) + self.extend([seqNoDo]) + elif "anchor" in msg: + pass + else: + qryDo = querying.QueryDoer(hby=self.hby, hab=self.agentHab, pre=pre, kvy=self.kvy) + self.extend([qryDo]) return super(Querier, self).recur(tyme, deeds)