[{"data":1,"prerenderedAt":3160},["ShallowReactive",2],{"article_list_async_":3},[4,2002],{"_path":5,"_dir":6,"_draft":7,"_partial":7,"_locale":8,"title":9,"description":10,"tags":11,"excerpt":10,"image":15,"publishDate":16,"body":17,"_type":1994,"_id":1995,"_source":1407,"_file":1996,"_stem":1997,"_extension":1998,"author":1999},"/dpopowich/2023-8/postgres-pubsub","2023-8",false,"","Using PostgreSQL for Pub/Sub","A+L has been working on a Single Page Application (SPA) wherein our client's users take on the role of Staff Users (think: project managers) as they aid their Customer Users in using the application to complete a complex project.",[12,13,14],"postgresql","python","async","/dpopowich/2023-8/img/psql_pub_sub.png","2024-04-15",{"type":18,"children":19,"toc":1980},"root",[20,53,58,67,167,179,184,192,211,232,237,261,266,272,294,299,304,310,315,324,372,408,474,490,679,696,702,707,758,770,817,837,868,886,909,921,958,970,995,1116,1121,1143,1178,1189,1206,1211,1462,1467,1573,1578,1584,1605,1616,1647,1653,1680,1711,1737,1814,1819,1890,1896,1921,1933,1945,1974],{"type":21,"tag":22,"props":23,"children":24},"element","p",{},[25,28,35,37,43,45,51],{"type":26,"value":27},"text","A+L has been working on a Single Page Application (SPA) wherein our client's users take on the role of ",{"type":21,"tag":29,"props":30,"children":32},"code",{"className":31},[],[33],{"type":26,"value":34},"Staff Users",{"type":26,"value":36}," (think: project managers) as they aid ",{"type":21,"tag":38,"props":39,"children":40},"em",{},[41],{"type":26,"value":42},"their",{"type":26,"value":44}," ",{"type":21,"tag":29,"props":46,"children":48},{"className":47},[],[49],{"type":26,"value":50},"Customer Users",{"type":26,"value":52}," in using the application to complete a complex project.",{"type":21,"tag":22,"props":54,"children":55},{},[56],{"type":26,"value":57},"The architecture is common to many SPAs:",{"type":21,"tag":22,"props":59,"children":60},{},[61],{"type":21,"tag":62,"props":63,"children":66},"img",{"alt":64,"src":65},"architecture","/dpopowich/2023-8/spa.png",[],{"type":21,"tag":68,"props":69,"children":70},"ul",{},[71,86,118,130,162],{"type":21,"tag":72,"props":73,"children":74},"li",{},[75,77],{"type":26,"value":76},"RDBMS - ",{"type":21,"tag":78,"props":79,"children":83},"a",{"href":80,"rel":81},"https://www.postgresql.org/",[82],"nofollow",[84],{"type":26,"value":85},"PostgreSQL",{"type":21,"tag":72,"props":87,"children":88},{},[89,91,98,100,107,109,116],{"type":26,"value":90},"REST API Server - ",{"type":21,"tag":78,"props":92,"children":95},{"href":93,"rel":94},"https://docs.python.org/3.11/library/asyncio.html#module-asyncio",[82],[96],{"type":26,"value":97},"asynchronous python-3.11",{"type":26,"value":99}," using ",{"type":21,"tag":78,"props":101,"children":104},{"href":102,"rel":103},"https://docs.sqlalchemy.org/en/20/orm/extensions/asyncio.html",[82],[105],{"type":26,"value":106},"SQLAlchemy 2.x",{"type":26,"value":108}," with ",{"type":21,"tag":78,"props":110,"children":113},{"href":111,"rel":112},"https://magicstack.github.io/asyncpg/current/",[82],[114],{"type":26,"value":115},"asyncpg",{"type":26,"value":117}," as the database engine.",{"type":21,"tag":72,"props":119,"children":120},{},[121,123],{"type":26,"value":122},"Reverse Proxy Server - ",{"type":21,"tag":78,"props":124,"children":127},{"href":125,"rel":126},"https://www.nginx.com/",[82],[128],{"type":26,"value":129},"nginx",{"type":21,"tag":72,"props":131,"children":132},{},[133,135,142,144,148,150,154,156,160],{"type":26,"value":134},"SPA Clients - ",{"type":21,"tag":78,"props":136,"children":139},{"href":137,"rel":138},"https://vuejs.org/",[82],[140],{"type":26,"value":141},"Vue",{"type":26,"value":143},".  As noted: the SPA supports different classes of users.  Our client's users are the ",{"type":21,"tag":38,"props":145,"children":146},{},[147],{"type":26,"value":34},{"type":26,"value":149},", taking on the role of Project Manager, aiding ",{"type":21,"tag":38,"props":151,"children":152},{},[153],{"type":26,"value":42},{"type":26,"value":155}," customers (",{"type":21,"tag":38,"props":157,"children":158},{},[159],{"type":26,"value":50},{"type":26,"value":161}," in the diagram) in completing a project.",{"type":21,"tag":72,"props":163,"children":164},{},[165],{"type":26,"value":166},"Notifications - early versions of the application configured the API servers to send notifications via an external SMTP server.",{"type":21,"tag":168,"props":169,"children":171},"h2",{"id":170},"live-updates",[172,177],{"type":21,"tag":38,"props":173,"children":174},{},[175],{"type":26,"value":176},"Live",{"type":26,"value":178}," Updates",{"type":21,"tag":22,"props":180,"children":181},{},[182],{"type":26,"value":183},"The SPA has a dashboard which shows a Customer User their progress in completing all their tasks; a simple graphic of a bar, 0% to 100%, with the bar filled to the percentage of tasks completed by the customer.",{"type":21,"tag":22,"props":185,"children":186},{},[187],{"type":21,"tag":62,"props":188,"children":191},{"alt":189,"src":190},"progress","/dpopowich/2023-8/img/progress.png",[],{"type":21,"tag":22,"props":193,"children":194},{},[195,197,202,204,209],{"type":26,"value":196},"Below the graphic, a list of outstanding tasks is displayed.  An interesting aspect of this application is that many customers, working on the same project, can affect ",{"type":21,"tag":38,"props":198,"children":199},{},[200],{"type":26,"value":201},"other",{"type":26,"value":203}," customer's progress.  E.g., a customer who is idle in the application may still see the progress increase as ",{"type":21,"tag":38,"props":205,"children":206},{},[207],{"type":26,"value":208},"another",{"type":26,"value":210}," customer completes a task.",{"type":21,"tag":22,"props":212,"children":213},{},[214,216,223,225,230],{"type":26,"value":215},"Early demands on the project (mostly time-demands on getting a ",{"type":21,"tag":78,"props":217,"children":220},{"href":218,"rel":219},"https://en.wikipedia.org/wiki/Minimum_viable_product",[82],[221],{"type":26,"value":222},"MVP",{"type":26,"value":224}," delivered for client review) prevented us from integrating websockets, so the SPA polls every ",{"type":21,"tag":38,"props":226,"children":227},{},[228],{"type":26,"value":229},"n",{"type":26,"value":231},"-minutes for progress via the REST API.",{"type":21,"tag":22,"props":233,"children":234},{},[235],{"type":26,"value":236},"As we approach publication of the first version of the product we're now exploring ideas we had to table during early MVP development.  These future enhancements mostly center around data updates and notifications:",{"type":21,"tag":68,"props":238,"children":239},{},[240,245,250],{"type":21,"tag":72,"props":241,"children":242},{},[243],{"type":26,"value":244},"Calculating progress is not cheap. Having every user poll every n-minutes is very expensive, especially when you consider it is difficult for the SPA to know how frequent to poll.  Since the server knows when data comes in from collaborating users that may affect progress, it's a clear win to have the server send notifications.",{"type":21,"tag":72,"props":246,"children":247},{},[248],{"type":26,"value":249},"There's a fair amount of complexity in completing tasks and it is expected customers will have several questions for staff, so there's a desire to add a live chat feature, allowing customers to ask questions.  These questions and answers are not ephemeral, but must be recorded for the life of the project.",{"type":21,"tag":72,"props":251,"children":252},{},[253,255,259],{"type":26,"value":254},"We're planning on removing immediate notification of events by email, rather, sending notifications via the same mechanism as live chat.  To handle missed notifications, a background task can poll the database for unseen notifications and send email only after ",{"type":21,"tag":38,"props":256,"children":257},{},[258],{"type":26,"value":229},{"type":26,"value":260},"-hours of going unread.  This will also limit \"spamming\" staff with each completed task (which they might already know about if they're active in the SPA), spooling up several notifications in one email.",{"type":21,"tag":22,"props":262,"children":263},{},[264],{"type":26,"value":265},"Enter websockets.  Integrating websockets gives the server a direct connection to the SPA and can send notifications for all these desired notifications.",{"type":21,"tag":168,"props":267,"children":269},{"id":268},"pubsub",[270],{"type":26,"value":271},"Pub/Sub",{"type":21,"tag":22,"props":273,"children":274},{},[275,277,283,285,292],{"type":26,"value":276},"When I think of implementing ",{"type":21,"tag":78,"props":278,"children":281},{"href":279,"rel":280},"https://en.wikipedia.org/wiki/Publish%E2%80%93subscribe_pattern",[82],[282],{"type":26,"value":271},{"type":26,"value":284}," for an application my go-to has been ",{"type":21,"tag":78,"props":286,"children":289},{"href":287,"rel":288},"https://redis.io/docs/interact/pubsub/",[82],[290],{"type":26,"value":291},"redis",{"type":26,"value":293},".  It is both performant and scalable.  It is an excellent choice for many use-cases and particularly useful for ephemeral communication that is not tied to any data.",{"type":21,"tag":22,"props":295,"children":296},{},[297],{"type":26,"value":298},"But if you need to know your notifications have been seen or need archives of all communication, you will need persistence of your messages.  This makes redis less desireable as you will need to maintain your data in two places: 1) a permanent storage, like a RDBMS, 2) your redis server.",{"type":21,"tag":22,"props":300,"children":301},{},[302],{"type":26,"value":303},"If you're using PostgreSQL for your data storage you already have (a little known) pub/sub engine.",{"type":21,"tag":168,"props":305,"children":307},{"id":306},"demo-preparation",[308],{"type":26,"value":309},"Demo Preparation",{"type":21,"tag":22,"props":311,"children":312},{},[313],{"type":26,"value":314},"You can run all the code in this tutorial if you have python-3 (tested on 3.8–3.11) and access to docker for installation of PostgreSQL.  If you have earlier versions of python and/or your own installation of PostgreSQL, your mileage may vary as you follow along.",{"type":21,"tag":316,"props":317,"children":318},"ol",{},[319],{"type":21,"tag":72,"props":320,"children":321},{},[322],{"type":26,"value":323},"Clone the repo holding demo SQL and python script:",{"type":21,"tag":325,"props":326,"children":330},"pre",{"className":327,"code":328,"language":329,"meta":8,"style":8},"language-console shiki shiki-themes github-light github-dark","$ git clone https://github.com/artandlogic/postgresql-pubsub-demo.git\n$ cd postgresql-pubsub-demo\n$ ls\ndemo.py  demo.sql  README.md\n","console",[331],{"type":21,"tag":29,"props":332,"children":333},{"__ignoreMap":8},[334,345,354,363],{"type":21,"tag":335,"props":336,"children":339},"span",{"class":337,"line":338},"line",1,[340],{"type":21,"tag":335,"props":341,"children":342},{},[343],{"type":26,"value":344},"$ git clone https://github.com/artandlogic/postgresql-pubsub-demo.git\n",{"type":21,"tag":335,"props":346,"children":348},{"class":337,"line":347},2,[349],{"type":21,"tag":335,"props":350,"children":351},{},[352],{"type":26,"value":353},"$ cd postgresql-pubsub-demo\n",{"type":21,"tag":335,"props":355,"children":357},{"class":337,"line":356},3,[358],{"type":21,"tag":335,"props":359,"children":360},{},[361],{"type":26,"value":362},"$ ls\n",{"type":21,"tag":335,"props":364,"children":366},{"class":337,"line":365},4,[367],{"type":21,"tag":335,"props":368,"children":369},{},[370],{"type":26,"value":371},"demo.py  demo.sql  README.md\n",{"type":21,"tag":316,"props":373,"children":374},{"start":347},[375,396],{"type":21,"tag":72,"props":376,"children":377},{},[378,380,386,388,394],{"type":26,"value":379},"Review two files in the repo: ",{"type":21,"tag":29,"props":381,"children":383},{"className":382},[],[384],{"type":26,"value":385},"demo.sql",{"type":26,"value":387},", SQL schema for the demo, and ",{"type":21,"tag":29,"props":389,"children":391},{"className":390},[],[392],{"type":26,"value":393},"demo.py",{"type":26,"value":395},", a simple asynchronous python application.",{"type":21,"tag":72,"props":397,"children":398},{},[399,401,406],{"type":26,"value":400},"Start PostgreSQL via docker.  The following commands will launch a container, loading the SQL downloaded in step 1.  It is important to run the command from the cloned directory containing ",{"type":21,"tag":29,"props":402,"children":404},{"className":403},[],[405],{"type":26,"value":385},{"type":26,"value":407},", so the database initializes properly.  This binds host port 8432 to the container's port 5432 (postgresql).  If that port is not available, edit as necessary",{"type":21,"tag":325,"props":409,"children":411},{"className":327,"code":410,"language":329,"meta":8,"style":8},"$ export PGHOST=localhost PGPORT=8432 PGUSER=demo PGPASSWORD=\"passw0rd!\"\n$ docker run --rm -d --name pg-pubsub-demo  \\\n    -v $(pwd):/docker-entrypoint-initdb.d \\\n    -e POSTGRES_USER=$PGUSER -e POSTGRES_PASSWORD=$PGPASSWORD \\\n    -p $PGPORT:5432 \\\n    postgres:14-alpine\n$ alias psql='docker exec -it pg-pubsub-demo psql -U demo'\n",[412],{"type":21,"tag":29,"props":413,"children":414},{"__ignoreMap":8},[415,423,431,439,447,456,465],{"type":21,"tag":335,"props":416,"children":417},{"class":337,"line":338},[418],{"type":21,"tag":335,"props":419,"children":420},{},[421],{"type":26,"value":422},"$ export PGHOST=localhost PGPORT=8432 PGUSER=demo PGPASSWORD=\"passw0rd!\"\n",{"type":21,"tag":335,"props":424,"children":425},{"class":337,"line":347},[426],{"type":21,"tag":335,"props":427,"children":428},{},[429],{"type":26,"value":430},"$ docker run --rm -d --name pg-pubsub-demo  \\\n",{"type":21,"tag":335,"props":432,"children":433},{"class":337,"line":356},[434],{"type":21,"tag":335,"props":435,"children":436},{},[437],{"type":26,"value":438},"    -v $(pwd):/docker-entrypoint-initdb.d \\\n",{"type":21,"tag":335,"props":440,"children":441},{"class":337,"line":365},[442],{"type":21,"tag":335,"props":443,"children":444},{},[445],{"type":26,"value":446},"    -e POSTGRES_USER=$PGUSER -e POSTGRES_PASSWORD=$PGPASSWORD \\\n",{"type":21,"tag":335,"props":448,"children":450},{"class":337,"line":449},5,[451],{"type":21,"tag":335,"props":452,"children":453},{},[454],{"type":26,"value":455},"    -p $PGPORT:5432 \\\n",{"type":21,"tag":335,"props":457,"children":459},{"class":337,"line":458},6,[460],{"type":21,"tag":335,"props":461,"children":462},{},[463],{"type":26,"value":464},"    postgres:14-alpine\n",{"type":21,"tag":335,"props":466,"children":468},{"class":337,"line":467},7,[469],{"type":21,"tag":335,"props":470,"children":471},{},[472],{"type":26,"value":473},"$ alias psql='docker exec -it pg-pubsub-demo psql -U demo'\n",{"type":21,"tag":316,"props":475,"children":476},{"start":365},[477],{"type":21,"tag":72,"props":478,"children":479},{},[480,482,488],{"type":26,"value":481},"Run ",{"type":21,"tag":29,"props":483,"children":485},{"className":484},[],[486],{"type":26,"value":487},"psql",{"type":26,"value":489}," to verify the setup.  You should see the following:",{"type":21,"tag":325,"props":491,"children":493},{"className":327,"code":492,"language":329,"meta":8,"style":8},"$ psql\npsql (14.5)\nType \"help\" for help.\n\ndemo=# \\d\n             List of relations\n Schema |      Name      |   Type   | Owner\n--------+----------------+----------+-------\n public | appuser        | table    | demo\n public | appuser_id_seq | sequence | demo\n public | message        | table    | demo\n public | message_id_seq | sequence | demo\n(4 rows)\n\ndemo=# select * from appuser;\n id | fullname\n----+----------\n  1 | Abe\n  2 | Bob\n  3 | Charlie\n(3 rows)\n",[494],{"type":21,"tag":29,"props":495,"children":496},{"__ignoreMap":8},[497,505,513,521,530,538,546,554,563,572,581,590,599,608,616,625,634,643,652,661,670],{"type":21,"tag":335,"props":498,"children":499},{"class":337,"line":338},[500],{"type":21,"tag":335,"props":501,"children":502},{},[503],{"type":26,"value":504},"$ psql\n",{"type":21,"tag":335,"props":506,"children":507},{"class":337,"line":347},[508],{"type":21,"tag":335,"props":509,"children":510},{},[511],{"type":26,"value":512},"psql (14.5)\n",{"type":21,"tag":335,"props":514,"children":515},{"class":337,"line":356},[516],{"type":21,"tag":335,"props":517,"children":518},{},[519],{"type":26,"value":520},"Type \"help\" for help.\n",{"type":21,"tag":335,"props":522,"children":523},{"class":337,"line":365},[524],{"type":21,"tag":335,"props":525,"children":527},{"emptyLinePlaceholder":526},true,[528],{"type":26,"value":529},"\n",{"type":21,"tag":335,"props":531,"children":532},{"class":337,"line":449},[533],{"type":21,"tag":335,"props":534,"children":535},{},[536],{"type":26,"value":537},"demo=# \\d\n",{"type":21,"tag":335,"props":539,"children":540},{"class":337,"line":458},[541],{"type":21,"tag":335,"props":542,"children":543},{},[544],{"type":26,"value":545},"             List of relations\n",{"type":21,"tag":335,"props":547,"children":548},{"class":337,"line":467},[549],{"type":21,"tag":335,"props":550,"children":551},{},[552],{"type":26,"value":553}," Schema |      Name      |   Type   | Owner\n",{"type":21,"tag":335,"props":555,"children":557},{"class":337,"line":556},8,[558],{"type":21,"tag":335,"props":559,"children":560},{},[561],{"type":26,"value":562},"--------+----------------+----------+-------\n",{"type":21,"tag":335,"props":564,"children":566},{"class":337,"line":565},9,[567],{"type":21,"tag":335,"props":568,"children":569},{},[570],{"type":26,"value":571}," public | appuser        | table    | demo\n",{"type":21,"tag":335,"props":573,"children":575},{"class":337,"line":574},10,[576],{"type":21,"tag":335,"props":577,"children":578},{},[579],{"type":26,"value":580}," public | appuser_id_seq | sequence | demo\n",{"type":21,"tag":335,"props":582,"children":584},{"class":337,"line":583},11,[585],{"type":21,"tag":335,"props":586,"children":587},{},[588],{"type":26,"value":589}," public | message        | table    | demo\n",{"type":21,"tag":335,"props":591,"children":593},{"class":337,"line":592},12,[594],{"type":21,"tag":335,"props":595,"children":596},{},[597],{"type":26,"value":598}," public | message_id_seq | sequence | demo\n",{"type":21,"tag":335,"props":600,"children":602},{"class":337,"line":601},13,[603],{"type":21,"tag":335,"props":604,"children":605},{},[606],{"type":26,"value":607},"(4 rows)\n",{"type":21,"tag":335,"props":609,"children":611},{"class":337,"line":610},14,[612],{"type":21,"tag":335,"props":613,"children":614},{"emptyLinePlaceholder":526},[615],{"type":26,"value":529},{"type":21,"tag":335,"props":617,"children":619},{"class":337,"line":618},15,[620],{"type":21,"tag":335,"props":621,"children":622},{},[623],{"type":26,"value":624},"demo=# select * from appuser;\n",{"type":21,"tag":335,"props":626,"children":628},{"class":337,"line":627},16,[629],{"type":21,"tag":335,"props":630,"children":631},{},[632],{"type":26,"value":633}," id | fullname\n",{"type":21,"tag":335,"props":635,"children":637},{"class":337,"line":636},17,[638],{"type":21,"tag":335,"props":639,"children":640},{},[641],{"type":26,"value":642},"----+----------\n",{"type":21,"tag":335,"props":644,"children":646},{"class":337,"line":645},18,[647],{"type":21,"tag":335,"props":648,"children":649},{},[650],{"type":26,"value":651},"  1 | Abe\n",{"type":21,"tag":335,"props":653,"children":655},{"class":337,"line":654},19,[656],{"type":21,"tag":335,"props":657,"children":658},{},[659],{"type":26,"value":660},"  2 | Bob\n",{"type":21,"tag":335,"props":662,"children":664},{"class":337,"line":663},20,[665],{"type":21,"tag":335,"props":666,"children":667},{},[668],{"type":26,"value":669},"  3 | Charlie\n",{"type":21,"tag":335,"props":671,"children":673},{"class":337,"line":672},21,[674],{"type":21,"tag":335,"props":675,"children":676},{},[677],{"type":26,"value":678},"(3 rows)\n",{"type":21,"tag":680,"props":681,"children":682},"blockquote",{},[683],{"type":21,"tag":22,"props":684,"children":685},{},[686,688,694],{"type":26,"value":687},"NOTE: When you are finished with the demo, you can stop/remove the container with ",{"type":21,"tag":29,"props":689,"children":691},{"className":690},[],[692],{"type":26,"value":693},"docker stop pg-pubsub-demo",{"type":26,"value":695},".",{"type":21,"tag":168,"props":697,"children":699},{"id":698},"notifylistenunlisten",[700],{"type":26,"value":701},"NOTIFY/LISTEN/UNLISTEN",{"type":21,"tag":22,"props":703,"children":704},{},[705],{"type":26,"value":706},"PostgreSQL has three non-standard SQL commands:",{"type":21,"tag":68,"props":708,"children":709},{},[710,726,742],{"type":21,"tag":72,"props":711,"children":712},{},[713,724],{"type":21,"tag":78,"props":714,"children":717},{"href":715,"rel":716},"https://www.postgresql.org/docs/14/sql-notify.html",[82],[718],{"type":21,"tag":29,"props":719,"children":721},{"className":720},[],[722],{"type":26,"value":723},"NOTIFY",{"type":26,"value":725}," - publish to a channel.",{"type":21,"tag":72,"props":727,"children":728},{},[729,740],{"type":21,"tag":78,"props":730,"children":733},{"href":731,"rel":732},"https://www.postgresql.org/docs/14/sql-listen.html",[82],[734],{"type":21,"tag":29,"props":735,"children":737},{"className":736},[],[738],{"type":26,"value":739},"LISTEN",{"type":26,"value":741}," - subscribe to a channel.",{"type":21,"tag":72,"props":743,"children":744},{},[745,756],{"type":21,"tag":78,"props":746,"children":749},{"href":747,"rel":748},"https://www.postgresql.org/docs/14/sql-unlisten.html",[82],[750],{"type":21,"tag":29,"props":751,"children":753},{"className":752},[],[754],{"type":26,"value":755},"UNLISTEN",{"type":26,"value":757}," - unsubscribe from a channel",{"type":21,"tag":22,"props":759,"children":760},{},[761,763,768],{"type":26,"value":762},"Let's look at examples with ",{"type":21,"tag":29,"props":764,"children":766},{"className":765},[],[767],{"type":26,"value":487},{"type":26,"value":769},":",{"type":21,"tag":325,"props":771,"children":773},{"className":327,"code":772,"language":329,"meta":8,"style":8},"demo=# listen foo;\nLISTEN\ndemo=# notify foo, 'Hello, world';\nNOTIFY\nAsynchronous notification \"foo\" with payload \"Hello, world\" received from server process with PID 60807.\n",[774],{"type":21,"tag":29,"props":775,"children":776},{"__ignoreMap":8},[777,785,793,801,809],{"type":21,"tag":335,"props":778,"children":779},{"class":337,"line":338},[780],{"type":21,"tag":335,"props":781,"children":782},{},[783],{"type":26,"value":784},"demo=# listen foo;\n",{"type":21,"tag":335,"props":786,"children":787},{"class":337,"line":347},[788],{"type":21,"tag":335,"props":789,"children":790},{},[791],{"type":26,"value":792},"LISTEN\n",{"type":21,"tag":335,"props":794,"children":795},{"class":337,"line":356},[796],{"type":21,"tag":335,"props":797,"children":798},{},[799],{"type":26,"value":800},"demo=# notify foo, 'Hello, world';\n",{"type":21,"tag":335,"props":802,"children":803},{"class":337,"line":365},[804],{"type":21,"tag":335,"props":805,"children":806},{},[807],{"type":26,"value":808},"NOTIFY\n",{"type":21,"tag":335,"props":810,"children":811},{"class":337,"line":449},[812],{"type":21,"tag":335,"props":813,"children":814},{},[815],{"type":26,"value":816},"Asynchronous notification \"foo\" with payload \"Hello, world\" received from server process with PID 60807.\n",{"type":21,"tag":22,"props":818,"children":819},{},[820,822,827,829,835],{"type":26,"value":821},"The ",{"type":21,"tag":29,"props":823,"children":825},{"className":824},[],[826],{"type":26,"value":723},{"type":26,"value":828}," command expects constants for both the channel name (a legal SQL identifier) and the text notification (type ",{"type":21,"tag":29,"props":830,"children":832},{"className":831},[],[833],{"type":26,"value":834},"TEXT",{"type":26,"value":836},").  You cannot use an expression:",{"type":21,"tag":325,"props":838,"children":840},{"className":327,"code":839,"language":329,"meta":8,"style":8},"demo=# notify foo, 'This will ' || 'not work.';\nERROR:  syntax error at or near \"||\"\nLINE 1: notify foo, 'This will ' || 'not work.';\n",[841],{"type":21,"tag":29,"props":842,"children":843},{"__ignoreMap":8},[844,852,860],{"type":21,"tag":335,"props":845,"children":846},{"class":337,"line":338},[847],{"type":21,"tag":335,"props":848,"children":849},{},[850],{"type":26,"value":851},"demo=# notify foo, 'This will ' || 'not work.';\n",{"type":21,"tag":335,"props":853,"children":854},{"class":337,"line":347},[855],{"type":21,"tag":335,"props":856,"children":857},{},[858],{"type":26,"value":859},"ERROR:  syntax error at or near \"||\"\n",{"type":21,"tag":335,"props":861,"children":862},{"class":337,"line":356},[863],{"type":21,"tag":335,"props":864,"children":865},{},[866],{"type":26,"value":867},"LINE 1: notify foo, 'This will ' || 'not work.';\n",{"type":21,"tag":22,"props":869,"children":870},{},[871,873,884],{"type":26,"value":872},"Fortunately, PostgreSQL provides the function ",{"type":21,"tag":78,"props":874,"children":877},{"href":875,"rel":876},"https://www.postgresql.org/docs/14/sql-notify.html#id-1.9.3.158.7.5",[82],[878],{"type":21,"tag":29,"props":879,"children":881},{"className":880},[],[882],{"type":26,"value":883},"pg_notify",{"type":26,"value":885}," which allows dynamic generation of the notifications:",{"type":21,"tag":325,"props":887,"children":889},{"className":327,"code":888,"language":329,"meta":8,"style":8},"demo=# select pg_notify('foo', 'This ' || 'is a ' || 'computed value!');\nAsynchronous notification \"foo\" with payload \"This is a computed value!\" received from server process with PID 60807.\n",[890],{"type":21,"tag":29,"props":891,"children":892},{"__ignoreMap":8},[893,901],{"type":21,"tag":335,"props":894,"children":895},{"class":337,"line":338},[896],{"type":21,"tag":335,"props":897,"children":898},{},[899],{"type":26,"value":900},"demo=# select pg_notify('foo', 'This ' || 'is a ' || 'computed value!');\n",{"type":21,"tag":335,"props":902,"children":903},{"class":337,"line":347},[904],{"type":21,"tag":335,"props":905,"children":906},{},[907],{"type":26,"value":908},"Asynchronous notification \"foo\" with payload \"This is a computed value!\" received from server process with PID 60807.\n",{"type":21,"tag":22,"props":910,"children":911},{},[912,914,919],{"type":26,"value":913},"A session can \"unsubscibe\" with the ",{"type":21,"tag":29,"props":915,"children":917},{"className":916},[],[918],{"type":26,"value":755},{"type":26,"value":920}," command:",{"type":21,"tag":325,"props":922,"children":924},{"className":327,"code":923,"language":329,"meta":8,"style":8},"demo=# unlisten foo;\nUNLISTEN\ndemo=# notify foo, 'Hello, world';\nNOTIFY\n",[925],{"type":21,"tag":29,"props":926,"children":927},{"__ignoreMap":8},[928,936,944,951],{"type":21,"tag":335,"props":929,"children":930},{"class":337,"line":338},[931],{"type":21,"tag":335,"props":932,"children":933},{},[934],{"type":26,"value":935},"demo=# unlisten foo;\n",{"type":21,"tag":335,"props":937,"children":938},{"class":337,"line":347},[939],{"type":21,"tag":335,"props":940,"children":941},{},[942],{"type":26,"value":943},"UNLISTEN\n",{"type":21,"tag":335,"props":945,"children":946},{"class":337,"line":356},[947],{"type":21,"tag":335,"props":948,"children":949},{},[950],{"type":26,"value":800},{"type":21,"tag":335,"props":952,"children":953},{"class":337,"line":365},[954],{"type":21,"tag":335,"props":955,"children":956},{},[957],{"type":26,"value":808},{"type":21,"tag":959,"props":960,"children":962},"h3",{"id":961},"notify-in-transactions",[963,968],{"type":21,"tag":29,"props":964,"children":966},{"className":965},[],[967],{"type":26,"value":723},{"type":26,"value":969}," in Transactions",{"type":21,"tag":22,"props":971,"children":972},{},[973,975,980,982,987,989,994],{"type":26,"value":974},"What is particularly attractive about ",{"type":21,"tag":29,"props":976,"children":978},{"className":977},[],[979],{"type":26,"value":723},{"type":26,"value":981}," is you can place it in a transaction and if the transaction fails, ",{"type":21,"tag":38,"props":983,"children":984},{},[985],{"type":26,"value":986},"notifications are not delivered",{"type":26,"value":988},".  Back in ",{"type":21,"tag":29,"props":990,"children":992},{"className":991},[],[993],{"type":26,"value":487},{"type":26,"value":769},{"type":21,"tag":325,"props":996,"children":998},{"className":327,"code":997,"language":329,"meta":8,"style":8},"demo=# listen foo;\nLISTEN\ndemo=# begin;\nBEGIN\ndemo=*# notify foo, 'This will not be delivered if we rollback!';\nNOTIFY\ndemo=*# rollback;\nROLLBACK\ndemo=# begin;\nBEGIN\ndemo=*# notify foo, 'This will be delivered when we commit!';\nNOTIFY\ndemo=*# commit;\nCOMMIT\nAsynchronous notification \"foo\" with payload \"This will be delivered when we commit!\" received from server process with PID 60807.\n",[999],{"type":21,"tag":29,"props":1000,"children":1001},{"__ignoreMap":8},[1002,1009,1016,1024,1032,1040,1047,1055,1063,1070,1077,1085,1092,1100,1108],{"type":21,"tag":335,"props":1003,"children":1004},{"class":337,"line":338},[1005],{"type":21,"tag":335,"props":1006,"children":1007},{},[1008],{"type":26,"value":784},{"type":21,"tag":335,"props":1010,"children":1011},{"class":337,"line":347},[1012],{"type":21,"tag":335,"props":1013,"children":1014},{},[1015],{"type":26,"value":792},{"type":21,"tag":335,"props":1017,"children":1018},{"class":337,"line":356},[1019],{"type":21,"tag":335,"props":1020,"children":1021},{},[1022],{"type":26,"value":1023},"demo=# begin;\n",{"type":21,"tag":335,"props":1025,"children":1026},{"class":337,"line":365},[1027],{"type":21,"tag":335,"props":1028,"children":1029},{},[1030],{"type":26,"value":1031},"BEGIN\n",{"type":21,"tag":335,"props":1033,"children":1034},{"class":337,"line":449},[1035],{"type":21,"tag":335,"props":1036,"children":1037},{},[1038],{"type":26,"value":1039},"demo=*# notify foo, 'This will not be delivered if we rollback!';\n",{"type":21,"tag":335,"props":1041,"children":1042},{"class":337,"line":458},[1043],{"type":21,"tag":335,"props":1044,"children":1045},{},[1046],{"type":26,"value":808},{"type":21,"tag":335,"props":1048,"children":1049},{"class":337,"line":467},[1050],{"type":21,"tag":335,"props":1051,"children":1052},{},[1053],{"type":26,"value":1054},"demo=*# rollback;\n",{"type":21,"tag":335,"props":1056,"children":1057},{"class":337,"line":556},[1058],{"type":21,"tag":335,"props":1059,"children":1060},{},[1061],{"type":26,"value":1062},"ROLLBACK\n",{"type":21,"tag":335,"props":1064,"children":1065},{"class":337,"line":565},[1066],{"type":21,"tag":335,"props":1067,"children":1068},{},[1069],{"type":26,"value":1023},{"type":21,"tag":335,"props":1071,"children":1072},{"class":337,"line":574},[1073],{"type":21,"tag":335,"props":1074,"children":1075},{},[1076],{"type":26,"value":1031},{"type":21,"tag":335,"props":1078,"children":1079},{"class":337,"line":583},[1080],{"type":21,"tag":335,"props":1081,"children":1082},{},[1083],{"type":26,"value":1084},"demo=*# notify foo, 'This will be delivered when we commit!';\n",{"type":21,"tag":335,"props":1086,"children":1087},{"class":337,"line":592},[1088],{"type":21,"tag":335,"props":1089,"children":1090},{},[1091],{"type":26,"value":808},{"type":21,"tag":335,"props":1093,"children":1094},{"class":337,"line":601},[1095],{"type":21,"tag":335,"props":1096,"children":1097},{},[1098],{"type":26,"value":1099},"demo=*# commit;\n",{"type":21,"tag":335,"props":1101,"children":1102},{"class":337,"line":610},[1103],{"type":21,"tag":335,"props":1104,"children":1105},{},[1106],{"type":26,"value":1107},"COMMIT\n",{"type":21,"tag":335,"props":1109,"children":1110},{"class":337,"line":618},[1111],{"type":21,"tag":335,"props":1112,"children":1113},{},[1114],{"type":26,"value":1115},"Asynchronous notification \"foo\" with payload \"This will be delivered when we commit!\" received from server process with PID 60807.\n",{"type":21,"tag":22,"props":1117,"children":1118},{},[1119],{"type":26,"value":1120},"This means we can write trigger functions that send notifications on events, making our notifications data-driven.",{"type":21,"tag":680,"props":1122,"children":1123},{},[1124],{"type":21,"tag":22,"props":1125,"children":1126},{},[1127,1129,1134,1136,1141],{"type":26,"value":1128},"NOTE: Compare and contrast with what we would need to do if we used redis for pub/sub.  After committing out postgres transaction we would have to make a separate call to redis.  This separation of data ",{"type":21,"tag":38,"props":1130,"children":1131},{},[1132],{"type":26,"value":1133},"event",{"type":26,"value":1135}," from event ",{"type":21,"tag":38,"props":1137,"children":1138},{},[1139],{"type":26,"value":1140},"notification",{"type":26,"value":1142}," would be a weak link in the system.",{"type":21,"tag":22,"props":1144,"children":1145},{},[1146,1148,1153,1155,1161,1163,1169,1171,1177],{"type":26,"value":1147},"If you look at ",{"type":21,"tag":29,"props":1149,"children":1151},{"className":1150},[],[1152],{"type":26,"value":385},{"type":26,"value":1154}," you will see a table, ",{"type":21,"tag":29,"props":1156,"children":1158},{"className":1157},[],[1159],{"type":26,"value":1160},"message",{"type":26,"value":1162},", for the permanent storage of all messages, and a trigger function, ",{"type":21,"tag":29,"props":1164,"children":1166},{"className":1165},[],[1167],{"type":26,"value":1168},"notify_message()",{"type":26,"value":1170},", that will send a notification (the JSON representation of the inserted record) on successful ",{"type":21,"tag":29,"props":1172,"children":1174},{"className":1173},[],[1175],{"type":26,"value":1176},"INSERT",{"type":26,"value":695},{"type":21,"tag":22,"props":1179,"children":1180},{},[1181,1183,1188],{"type":26,"value":1182},"In ",{"type":21,"tag":29,"props":1184,"children":1186},{"className":1185},[],[1187],{"type":26,"value":487},{"type":26,"value":769},{"type":21,"tag":325,"props":1190,"children":1194},{"className":1191,"code":1192,"language":1193,"meta":8,"style":8},"language-sql shiki shiki-themes github-light github-dark","LISTEN broadcast;\n","sql",[1195],{"type":21,"tag":29,"props":1196,"children":1197},{"__ignoreMap":8},[1198],{"type":21,"tag":335,"props":1199,"children":1200},{"class":337,"line":338},[1201],{"type":21,"tag":335,"props":1202,"children":1204},{"style":1203},"--shiki-default:#24292E;--shiki-dark:#E1E4E8",[1205],{"type":26,"value":1192},{"type":21,"tag":22,"props":1207,"children":1208},{},[1209],{"type":26,"value":1210},"User \"Abe\" delivers a message to the group on channel \"broadcast\":",{"type":21,"tag":325,"props":1212,"children":1214},{"className":1191,"code":1213,"language":1193,"meta":8,"style":8},"-- Abe saves a message:\ninsert into message (sender, channel, content)\n     select id, 'broadcast', 'Hello, everone!'\n       from appuser where fullname = 'Abe';\nINSERT 0 1\nAsynchronous notification \"broadcast\" with payload \"{\"id\":2,\"sender\":1,\"channel\":\"broadcast\",\"content\":\"Hello, everone!\"}\" received from server process with PID 61006.\n",[1215],{"type":21,"tag":29,"props":1216,"children":1217},{"__ignoreMap":8},[1218,1227,1246,1275,1313,1331],{"type":21,"tag":335,"props":1219,"children":1220},{"class":337,"line":338},[1221],{"type":21,"tag":335,"props":1222,"children":1224},{"style":1223},"--shiki-default:#6A737D;--shiki-dark:#6A737D",[1225],{"type":26,"value":1226},"-- Abe saves a message:\n",{"type":21,"tag":335,"props":1228,"children":1229},{"class":337,"line":347},[1230,1236,1241],{"type":21,"tag":335,"props":1231,"children":1233},{"style":1232},"--shiki-default:#D73A49;--shiki-dark:#F97583",[1234],{"type":26,"value":1235},"insert into",{"type":21,"tag":335,"props":1237,"children":1238},{"style":1232},[1239],{"type":26,"value":1240}," message",{"type":21,"tag":335,"props":1242,"children":1243},{"style":1203},[1244],{"type":26,"value":1245}," (sender, channel, content)\n",{"type":21,"tag":335,"props":1247,"children":1248},{"class":337,"line":356},[1249,1254,1259,1265,1270],{"type":21,"tag":335,"props":1250,"children":1251},{"style":1232},[1252],{"type":26,"value":1253},"     select",{"type":21,"tag":335,"props":1255,"children":1256},{"style":1203},[1257],{"type":26,"value":1258}," id, ",{"type":21,"tag":335,"props":1260,"children":1262},{"style":1261},"--shiki-default:#032F62;--shiki-dark:#9ECBFF",[1263],{"type":26,"value":1264},"'broadcast'",{"type":21,"tag":335,"props":1266,"children":1267},{"style":1203},[1268],{"type":26,"value":1269},", ",{"type":21,"tag":335,"props":1271,"children":1272},{"style":1261},[1273],{"type":26,"value":1274},"'Hello, everone!'\n",{"type":21,"tag":335,"props":1276,"children":1277},{"class":337,"line":365},[1278,1283,1288,1293,1298,1303,1308],{"type":21,"tag":335,"props":1279,"children":1280},{"style":1232},[1281],{"type":26,"value":1282},"       from",{"type":21,"tag":335,"props":1284,"children":1285},{"style":1203},[1286],{"type":26,"value":1287}," appuser ",{"type":21,"tag":335,"props":1289,"children":1290},{"style":1232},[1291],{"type":26,"value":1292},"where",{"type":21,"tag":335,"props":1294,"children":1295},{"style":1203},[1296],{"type":26,"value":1297}," fullname ",{"type":21,"tag":335,"props":1299,"children":1300},{"style":1232},[1301],{"type":26,"value":1302},"=",{"type":21,"tag":335,"props":1304,"children":1305},{"style":1261},[1306],{"type":26,"value":1307}," 'Abe'",{"type":21,"tag":335,"props":1309,"children":1310},{"style":1203},[1311],{"type":26,"value":1312},";\n",{"type":21,"tag":335,"props":1314,"children":1315},{"class":337,"line":449},[1316,1320,1326],{"type":21,"tag":335,"props":1317,"children":1318},{"style":1232},[1319],{"type":26,"value":1176},{"type":21,"tag":335,"props":1321,"children":1323},{"style":1322},"--shiki-default:#005CC5;--shiki-dark:#79B8FF",[1324],{"type":26,"value":1325}," 0",{"type":21,"tag":335,"props":1327,"children":1328},{"style":1322},[1329],{"type":26,"value":1330}," 1\n",{"type":21,"tag":335,"props":1332,"children":1333},{"class":337,"line":458},[1334,1339,1343,1348,1353,1358,1363,1368,1373,1378,1383,1388,1393,1398,1403,1408,1412,1417,1422,1427,1432,1437,1442,1447,1452,1457],{"type":21,"tag":335,"props":1335,"children":1336},{"style":1203},[1337],{"type":26,"value":1338},"Asynchronous ",{"type":21,"tag":335,"props":1340,"children":1341},{"style":1232},[1342],{"type":26,"value":1140},{"type":21,"tag":335,"props":1344,"children":1345},{"style":1261},[1346],{"type":26,"value":1347}," \"broadcast\"",{"type":21,"tag":335,"props":1349,"children":1350},{"style":1232},[1351],{"type":26,"value":1352}," with",{"type":21,"tag":335,"props":1354,"children":1355},{"style":1203},[1356],{"type":26,"value":1357}," payload ",{"type":21,"tag":335,"props":1359,"children":1360},{"style":1261},[1361],{"type":26,"value":1362},"\"{\"",{"type":21,"tag":335,"props":1364,"children":1365},{"style":1203},[1366],{"type":26,"value":1367},"id",{"type":21,"tag":335,"props":1369,"children":1370},{"style":1261},[1371],{"type":26,"value":1372},"\":2,\"",{"type":21,"tag":335,"props":1374,"children":1375},{"style":1203},[1376],{"type":26,"value":1377},"sender",{"type":21,"tag":335,"props":1379,"children":1380},{"style":1261},[1381],{"type":26,"value":1382},"\":1,\"",{"type":21,"tag":335,"props":1384,"children":1385},{"style":1203},[1386],{"type":26,"value":1387},"channel",{"type":21,"tag":335,"props":1389,"children":1390},{"style":1261},[1391],{"type":26,"value":1392},"\":\"",{"type":21,"tag":335,"props":1394,"children":1395},{"style":1203},[1396],{"type":26,"value":1397},"broadcast",{"type":21,"tag":335,"props":1399,"children":1400},{"style":1261},[1401],{"type":26,"value":1402},"\",\"",{"type":21,"tag":335,"props":1404,"children":1405},{"style":1203},[1406],{"type":26,"value":1407},"content",{"type":21,"tag":335,"props":1409,"children":1410},{"style":1261},[1411],{"type":26,"value":1392},{"type":21,"tag":335,"props":1413,"children":1414},{"style":1203},[1415],{"type":26,"value":1416},"Hello, everone!",{"type":21,"tag":335,"props":1418,"children":1419},{"style":1261},[1420],{"type":26,"value":1421},"\"}\"",{"type":21,"tag":335,"props":1423,"children":1424},{"style":1203},[1425],{"type":26,"value":1426}," received ",{"type":21,"tag":335,"props":1428,"children":1429},{"style":1232},[1430],{"type":26,"value":1431},"from",{"type":21,"tag":335,"props":1433,"children":1434},{"style":1232},[1435],{"type":26,"value":1436}," server",{"type":21,"tag":335,"props":1438,"children":1439},{"style":1203},[1440],{"type":26,"value":1441}," process ",{"type":21,"tag":335,"props":1443,"children":1444},{"style":1232},[1445],{"type":26,"value":1446},"with",{"type":21,"tag":335,"props":1448,"children":1449},{"style":1203},[1450],{"type":26,"value":1451}," PID ",{"type":21,"tag":335,"props":1453,"children":1454},{"style":1322},[1455],{"type":26,"value":1456},"61006",{"type":21,"tag":335,"props":1458,"children":1459},{"style":1203},[1460],{"type":26,"value":1461},".\n",{"type":21,"tag":22,"props":1463,"children":1464},{},[1465],{"type":26,"value":1466},"We're also protected from false-notifications (which we would be susceptible to if we managed notifcations in application logic, i.e., first complete the transaction, followed by manually sending notifications).  Let's say \"Bob\" sends a message as part of a large transaction that fails:",{"type":21,"tag":325,"props":1468,"children":1470},{"className":327,"code":1469,"language":329,"meta":8,"style":8},"demo=# begin;\nBEGIN\ndemo=*# -- insert lots of work in the transaction here\ndemo=*#\ndemo=*# -- Bob saves a message:\ndemo=*# insert into message (sender, channel, content)\ndemo-*>      select id, 'broadcast', 'I just completed the...'\ndemo-*>        from appuser where fullname = 'Bob';\nINSERT 0 1\ndemo=*#\ndemo=*# -- insert lots more work in the transaction here that fails\ndemo=*# rollback;\nROLLBACK\n",[1471],{"type":21,"tag":29,"props":1472,"children":1473},{"__ignoreMap":8},[1474,1481,1488,1496,1504,1512,1520,1528,1536,1544,1551,1559,1566],{"type":21,"tag":335,"props":1475,"children":1476},{"class":337,"line":338},[1477],{"type":21,"tag":335,"props":1478,"children":1479},{},[1480],{"type":26,"value":1023},{"type":21,"tag":335,"props":1482,"children":1483},{"class":337,"line":347},[1484],{"type":21,"tag":335,"props":1485,"children":1486},{},[1487],{"type":26,"value":1031},{"type":21,"tag":335,"props":1489,"children":1490},{"class":337,"line":356},[1491],{"type":21,"tag":335,"props":1492,"children":1493},{},[1494],{"type":26,"value":1495},"demo=*# -- insert lots of work in the transaction here\n",{"type":21,"tag":335,"props":1497,"children":1498},{"class":337,"line":365},[1499],{"type":21,"tag":335,"props":1500,"children":1501},{},[1502],{"type":26,"value":1503},"demo=*#\n",{"type":21,"tag":335,"props":1505,"children":1506},{"class":337,"line":449},[1507],{"type":21,"tag":335,"props":1508,"children":1509},{},[1510],{"type":26,"value":1511},"demo=*# -- Bob saves a message:\n",{"type":21,"tag":335,"props":1513,"children":1514},{"class":337,"line":458},[1515],{"type":21,"tag":335,"props":1516,"children":1517},{},[1518],{"type":26,"value":1519},"demo=*# insert into message (sender, channel, content)\n",{"type":21,"tag":335,"props":1521,"children":1522},{"class":337,"line":467},[1523],{"type":21,"tag":335,"props":1524,"children":1525},{},[1526],{"type":26,"value":1527},"demo-*>      select id, 'broadcast', 'I just completed the...'\n",{"type":21,"tag":335,"props":1529,"children":1530},{"class":337,"line":556},[1531],{"type":21,"tag":335,"props":1532,"children":1533},{},[1534],{"type":26,"value":1535},"demo-*>        from appuser where fullname = 'Bob';\n",{"type":21,"tag":335,"props":1537,"children":1538},{"class":337,"line":565},[1539],{"type":21,"tag":335,"props":1540,"children":1541},{},[1542],{"type":26,"value":1543},"INSERT 0 1\n",{"type":21,"tag":335,"props":1545,"children":1546},{"class":337,"line":574},[1547],{"type":21,"tag":335,"props":1548,"children":1549},{},[1550],{"type":26,"value":1503},{"type":21,"tag":335,"props":1552,"children":1553},{"class":337,"line":583},[1554],{"type":21,"tag":335,"props":1555,"children":1556},{},[1557],{"type":26,"value":1558},"demo=*# -- insert lots more work in the transaction here that fails\n",{"type":21,"tag":335,"props":1560,"children":1561},{"class":337,"line":592},[1562],{"type":21,"tag":335,"props":1563,"children":1564},{},[1565],{"type":26,"value":1054},{"type":21,"tag":335,"props":1567,"children":1568},{"class":337,"line":601},[1569],{"type":21,"tag":335,"props":1570,"children":1571},{},[1572],{"type":26,"value":1062},{"type":21,"tag":22,"props":1574,"children":1575},{},[1576],{"type":26,"value":1577},"No notification delivered!",{"type":21,"tag":168,"props":1579,"children":1581},{"id":1580},"in-python",[1582],{"type":26,"value":1583},"In Python",{"type":21,"tag":22,"props":1585,"children":1586},{},[1587,1589,1594,1596,1603],{"type":26,"value":1588},"As noted above, our architecture uses ",{"type":21,"tag":29,"props":1590,"children":1592},{"className":1591},[],[1593],{"type":26,"value":115},{"type":26,"value":1595}," which has support for ",{"type":21,"tag":78,"props":1597,"children":1600},{"href":1598,"rel":1599},"https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.connection.Connection.add_listener",[82],[1601],{"type":26,"value":1602},"registering callbacks",{"type":26,"value":1604}," on notifications.",{"type":21,"tag":22,"props":1606,"children":1607},{},[1608,1610,1615],{"type":26,"value":1609},"First we must create a virtualenv to install ",{"type":21,"tag":29,"props":1611,"children":1613},{"className":1612},[],[1614],{"type":26,"value":115},{"type":26,"value":695},{"type":21,"tag":325,"props":1617,"children":1619},{"className":327,"code":1618,"language":329,"meta":8,"style":8},"$ python3.11 -m venv venv\n$ source venv/bin/activate\n(venv) $ pip install asyncpg\n",[1620],{"type":21,"tag":29,"props":1621,"children":1622},{"__ignoreMap":8},[1623,1631,1639],{"type":21,"tag":335,"props":1624,"children":1625},{"class":337,"line":338},[1626],{"type":21,"tag":335,"props":1627,"children":1628},{},[1629],{"type":26,"value":1630},"$ python3.11 -m venv venv\n",{"type":21,"tag":335,"props":1632,"children":1633},{"class":337,"line":347},[1634],{"type":21,"tag":335,"props":1635,"children":1636},{},[1637],{"type":26,"value":1638},"$ source venv/bin/activate\n",{"type":21,"tag":335,"props":1640,"children":1641},{"class":337,"line":356},[1642],{"type":21,"tag":335,"props":1643,"children":1644},{},[1645],{"type":26,"value":1646},"(venv) $ pip install asyncpg\n",{"type":21,"tag":959,"props":1648,"children":1650},{"id":1649},"simple-demo",[1651],{"type":26,"value":1652},"Simple Demo",{"type":21,"tag":22,"props":1654,"children":1655},{},[1656,1658,1663,1665,1670,1672,1678],{"type":26,"value":1657},"The python script, ",{"type":21,"tag":29,"props":1659,"children":1661},{"className":1660},[],[1662],{"type":26,"value":393},{"type":26,"value":1664},", establishes a connection to PostgreSQL using the ",{"type":21,"tag":29,"props":1666,"children":1668},{"className":1667},[],[1669],{"type":26,"value":115},{"type":26,"value":1671}," package and adds a listener for notifications on channel ",{"type":21,"tag":29,"props":1673,"children":1675},{"className":1674},[],[1676],{"type":26,"value":1677},"\"broadcast\"",{"type":26,"value":1679},".  It then simulates a server waiting on requests by sleeping.",{"type":21,"tag":325,"props":1681,"children":1683},{"className":327,"code":1682,"language":329,"meta":8,"style":8},"(venv) $ python demo.py\nType Ctrl-c to exit\nMain task Waiting for notifications...\n",[1684],{"type":21,"tag":29,"props":1685,"children":1686},{"__ignoreMap":8},[1687,1695,1703],{"type":21,"tag":335,"props":1688,"children":1689},{"class":337,"line":338},[1690],{"type":21,"tag":335,"props":1691,"children":1692},{},[1693],{"type":26,"value":1694},"(venv) $ python demo.py\n",{"type":21,"tag":335,"props":1696,"children":1697},{"class":337,"line":347},[1698],{"type":21,"tag":335,"props":1699,"children":1700},{},[1701],{"type":26,"value":1702},"Type Ctrl-c to exit\n",{"type":21,"tag":335,"props":1704,"children":1705},{"class":337,"line":356},[1706],{"type":21,"tag":335,"props":1707,"children":1708},{},[1709],{"type":26,"value":1710},"Main task Waiting for notifications...\n",{"type":21,"tag":22,"props":1712,"children":1713},{},[1714,1716,1721,1723,1728,1730,1735],{"type":26,"value":1715},"With the application running in one terminal, run ",{"type":21,"tag":29,"props":1717,"children":1719},{"className":1718},[],[1720],{"type":26,"value":487},{"type":26,"value":1722}," in another.  In that session, when an ",{"type":21,"tag":29,"props":1724,"children":1726},{"className":1725},[],[1727],{"type":26,"value":1176},{"type":26,"value":1729}," on table ",{"type":21,"tag":29,"props":1731,"children":1733},{"className":1732},[],[1734],{"type":26,"value":1160},{"type":26,"value":1736}," is committed, we will see our python application report it:",{"type":21,"tag":325,"props":1738,"children":1740},{"className":1191,"code":1739,"language":1193,"meta":8,"style":8},"insert into message (sender, channel, content)\n     select id, 'broadcast', 'Hello, demo application!'\n       from appuser where fullname = 'Abe';\n",[1741],{"type":21,"tag":29,"props":1742,"children":1743},{"__ignoreMap":8},[1744,1759,1783],{"type":21,"tag":335,"props":1745,"children":1746},{"class":337,"line":338},[1747,1751,1755],{"type":21,"tag":335,"props":1748,"children":1749},{"style":1232},[1750],{"type":26,"value":1235},{"type":21,"tag":335,"props":1752,"children":1753},{"style":1232},[1754],{"type":26,"value":1240},{"type":21,"tag":335,"props":1756,"children":1757},{"style":1203},[1758],{"type":26,"value":1245},{"type":21,"tag":335,"props":1760,"children":1761},{"class":337,"line":347},[1762,1766,1770,1774,1778],{"type":21,"tag":335,"props":1763,"children":1764},{"style":1232},[1765],{"type":26,"value":1253},{"type":21,"tag":335,"props":1767,"children":1768},{"style":1203},[1769],{"type":26,"value":1258},{"type":21,"tag":335,"props":1771,"children":1772},{"style":1261},[1773],{"type":26,"value":1264},{"type":21,"tag":335,"props":1775,"children":1776},{"style":1203},[1777],{"type":26,"value":1269},{"type":21,"tag":335,"props":1779,"children":1780},{"style":1261},[1781],{"type":26,"value":1782},"'Hello, demo application!'\n",{"type":21,"tag":335,"props":1784,"children":1785},{"class":337,"line":356},[1786,1790,1794,1798,1802,1806,1810],{"type":21,"tag":335,"props":1787,"children":1788},{"style":1232},[1789],{"type":26,"value":1282},{"type":21,"tag":335,"props":1791,"children":1792},{"style":1203},[1793],{"type":26,"value":1287},{"type":21,"tag":335,"props":1795,"children":1796},{"style":1232},[1797],{"type":26,"value":1292},{"type":21,"tag":335,"props":1799,"children":1800},{"style":1203},[1801],{"type":26,"value":1297},{"type":21,"tag":335,"props":1803,"children":1804},{"style":1232},[1805],{"type":26,"value":1302},{"type":21,"tag":335,"props":1807,"children":1808},{"style":1261},[1809],{"type":26,"value":1307},{"type":21,"tag":335,"props":1811,"children":1812},{"style":1203},[1813],{"type":26,"value":1312},{"type":21,"tag":22,"props":1815,"children":1816},{},[1817],{"type":26,"value":1818},"In the other terminal we will see:",{"type":21,"tag":325,"props":1820,"children":1822},{"className":327,"code":1821,"language":329,"meta":8,"style":8},"RECEIVED notification from \u003Casyncpg.connection.Connection object at 0x7f23f93b7920>[pid: 202] on channel broadcast:\n{\n    \"id\": 9,\n    \"sender\": 1,\n    \"channel\": \"broadcast\",\n    \"content\": \"Hello, demo application!\"\n}\n------------------------------------------------------------------------------\n",[1823],{"type":21,"tag":29,"props":1824,"children":1825},{"__ignoreMap":8},[1826,1834,1842,1850,1858,1866,1874,1882],{"type":21,"tag":335,"props":1827,"children":1828},{"class":337,"line":338},[1829],{"type":21,"tag":335,"props":1830,"children":1831},{},[1832],{"type":26,"value":1833},"RECEIVED notification from \u003Casyncpg.connection.Connection object at 0x7f23f93b7920>[pid: 202] on channel broadcast:\n",{"type":21,"tag":335,"props":1835,"children":1836},{"class":337,"line":347},[1837],{"type":21,"tag":335,"props":1838,"children":1839},{},[1840],{"type":26,"value":1841},"{\n",{"type":21,"tag":335,"props":1843,"children":1844},{"class":337,"line":356},[1845],{"type":21,"tag":335,"props":1846,"children":1847},{},[1848],{"type":26,"value":1849},"    \"id\": 9,\n",{"type":21,"tag":335,"props":1851,"children":1852},{"class":337,"line":365},[1853],{"type":21,"tag":335,"props":1854,"children":1855},{},[1856],{"type":26,"value":1857},"    \"sender\": 1,\n",{"type":21,"tag":335,"props":1859,"children":1860},{"class":337,"line":449},[1861],{"type":21,"tag":335,"props":1862,"children":1863},{},[1864],{"type":26,"value":1865},"    \"channel\": \"broadcast\",\n",{"type":21,"tag":335,"props":1867,"children":1868},{"class":337,"line":458},[1869],{"type":21,"tag":335,"props":1870,"children":1871},{},[1872],{"type":26,"value":1873},"    \"content\": \"Hello, demo application!\"\n",{"type":21,"tag":335,"props":1875,"children":1876},{"class":337,"line":467},[1877],{"type":21,"tag":335,"props":1878,"children":1879},{},[1880],{"type":26,"value":1881},"}\n",{"type":21,"tag":335,"props":1883,"children":1884},{"class":337,"line":556},[1885],{"type":21,"tag":335,"props":1886,"children":1887},{},[1888],{"type":26,"value":1889},"------------------------------------------------------------------------------\n",{"type":21,"tag":168,"props":1891,"children":1893},{"id":1892},"summary",[1894],{"type":26,"value":1895},"Summary",{"type":21,"tag":22,"props":1897,"children":1898},{},[1899,1901,1906,1907,1912,1914,1919],{"type":26,"value":1900},"For applications already using PostgreSQL you can use ",{"type":21,"tag":29,"props":1902,"children":1904},{"className":1903},[],[1905],{"type":26,"value":723},{"type":26,"value":1269},{"type":21,"tag":29,"props":1908,"children":1910},{"className":1909},[],[1911],{"type":26,"value":739},{"type":26,"value":1913},", and ",{"type":21,"tag":29,"props":1915,"children":1917},{"className":1916},[],[1918],{"type":26,"value":755},{"type":26,"value":1920}," to manage pub/sub for your application without having to install redis solely to add pub/sub infrastructure, allowing you to reduce the footprint of your application suite.",{"type":21,"tag":22,"props":1922,"children":1923},{},[1924,1926,1931],{"type":26,"value":1925},"By placing ",{"type":21,"tag":29,"props":1927,"children":1929},{"className":1928},[],[1930],{"type":26,"value":723},{"type":26,"value":1932}," in transactions, and more specifically, triggers, you can make your notifications data-driven.",{"type":21,"tag":22,"props":1934,"children":1935},{},[1936,1938,1943],{"type":26,"value":1937},"For asynchronous applications using ",{"type":21,"tag":29,"props":1939,"children":1941},{"className":1940},[],[1942],{"type":26,"value":115},{"type":26,"value":1944},", an API exists for registering callbacks which will be run asynchronously as notifications are delivered.",{"type":21,"tag":680,"props":1946,"children":1947},{},[1948],{"type":21,"tag":22,"props":1949,"children":1950},{},[1951,1953,1964,1966,1973],{"type":26,"value":1952},"NOTE, not shown here: if you're using synchronous python with ",{"type":21,"tag":78,"props":1954,"children":1957},{"href":1955,"rel":1956},"https://www.psycopg.org/docs/",[82],[1958],{"type":21,"tag":29,"props":1959,"children":1961},{"className":1960},[],[1962],{"type":26,"value":1963},"psycopg",{"type":26,"value":1965},", the library has support for ",{"type":21,"tag":78,"props":1967,"children":1970},{"href":1968,"rel":1969},"https://www.psycopg.org/docs/advanced.html#asynchronous-notifications",[82],[1971],{"type":26,"value":1972},"receiving notifications",{"type":26,"value":695},{"type":21,"tag":1975,"props":1976,"children":1977},"style",{},[1978],{"type":26,"value":1979},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":8,"searchDepth":356,"depth":356,"links":1981},[1982,1984,1985,1986,1990,1993],{"id":170,"depth":347,"text":1983},"Live Updates",{"id":268,"depth":347,"text":271},{"id":306,"depth":347,"text":309},{"id":698,"depth":347,"text":701,"children":1987},[1988],{"id":961,"depth":356,"text":1989},"NOTIFY in Transactions",{"id":1580,"depth":347,"text":1583,"children":1991},[1992],{"id":1649,"depth":356,"text":1652},{"id":1892,"depth":347,"text":1895},"markdown","content:dpopowich:2023-8:postgres-pubsub.md","dpopowich/2023-8/postgres-pubsub.md","dpopowich/2023-8/postgres-pubsub","md",{"user":2000,"name":2001},"dpopowich","Daniel Popowich",{"_path":2003,"_dir":2004,"_draft":7,"_partial":7,"_locale":8,"title":2005,"description":2006,"publishDate":2004,"tags":2007,"excerpt":2009,"body":2010,"_type":1994,"_id":3156,"_source":1407,"_file":3157,"_stem":3158,"_extension":1998,"author":3159},"/dpopowich/2021-07-30/data-collector","2021-07-30","Asynchronous Python - A Real World Example","A dive into a real example of async Python usage.",[13,2008,14],"how-to","We have a customer that developed a hardware device to make physical measurements.  Some years ago we wrote a suite of software tools for the customer: a tablet application for configuring the hardware device, a django web server to receive uploaded XML documents generated by the device, and a user-facing web application (using the same django server), providing reporting and data analytics.",{"type":18,"children":2011,"toc":3151},[2012,2016,2022,2026,2031,2039,2083,2089,2102,2107,2162,2167,2175,2180,2216,2222,2245,2250,2357,2370,2380,2385,2739,2744,3130,3134,3139,3147],{"type":21,"tag":2013,"props":2014,"children":2015},"toc",{},[],{"type":21,"tag":168,"props":2017,"children":2019},{"id":2018},"introduction",[2020],{"type":26,"value":2021},"Introduction",{"type":21,"tag":22,"props":2023,"children":2024},{},[2025],{"type":26,"value":2009},{"type":21,"tag":22,"props":2027,"children":2028},{},[2029],{"type":26,"value":2030},"The architecture roughly looks like this (simplified for clarity):",{"type":21,"tag":22,"props":2032,"children":2033},{},[2034],{"type":21,"tag":62,"props":2035,"children":2038},{"alt":2036,"src":2037},"Legacy App Architecture","/dpopowich/2021-07-30/img/legacy.png",[],{"type":21,"tag":316,"props":2040,"children":2041},{},[2042,2053,2063,2073],{"type":21,"tag":72,"props":2043,"children":2044},{},[2045,2051],{"type":21,"tag":2046,"props":2047,"children":2048},"strong",{},[2049],{"type":26,"value":2050},"H/W device:",{"type":26,"value":2052}," takes data measurements, generates an XML document, and uploads the XML document to the (3) Web App via a cellular WAN.",{"type":21,"tag":72,"props":2054,"children":2055},{},[2056,2061],{"type":21,"tag":2046,"props":2057,"children":2058},{},[2059],{"type":26,"value":2060},"Tablet Application:",{"type":26,"value":2062}," used in the field to configure the device, setting parameters for daily operation and authentication tokens for secure uploads.",{"type":21,"tag":72,"props":2064,"children":2065},{},[2066,2071],{"type":21,"tag":2046,"props":2067,"children":2068},{},[2069],{"type":26,"value":2070},"DJANGO Web App:",{"type":26,"value":2072}," the web app receives the XML documents, parses them, and stores the data in a relational database.  It also serves requests from a client-facing web application offering reports and analytics.",{"type":21,"tag":72,"props":2074,"children":2075},{},[2076,2081],{"type":21,"tag":2046,"props":2077,"children":2078},{},[2079],{"type":26,"value":2080},"Web Application:",{"type":26,"value":2082}," a modern browser web application for viewing data and analytics.",{"type":21,"tag":168,"props":2084,"children":2086},{"id":2085},"the-challenge",[2087],{"type":26,"value":2088},"The Challenge",{"type":21,"tag":22,"props":2090,"children":2091},{},[2092,2094,2101],{"type":26,"value":2093},"After several years of successful operation our customer came back to us with a new requirement: they wanted to use devices with a new mode of communication to ease the burden and costs of using a cellular network, devices based on ",{"type":21,"tag":78,"props":2095,"children":2098},{"href":2096,"rel":2097},"https://en.wikipedia.org/wiki/LoRa",[82],[2099],{"type":26,"value":2100},"LoRaWAN",{"type":26,"value":695},{"type":21,"tag":22,"props":2103,"children":2104},{},[2105],{"type":26,"value":2106},"LoRaWAN, a low-power, wide-area network, presents several challenges:",{"type":21,"tag":316,"props":2108,"children":2109},{},[2110,2123,2136,2149],{"type":21,"tag":72,"props":2111,"children":2112},{},[2113,2115],{"type":26,"value":2114},"It can only transmit 10s to 100s of bytes at a time.  This is too small to deliver our 2-3K XML documents, so immediately we know our messages will need to be broken up into several payloads.\n",{"type":21,"tag":68,"props":2116,"children":2117},{},[2118],{"type":21,"tag":72,"props":2119,"children":2120},{},[2121],{"type":26,"value":2122},"Our solution: if we don't send XML, but rather the data within the XML as a packed struct, we can shrink the message from several K to approximately 600 bytes, reducing greatly the number of payloads we need to deliver to complete a message.",{"type":21,"tag":72,"props":2124,"children":2125},{},[2126,2128],{"type":26,"value":2127},"When connecting, the device and LoRa gateway will calculate the bandwidth, so each message may be delivered at different data rates and therefore different payload sizes.  E.g., message m1 may be delivered in 11-byte payloads, while m2, connecting at a different time, may deliver 242-byte payloads.\n",{"type":21,"tag":68,"props":2129,"children":2130},{},[2131],{"type":21,"tag":72,"props":2132,"children":2133},{},[2134],{"type":26,"value":2135},"Our solution: if each payload is given a header we can identify which message each payload belongs to.  The first payload can include an expected payload count for the complete message. (Metadata delivered by the LoRa gateway identifies the device, so which device is sending the message we get for \"free\".)",{"type":21,"tag":72,"props":2137,"children":2138},{},[2139,2141],{"type":26,"value":2140},"Payloads may arrive out-of-order.\n",{"type":21,"tag":68,"props":2142,"children":2143},{},[2144],{"type":21,"tag":72,"props":2145,"children":2146},{},[2147],{"type":26,"value":2148},"Our solution: use the header to identify the payload order, 0, 1, 2, etc.  On receiving a complete message (known based on count from payload-0) we can sort the payloads to reassemble the packed struct.",{"type":21,"tag":72,"props":2150,"children":2151},{},[2152,2154],{"type":26,"value":2153},"Individual payloads may be lost.\n",{"type":21,"tag":68,"props":2155,"children":2156},{},[2157],{"type":21,"tag":72,"props":2158,"children":2159},{},[2160],{"type":26,"value":2161},"Our solution: some data is required, some data is merely metadata not critical to a successful upload.  If we're missing a required payload (particularly payload-0) we log the incident with the message we did received, but otherwise we post the message.",{"type":21,"tag":22,"props":2163,"children":2164},{},[2165],{"type":26,"value":2166},"Putting this all together it becomes clear we need a data processing pipeline; something like this:",{"type":21,"tag":22,"props":2168,"children":2169},{},[2170],{"type":21,"tag":62,"props":2171,"children":2174},{"alt":2172,"src":2173},"Pipeline","/dpopowich/2021-07-30/img/pipeline.png",[],{"type":21,"tag":22,"props":2176,"children":2177},{},[2178],{"type":26,"value":2179},"The components of the data-collector:",{"type":21,"tag":316,"props":2181,"children":2182},{},[2183,2188,2206,2211],{"type":21,"tag":72,"props":2184,"children":2185},{},[2186],{"type":26,"value":2187},"An HTTP server accepting POSTs of payloads.  After validating a payload, it is stored in (2) cache.",{"type":21,"tag":72,"props":2189,"children":2190},{},[2191,2193],{"type":26,"value":2192},"The cache will store payloads for a message and will pass on the message ID to the (3) aggregator when one of two conditions are met:\n",{"type":21,"tag":316,"props":2194,"children":2195},{},[2196,2201],{"type":21,"tag":72,"props":2197,"children":2198},{},[2199],{"type":26,"value":2200},"When it has received all payloads, based on having received the expected count of payloads for a message given by payload-0.",{"type":21,"tag":72,"props":2202,"children":2203},{},[2204],{"type":26,"value":2205},"Because payloads may be lost, we may never receive a full message, so the cache is configured with a timeout per message.  If a full message is never received the timeout will send the message, as-is, to the aggregator.",{"type":21,"tag":72,"props":2207,"children":2208},{},[2209],{"type":26,"value":2210},"The aggregator will collect all payloads, assemble them in order and, when possible, generate an XML document passing it on to (4) the Delivery Agent.",{"type":21,"tag":72,"props":2212,"children":2213},{},[2214],{"type":26,"value":2215},"The Delivery Agent, on receiving an XML document will manage a connection (and necessary authentication) with the Web App and upload the XML (via a POST call), managing retries and logging on any failure.",{"type":21,"tag":168,"props":2217,"children":2219},{"id":2218},"implementing-with-asynchronous-python",[2220],{"type":26,"value":2221},"Implementing with Asynchronous Python",{"type":21,"tag":22,"props":2223,"children":2224},{},[2225,2227,2234,2236,2243],{"type":26,"value":2226},"Modeling a pipeline in asynchronous Python is a simple matter of creating ",{"type":21,"tag":78,"props":2228,"children":2231},{"href":2229,"rel":2230},"https://docs.python.org/3.9/library/asyncio-task.html#creating-tasks",[82],[2232],{"type":26,"value":2233},"tasks",{"type":26,"value":2235}," and ",{"type":21,"tag":78,"props":2237,"children":2240},{"href":2238,"rel":2239},"https://docs.python.org/3.9/library/asyncio-queue.html#queue",[82],[2241],{"type":26,"value":2242},"queues",{"type":26,"value":2244},".  Let's look at the last two stages of the pipeline, the aggregator and delivery agent as examples.",{"type":21,"tag":22,"props":2246,"children":2247},{},[2248],{"type":26,"value":2249},"We will create two queues on application startup and two long-running tasks:",{"type":21,"tag":325,"props":2251,"children":2254},{"className":2252,"code":2253,"language":13,"meta":8,"style":8},"language-python shiki shiki-themes github-light github-dark","\nimport asyncio\n\n# create queues - for simplicity we ignore sizing the queue here\naggregator_queue = asyncio.Queue()\ndelivery_queue = asyncio.Queue()\n\n# create tasks - two async functions discussed below\nasyncio.create_task(aggregator)\nasyncio.create_task(delivery_agent)\n\n",[2255],{"type":21,"tag":29,"props":2256,"children":2257},{"__ignoreMap":8},[2258,2265,2278,2285,2293,2310,2326,2333,2341,2349],{"type":21,"tag":335,"props":2259,"children":2260},{"class":337,"line":338},[2261],{"type":21,"tag":335,"props":2262,"children":2263},{"emptyLinePlaceholder":526},[2264],{"type":26,"value":529},{"type":21,"tag":335,"props":2266,"children":2267},{"class":337,"line":347},[2268,2273],{"type":21,"tag":335,"props":2269,"children":2270},{"style":1232},[2271],{"type":26,"value":2272},"import",{"type":21,"tag":335,"props":2274,"children":2275},{"style":1203},[2276],{"type":26,"value":2277}," asyncio\n",{"type":21,"tag":335,"props":2279,"children":2280},{"class":337,"line":356},[2281],{"type":21,"tag":335,"props":2282,"children":2283},{"emptyLinePlaceholder":526},[2284],{"type":26,"value":529},{"type":21,"tag":335,"props":2286,"children":2287},{"class":337,"line":365},[2288],{"type":21,"tag":335,"props":2289,"children":2290},{"style":1223},[2291],{"type":26,"value":2292},"# create queues - for simplicity we ignore sizing the queue here\n",{"type":21,"tag":335,"props":2294,"children":2295},{"class":337,"line":449},[2296,2301,2305],{"type":21,"tag":335,"props":2297,"children":2298},{"style":1203},[2299],{"type":26,"value":2300},"aggregator_queue ",{"type":21,"tag":335,"props":2302,"children":2303},{"style":1232},[2304],{"type":26,"value":1302},{"type":21,"tag":335,"props":2306,"children":2307},{"style":1203},[2308],{"type":26,"value":2309}," asyncio.Queue()\n",{"type":21,"tag":335,"props":2311,"children":2312},{"class":337,"line":458},[2313,2318,2322],{"type":21,"tag":335,"props":2314,"children":2315},{"style":1203},[2316],{"type":26,"value":2317},"delivery_queue ",{"type":21,"tag":335,"props":2319,"children":2320},{"style":1232},[2321],{"type":26,"value":1302},{"type":21,"tag":335,"props":2323,"children":2324},{"style":1203},[2325],{"type":26,"value":2309},{"type":21,"tag":335,"props":2327,"children":2328},{"class":337,"line":467},[2329],{"type":21,"tag":335,"props":2330,"children":2331},{"emptyLinePlaceholder":526},[2332],{"type":26,"value":529},{"type":21,"tag":335,"props":2334,"children":2335},{"class":337,"line":556},[2336],{"type":21,"tag":335,"props":2337,"children":2338},{"style":1223},[2339],{"type":26,"value":2340},"# create tasks - two async functions discussed below\n",{"type":21,"tag":335,"props":2342,"children":2343},{"class":337,"line":565},[2344],{"type":21,"tag":335,"props":2345,"children":2346},{"style":1203},[2347],{"type":26,"value":2348},"asyncio.create_task(aggregator)\n",{"type":21,"tag":335,"props":2350,"children":2351},{"class":337,"line":574},[2352],{"type":21,"tag":335,"props":2353,"children":2354},{"style":1203},[2355],{"type":26,"value":2356},"asyncio.create_task(delivery_agent)\n",{"type":21,"tag":22,"props":2358,"children":2359},{},[2360,2362,2368],{"type":26,"value":2361},"The cache can push message IDs onto ",{"type":21,"tag":29,"props":2363,"children":2365},{"className":2364},[],[2366],{"type":26,"value":2367},"aggregator_queue",{"type":26,"value":2369}," whenever a\nmessage completes or times out:",{"type":21,"tag":325,"props":2371,"children":2375},{"className":2372,"code":2374,"language":26},[2373],"language-text","# in cache implementation...\naggregator_queue.put_nowait(message_id)\n",[2376],{"type":21,"tag":29,"props":2377,"children":2378},{"__ignoreMap":8},[2379],{"type":26,"value":2374},{"type":21,"tag":22,"props":2381,"children":2382},{},[2383],{"type":26,"value":2384},"Meanwhile, the aggregator can be implemented structurally like this:",{"type":21,"tag":325,"props":2386,"children":2388},{"className":2252,"code":2387,"language":13,"meta":8,"style":8},"async def aggregator():\n   \"\"\"Aggregator: collect messages and generate XML documents.\n   \"\"\"\n   while True:\n      # wait for incoming message\n      message_id = await aggregator_queue.get()\n\n      try:\n         # fetch payloads and expected count from cache\n         payloads, count = await cache.get_message(message_id)\n\n         # generate full message from payloads\n         message = generate_message(payloads, count)\n\n         # we have a full message: generate XML\n         xml = generate_xml(message)\n\n         # pass on to the next step in the pipeline, a tuple of our\n         # message ID and generated xml\n         delivery_queue.put_nowait((message_id, xml))\n\n      except Exception as exc:\n         # we log any error that might have occurred from preceding calls\n         logger.error('Error processing message id %s: %s', message_id, exc)\n\n      finally:\n         # we tell the queue the task is complete\n         aggregator_queue.task_done()\n",[2389],{"type":21,"tag":29,"props":2390,"children":2391},{"__ignoreMap":8},[2392,2415,2423,2431,2449,2457,2479,2486,2498,2506,2527,2534,2542,2559,2566,2574,2591,2598,2606,2614,2622,2629,2653,2662,2700,2708,2721,2730],{"type":21,"tag":335,"props":2393,"children":2394},{"class":337,"line":338},[2395,2399,2404,2410],{"type":21,"tag":335,"props":2396,"children":2397},{"style":1232},[2398],{"type":26,"value":14},{"type":21,"tag":335,"props":2400,"children":2401},{"style":1232},[2402],{"type":26,"value":2403}," def",{"type":21,"tag":335,"props":2405,"children":2407},{"style":2406},"--shiki-default:#6F42C1;--shiki-dark:#B392F0",[2408],{"type":26,"value":2409}," aggregator",{"type":21,"tag":335,"props":2411,"children":2412},{"style":1203},[2413],{"type":26,"value":2414},"():\n",{"type":21,"tag":335,"props":2416,"children":2417},{"class":337,"line":347},[2418],{"type":21,"tag":335,"props":2419,"children":2420},{"style":1261},[2421],{"type":26,"value":2422},"   \"\"\"Aggregator: collect messages and generate XML documents.\n",{"type":21,"tag":335,"props":2424,"children":2425},{"class":337,"line":356},[2426],{"type":21,"tag":335,"props":2427,"children":2428},{"style":1261},[2429],{"type":26,"value":2430},"   \"\"\"\n",{"type":21,"tag":335,"props":2432,"children":2433},{"class":337,"line":365},[2434,2439,2444],{"type":21,"tag":335,"props":2435,"children":2436},{"style":1232},[2437],{"type":26,"value":2438},"   while",{"type":21,"tag":335,"props":2440,"children":2441},{"style":1322},[2442],{"type":26,"value":2443}," True",{"type":21,"tag":335,"props":2445,"children":2446},{"style":1203},[2447],{"type":26,"value":2448},":\n",{"type":21,"tag":335,"props":2450,"children":2451},{"class":337,"line":449},[2452],{"type":21,"tag":335,"props":2453,"children":2454},{"style":1223},[2455],{"type":26,"value":2456},"      # wait for incoming message\n",{"type":21,"tag":335,"props":2458,"children":2459},{"class":337,"line":458},[2460,2465,2469,2474],{"type":21,"tag":335,"props":2461,"children":2462},{"style":1203},[2463],{"type":26,"value":2464},"      message_id ",{"type":21,"tag":335,"props":2466,"children":2467},{"style":1232},[2468],{"type":26,"value":1302},{"type":21,"tag":335,"props":2470,"children":2471},{"style":1232},[2472],{"type":26,"value":2473}," await",{"type":21,"tag":335,"props":2475,"children":2476},{"style":1203},[2477],{"type":26,"value":2478}," aggregator_queue.get()\n",{"type":21,"tag":335,"props":2480,"children":2481},{"class":337,"line":467},[2482],{"type":21,"tag":335,"props":2483,"children":2484},{"emptyLinePlaceholder":526},[2485],{"type":26,"value":529},{"type":21,"tag":335,"props":2487,"children":2488},{"class":337,"line":556},[2489,2494],{"type":21,"tag":335,"props":2490,"children":2491},{"style":1232},[2492],{"type":26,"value":2493},"      try",{"type":21,"tag":335,"props":2495,"children":2496},{"style":1203},[2497],{"type":26,"value":2448},{"type":21,"tag":335,"props":2499,"children":2500},{"class":337,"line":565},[2501],{"type":21,"tag":335,"props":2502,"children":2503},{"style":1223},[2504],{"type":26,"value":2505},"         # fetch payloads and expected count from cache\n",{"type":21,"tag":335,"props":2507,"children":2508},{"class":337,"line":574},[2509,2514,2518,2522],{"type":21,"tag":335,"props":2510,"children":2511},{"style":1203},[2512],{"type":26,"value":2513},"         payloads, count ",{"type":21,"tag":335,"props":2515,"children":2516},{"style":1232},[2517],{"type":26,"value":1302},{"type":21,"tag":335,"props":2519,"children":2520},{"style":1232},[2521],{"type":26,"value":2473},{"type":21,"tag":335,"props":2523,"children":2524},{"style":1203},[2525],{"type":26,"value":2526}," cache.get_message(message_id)\n",{"type":21,"tag":335,"props":2528,"children":2529},{"class":337,"line":583},[2530],{"type":21,"tag":335,"props":2531,"children":2532},{"emptyLinePlaceholder":526},[2533],{"type":26,"value":529},{"type":21,"tag":335,"props":2535,"children":2536},{"class":337,"line":592},[2537],{"type":21,"tag":335,"props":2538,"children":2539},{"style":1223},[2540],{"type":26,"value":2541},"         # generate full message from payloads\n",{"type":21,"tag":335,"props":2543,"children":2544},{"class":337,"line":601},[2545,2550,2554],{"type":21,"tag":335,"props":2546,"children":2547},{"style":1203},[2548],{"type":26,"value":2549},"         message ",{"type":21,"tag":335,"props":2551,"children":2552},{"style":1232},[2553],{"type":26,"value":1302},{"type":21,"tag":335,"props":2555,"children":2556},{"style":1203},[2557],{"type":26,"value":2558}," generate_message(payloads, count)\n",{"type":21,"tag":335,"props":2560,"children":2561},{"class":337,"line":610},[2562],{"type":21,"tag":335,"props":2563,"children":2564},{"emptyLinePlaceholder":526},[2565],{"type":26,"value":529},{"type":21,"tag":335,"props":2567,"children":2568},{"class":337,"line":618},[2569],{"type":21,"tag":335,"props":2570,"children":2571},{"style":1223},[2572],{"type":26,"value":2573},"         # we have a full message: generate XML\n",{"type":21,"tag":335,"props":2575,"children":2576},{"class":337,"line":627},[2577,2582,2586],{"type":21,"tag":335,"props":2578,"children":2579},{"style":1203},[2580],{"type":26,"value":2581},"         xml ",{"type":21,"tag":335,"props":2583,"children":2584},{"style":1232},[2585],{"type":26,"value":1302},{"type":21,"tag":335,"props":2587,"children":2588},{"style":1203},[2589],{"type":26,"value":2590}," generate_xml(message)\n",{"type":21,"tag":335,"props":2592,"children":2593},{"class":337,"line":636},[2594],{"type":21,"tag":335,"props":2595,"children":2596},{"emptyLinePlaceholder":526},[2597],{"type":26,"value":529},{"type":21,"tag":335,"props":2599,"children":2600},{"class":337,"line":645},[2601],{"type":21,"tag":335,"props":2602,"children":2603},{"style":1223},[2604],{"type":26,"value":2605},"         # pass on to the next step in the pipeline, a tuple of our\n",{"type":21,"tag":335,"props":2607,"children":2608},{"class":337,"line":654},[2609],{"type":21,"tag":335,"props":2610,"children":2611},{"style":1223},[2612],{"type":26,"value":2613},"         # message ID and generated xml\n",{"type":21,"tag":335,"props":2615,"children":2616},{"class":337,"line":663},[2617],{"type":21,"tag":335,"props":2618,"children":2619},{"style":1203},[2620],{"type":26,"value":2621},"         delivery_queue.put_nowait((message_id, xml))\n",{"type":21,"tag":335,"props":2623,"children":2624},{"class":337,"line":672},[2625],{"type":21,"tag":335,"props":2626,"children":2627},{"emptyLinePlaceholder":526},[2628],{"type":26,"value":529},{"type":21,"tag":335,"props":2630,"children":2632},{"class":337,"line":2631},22,[2633,2638,2643,2648],{"type":21,"tag":335,"props":2634,"children":2635},{"style":1232},[2636],{"type":26,"value":2637},"      except",{"type":21,"tag":335,"props":2639,"children":2640},{"style":1322},[2641],{"type":26,"value":2642}," Exception",{"type":21,"tag":335,"props":2644,"children":2645},{"style":1232},[2646],{"type":26,"value":2647}," as",{"type":21,"tag":335,"props":2649,"children":2650},{"style":1203},[2651],{"type":26,"value":2652}," exc:\n",{"type":21,"tag":335,"props":2654,"children":2656},{"class":337,"line":2655},23,[2657],{"type":21,"tag":335,"props":2658,"children":2659},{"style":1223},[2660],{"type":26,"value":2661},"         # we log any error that might have occurred from preceding calls\n",{"type":21,"tag":335,"props":2663,"children":2665},{"class":337,"line":2664},24,[2666,2671,2676,2681,2686,2690,2695],{"type":21,"tag":335,"props":2667,"children":2668},{"style":1203},[2669],{"type":26,"value":2670},"         logger.error(",{"type":21,"tag":335,"props":2672,"children":2673},{"style":1261},[2674],{"type":26,"value":2675},"'Error processing message id ",{"type":21,"tag":335,"props":2677,"children":2678},{"style":1322},[2679],{"type":26,"value":2680},"%s",{"type":21,"tag":335,"props":2682,"children":2683},{"style":1261},[2684],{"type":26,"value":2685},": ",{"type":21,"tag":335,"props":2687,"children":2688},{"style":1322},[2689],{"type":26,"value":2680},{"type":21,"tag":335,"props":2691,"children":2692},{"style":1261},[2693],{"type":26,"value":2694},"'",{"type":21,"tag":335,"props":2696,"children":2697},{"style":1203},[2698],{"type":26,"value":2699},", message_id, exc)\n",{"type":21,"tag":335,"props":2701,"children":2703},{"class":337,"line":2702},25,[2704],{"type":21,"tag":335,"props":2705,"children":2706},{"emptyLinePlaceholder":526},[2707],{"type":26,"value":529},{"type":21,"tag":335,"props":2709,"children":2711},{"class":337,"line":2710},26,[2712,2717],{"type":21,"tag":335,"props":2713,"children":2714},{"style":1232},[2715],{"type":26,"value":2716},"      finally",{"type":21,"tag":335,"props":2718,"children":2719},{"style":1203},[2720],{"type":26,"value":2448},{"type":21,"tag":335,"props":2722,"children":2724},{"class":337,"line":2723},27,[2725],{"type":21,"tag":335,"props":2726,"children":2727},{"style":1223},[2728],{"type":26,"value":2729},"         # we tell the queue the task is complete\n",{"type":21,"tag":335,"props":2731,"children":2733},{"class":337,"line":2732},28,[2734],{"type":21,"tag":335,"props":2735,"children":2736},{"style":1203},[2737],{"type":26,"value":2738},"         aggregator_queue.task_done()\n",{"type":21,"tag":22,"props":2740,"children":2741},{},[2742],{"type":26,"value":2743},"With a similar structure we can implement the delivery agent:",{"type":21,"tag":325,"props":2745,"children":2747},{"className":2252,"code":2746,"language":13,"meta":8,"style":8},"async def delivery_agent():\n   \"\"\"Delivery agent: listen for incoming XML docs and deliver to Web App\n   \"\"\"\n   # set up initial connection/authentication here.  Let's imagine we\n   # have a factory function, `web_app()` that returns our current\n   # connection to the Web App by way of an [aiohttp\n   # ClientSession](https://docs.aiohttp.org/en/stable/client_advanced.html#client-session).\n\n   # having set up the connection management, we enter our loop:\n   while True:\n      # wait for incoming message\n      message_id, xml = await delivery_queue.get()\n\n      try:\n         # function to generate POST data for REST call\n         data = gen_post_data(xml)\n\n         # get current session from factory and upload\n         session = web_app()\n         resp = await session.post(URL, data=data)\n         # process resp here, handling non 2XX responses, perhaps\n         # running the above POST in a loop for N-attempts for\n         # recoverable errors, otherwise raising an appropriate exception.\n\n      except Exception as exc:\n         # we log any error that might have occurred from preceding calls\n         logger.error('Could not upload xml for message_id %s: %s', message_id, exc)\n         # perhaps we save the generated XML for review of the failure?\n\n      finally:\n         # we tell the queue the task is complete\n         delivery_queue.task_done()\n",[2748],{"type":21,"tag":29,"props":2749,"children":2750},{"__ignoreMap":8},[2751,2771,2779,2786,2794,2802,2810,2818,2825,2833,2848,2855,2876,2883,2894,2902,2919,2926,2934,2951,2996,3004,3012,3020,3027,3046,3053,3085,3093,3101,3113,3121],{"type":21,"tag":335,"props":2752,"children":2753},{"class":337,"line":338},[2754,2758,2762,2767],{"type":21,"tag":335,"props":2755,"children":2756},{"style":1232},[2757],{"type":26,"value":14},{"type":21,"tag":335,"props":2759,"children":2760},{"style":1232},[2761],{"type":26,"value":2403},{"type":21,"tag":335,"props":2763,"children":2764},{"style":2406},[2765],{"type":26,"value":2766}," delivery_agent",{"type":21,"tag":335,"props":2768,"children":2769},{"style":1203},[2770],{"type":26,"value":2414},{"type":21,"tag":335,"props":2772,"children":2773},{"class":337,"line":347},[2774],{"type":21,"tag":335,"props":2775,"children":2776},{"style":1261},[2777],{"type":26,"value":2778},"   \"\"\"Delivery agent: listen for incoming XML docs and deliver to Web App\n",{"type":21,"tag":335,"props":2780,"children":2781},{"class":337,"line":356},[2782],{"type":21,"tag":335,"props":2783,"children":2784},{"style":1261},[2785],{"type":26,"value":2430},{"type":21,"tag":335,"props":2787,"children":2788},{"class":337,"line":365},[2789],{"type":21,"tag":335,"props":2790,"children":2791},{"style":1223},[2792],{"type":26,"value":2793},"   # set up initial connection/authentication here.  Let's imagine we\n",{"type":21,"tag":335,"props":2795,"children":2796},{"class":337,"line":449},[2797],{"type":21,"tag":335,"props":2798,"children":2799},{"style":1223},[2800],{"type":26,"value":2801},"   # have a factory function, `web_app()` that returns our current\n",{"type":21,"tag":335,"props":2803,"children":2804},{"class":337,"line":458},[2805],{"type":21,"tag":335,"props":2806,"children":2807},{"style":1223},[2808],{"type":26,"value":2809},"   # connection to the Web App by way of an [aiohttp\n",{"type":21,"tag":335,"props":2811,"children":2812},{"class":337,"line":467},[2813],{"type":21,"tag":335,"props":2814,"children":2815},{"style":1223},[2816],{"type":26,"value":2817},"   # ClientSession](https://docs.aiohttp.org/en/stable/client_advanced.html#client-session).\n",{"type":21,"tag":335,"props":2819,"children":2820},{"class":337,"line":556},[2821],{"type":21,"tag":335,"props":2822,"children":2823},{"emptyLinePlaceholder":526},[2824],{"type":26,"value":529},{"type":21,"tag":335,"props":2826,"children":2827},{"class":337,"line":565},[2828],{"type":21,"tag":335,"props":2829,"children":2830},{"style":1223},[2831],{"type":26,"value":2832},"   # having set up the connection management, we enter our loop:\n",{"type":21,"tag":335,"props":2834,"children":2835},{"class":337,"line":574},[2836,2840,2844],{"type":21,"tag":335,"props":2837,"children":2838},{"style":1232},[2839],{"type":26,"value":2438},{"type":21,"tag":335,"props":2841,"children":2842},{"style":1322},[2843],{"type":26,"value":2443},{"type":21,"tag":335,"props":2845,"children":2846},{"style":1203},[2847],{"type":26,"value":2448},{"type":21,"tag":335,"props":2849,"children":2850},{"class":337,"line":583},[2851],{"type":21,"tag":335,"props":2852,"children":2853},{"style":1223},[2854],{"type":26,"value":2456},{"type":21,"tag":335,"props":2856,"children":2857},{"class":337,"line":592},[2858,2863,2867,2871],{"type":21,"tag":335,"props":2859,"children":2860},{"style":1203},[2861],{"type":26,"value":2862},"      message_id, xml ",{"type":21,"tag":335,"props":2864,"children":2865},{"style":1232},[2866],{"type":26,"value":1302},{"type":21,"tag":335,"props":2868,"children":2869},{"style":1232},[2870],{"type":26,"value":2473},{"type":21,"tag":335,"props":2872,"children":2873},{"style":1203},[2874],{"type":26,"value":2875}," delivery_queue.get()\n",{"type":21,"tag":335,"props":2877,"children":2878},{"class":337,"line":601},[2879],{"type":21,"tag":335,"props":2880,"children":2881},{"emptyLinePlaceholder":526},[2882],{"type":26,"value":529},{"type":21,"tag":335,"props":2884,"children":2885},{"class":337,"line":610},[2886,2890],{"type":21,"tag":335,"props":2887,"children":2888},{"style":1232},[2889],{"type":26,"value":2493},{"type":21,"tag":335,"props":2891,"children":2892},{"style":1203},[2893],{"type":26,"value":2448},{"type":21,"tag":335,"props":2895,"children":2896},{"class":337,"line":618},[2897],{"type":21,"tag":335,"props":2898,"children":2899},{"style":1223},[2900],{"type":26,"value":2901},"         # function to generate POST data for REST call\n",{"type":21,"tag":335,"props":2903,"children":2904},{"class":337,"line":627},[2905,2910,2914],{"type":21,"tag":335,"props":2906,"children":2907},{"style":1203},[2908],{"type":26,"value":2909},"         data ",{"type":21,"tag":335,"props":2911,"children":2912},{"style":1232},[2913],{"type":26,"value":1302},{"type":21,"tag":335,"props":2915,"children":2916},{"style":1203},[2917],{"type":26,"value":2918}," gen_post_data(xml)\n",{"type":21,"tag":335,"props":2920,"children":2921},{"class":337,"line":636},[2922],{"type":21,"tag":335,"props":2923,"children":2924},{"emptyLinePlaceholder":526},[2925],{"type":26,"value":529},{"type":21,"tag":335,"props":2927,"children":2928},{"class":337,"line":645},[2929],{"type":21,"tag":335,"props":2930,"children":2931},{"style":1223},[2932],{"type":26,"value":2933},"         # get current session from factory and upload\n",{"type":21,"tag":335,"props":2935,"children":2936},{"class":337,"line":654},[2937,2942,2946],{"type":21,"tag":335,"props":2938,"children":2939},{"style":1203},[2940],{"type":26,"value":2941},"         session ",{"type":21,"tag":335,"props":2943,"children":2944},{"style":1232},[2945],{"type":26,"value":1302},{"type":21,"tag":335,"props":2947,"children":2948},{"style":1203},[2949],{"type":26,"value":2950}," web_app()\n",{"type":21,"tag":335,"props":2952,"children":2953},{"class":337,"line":663},[2954,2959,2963,2967,2972,2977,2981,2987,2991],{"type":21,"tag":335,"props":2955,"children":2956},{"style":1203},[2957],{"type":26,"value":2958},"         resp ",{"type":21,"tag":335,"props":2960,"children":2961},{"style":1232},[2962],{"type":26,"value":1302},{"type":21,"tag":335,"props":2964,"children":2965},{"style":1232},[2966],{"type":26,"value":2473},{"type":21,"tag":335,"props":2968,"children":2969},{"style":1203},[2970],{"type":26,"value":2971}," session.post(",{"type":21,"tag":335,"props":2973,"children":2974},{"style":1322},[2975],{"type":26,"value":2976},"URL",{"type":21,"tag":335,"props":2978,"children":2979},{"style":1203},[2980],{"type":26,"value":1269},{"type":21,"tag":335,"props":2982,"children":2984},{"style":2983},"--shiki-default:#E36209;--shiki-dark:#FFAB70",[2985],{"type":26,"value":2986},"data",{"type":21,"tag":335,"props":2988,"children":2989},{"style":1232},[2990],{"type":26,"value":1302},{"type":21,"tag":335,"props":2992,"children":2993},{"style":1203},[2994],{"type":26,"value":2995},"data)\n",{"type":21,"tag":335,"props":2997,"children":2998},{"class":337,"line":672},[2999],{"type":21,"tag":335,"props":3000,"children":3001},{"style":1223},[3002],{"type":26,"value":3003},"         # process resp here, handling non 2XX responses, perhaps\n",{"type":21,"tag":335,"props":3005,"children":3006},{"class":337,"line":2631},[3007],{"type":21,"tag":335,"props":3008,"children":3009},{"style":1223},[3010],{"type":26,"value":3011},"         # running the above POST in a loop for N-attempts for\n",{"type":21,"tag":335,"props":3013,"children":3014},{"class":337,"line":2655},[3015],{"type":21,"tag":335,"props":3016,"children":3017},{"style":1223},[3018],{"type":26,"value":3019},"         # recoverable errors, otherwise raising an appropriate exception.\n",{"type":21,"tag":335,"props":3021,"children":3022},{"class":337,"line":2664},[3023],{"type":21,"tag":335,"props":3024,"children":3025},{"emptyLinePlaceholder":526},[3026],{"type":26,"value":529},{"type":21,"tag":335,"props":3028,"children":3029},{"class":337,"line":2702},[3030,3034,3038,3042],{"type":21,"tag":335,"props":3031,"children":3032},{"style":1232},[3033],{"type":26,"value":2637},{"type":21,"tag":335,"props":3035,"children":3036},{"style":1322},[3037],{"type":26,"value":2642},{"type":21,"tag":335,"props":3039,"children":3040},{"style":1232},[3041],{"type":26,"value":2647},{"type":21,"tag":335,"props":3043,"children":3044},{"style":1203},[3045],{"type":26,"value":2652},{"type":21,"tag":335,"props":3047,"children":3048},{"class":337,"line":2710},[3049],{"type":21,"tag":335,"props":3050,"children":3051},{"style":1223},[3052],{"type":26,"value":2661},{"type":21,"tag":335,"props":3054,"children":3055},{"class":337,"line":2723},[3056,3060,3065,3069,3073,3077,3081],{"type":21,"tag":335,"props":3057,"children":3058},{"style":1203},[3059],{"type":26,"value":2670},{"type":21,"tag":335,"props":3061,"children":3062},{"style":1261},[3063],{"type":26,"value":3064},"'Could not upload xml for message_id ",{"type":21,"tag":335,"props":3066,"children":3067},{"style":1322},[3068],{"type":26,"value":2680},{"type":21,"tag":335,"props":3070,"children":3071},{"style":1261},[3072],{"type":26,"value":2685},{"type":21,"tag":335,"props":3074,"children":3075},{"style":1322},[3076],{"type":26,"value":2680},{"type":21,"tag":335,"props":3078,"children":3079},{"style":1261},[3080],{"type":26,"value":2694},{"type":21,"tag":335,"props":3082,"children":3083},{"style":1203},[3084],{"type":26,"value":2699},{"type":21,"tag":335,"props":3086,"children":3087},{"class":337,"line":2732},[3088],{"type":21,"tag":335,"props":3089,"children":3090},{"style":1223},[3091],{"type":26,"value":3092},"         # perhaps we save the generated XML for review of the failure?\n",{"type":21,"tag":335,"props":3094,"children":3096},{"class":337,"line":3095},29,[3097],{"type":21,"tag":335,"props":3098,"children":3099},{"emptyLinePlaceholder":526},[3100],{"type":26,"value":529},{"type":21,"tag":335,"props":3102,"children":3104},{"class":337,"line":3103},30,[3105,3109],{"type":21,"tag":335,"props":3106,"children":3107},{"style":1232},[3108],{"type":26,"value":2716},{"type":21,"tag":335,"props":3110,"children":3111},{"style":1203},[3112],{"type":26,"value":2448},{"type":21,"tag":335,"props":3114,"children":3116},{"class":337,"line":3115},31,[3117],{"type":21,"tag":335,"props":3118,"children":3119},{"style":1223},[3120],{"type":26,"value":2729},{"type":21,"tag":335,"props":3122,"children":3124},{"class":337,"line":3123},32,[3125],{"type":21,"tag":335,"props":3126,"children":3127},{"style":1203},[3128],{"type":26,"value":3129},"         delivery_queue.task_done()\n",{"type":21,"tag":3131,"props":3132,"children":3133},"hr",{},[],{"type":21,"tag":22,"props":3135,"children":3136},{},[3137],{"type":26,"value":3138},"As we complete our implementation we end up with a software architecture that mirrors our designed pipeline:",{"type":21,"tag":22,"props":3140,"children":3141},{},[3142],{"type":21,"tag":62,"props":3143,"children":3146},{"alt":3144,"src":3145},"Data Collector Architecture","/dpopowich/2021-07-30/img/data-collector.png",[],{"type":21,"tag":1975,"props":3148,"children":3149},{},[3150],{"type":26,"value":1979},{"title":8,"searchDepth":356,"depth":356,"links":3152},[3153,3154,3155],{"id":2018,"depth":347,"text":2021},{"id":2085,"depth":347,"text":2088},{"id":2218,"depth":347,"text":2221},"content:dpopowich:2021-07-30:data-collector.md","dpopowich/2021-07-30/data-collector.md","dpopowich/2021-07-30/data-collector",{"user":2000,"name":2001},1780330266263]