diff --git a/docs/Security-Compliance/File-Transfer-TDRS/README.md b/docs/Security-Compliance/File-Transfer-TDRS/README.md deleted file mode 100644 index 0333257be..000000000 --- a/docs/Security-Compliance/File-Transfer-TDRS/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Boundary diagram with file transfer from TDP to TDRS - - -The [TDP boundary diagram](../boundary-diagram.md) has been _temporarily_ modified herein to reflect updates to the data flow. - - - - -### Updated Data flow - -Data files from grantees will, for the most part, follow the original flow: -- users with `OFA Admin` and (STT) `Data Analyst` roles will upload and submit files via the web application -- upon submission, files will be scanned for viruses via ClamAV. Infected files will be discarded, and clean files will be stored in cloud.gov AWS S3 buckets. - -#### _What's new?_ -Files stored in cloud.gov's encrypted AWS S3 buckets will be transferred via SFTP to the ACFTitan server, which lives within the legacy system's (TDRS) ATO boundary diagram, as shown below. A more complete visual of the TDRS architecture and ATO boundary can be found [here](https://hhsgov.sharepoint.com/sites/TANFDataPortalOFA-TestPrivateChannel/Shared%20Documents/Forms/AllItems.aspx?id=%2Fsites%2FTANFDataPortalOFA%2DTestPrivateChannel%2FShared%20Documents%2FTest%20Private%20Channel%2FExamples%2FTDRS%2FBoundary%20Diagram%2FTANF%20Network%20Diagram%2EJPG&parent=%2Fsites%2FTANFDataPortalOFA%2DTestPrivateChannel%2FShared%20Documents%2FTest%20Private%20Channel%2FExamples%2FTDRS%2FBoundary%20Diagram) :lock:. - -These files will be picked up by the ACF OCIO Ops team, who maintain TDRS, for data processing and dB storage. This transfer process is temporary until TDP reaches parity with TDRS in terms of data processing, validation, and dB storage. More background on TDRS functionality can be found [here](../../Background/Current-TDRS.md) - -![Boundary diagram](diagram.png) - -## Updating - -- Download latest version pdf diagram [draw.io](diagram.drawio) -- Edit this diagram with [draw.io](https://app.diagrams.net/) -- Update the image and point download link to correct file diff --git a/docs/Security-Compliance/File-Transfer-TDRS/diagram.drawio b/docs/Security-Compliance/File-Transfer-TDRS/diagram.drawio deleted file mode 100644 index 2c22f92dd..000000000 --- a/docs/Security-Compliance/File-Transfer-TDRS/diagram.drawio +++ /dev/null @@ -1 +0,0 @@ -7P1Zl5tG9wcKf5qsdd6LeDELLplBDBIgQHCTxSTEJBBCQvDpT5W62+5B7Th2+0n+522StqAoiqpde//2UAN/4Hxzlfuo2xttmtV/YEh6/QMX/sAwnGAI8ANTpocUFEWxh5S8L9LHtG8JTjFnj4nIY+q5SLPTi4xD29ZD0b1MTNrDIUuGF2lR37fjy2y7tn751i7KszcJThLVb1P9Ih32j6kLBPl2Q8mKfP/0avTpThM95X5MOO2jtB2fJeHiHzjft+3wcNZc+ayG5HsizMNz0jt3v9aszw7DjzwwGp5ESUM6nHdpjQe70aKoP3HqoZhLVJ8fm8z6DkiQ2wtft+f0se7D9ESRri0Ow42qJAf+B+/kkT9IcIeHV18w8lXC6+vFywT07RUs42XC6+vFywT0dfHoq/ejryv4LOHN1YvikVfvR55VEPyPc+15qItDxn/lPwQk5n2UFqBb+LZue5B2aA+Aetx+aGpwhYLTcV8MmdNFCaTqCIQHpO3aw/AoAUBKHq8fCQ9LBQzUwfPmmkNp+xKNJ+JL3rfn7vZKFcjA3bt/gdO/EtiZf0X1AAsa+rbKnir3B4aD/yTINNyuqOtXlb5k/VAAgWDrIoflDy18XfR41T+wPncCTSkOuZ7tIA1w5LH6996RRqd9lj616VaVJ9EiQMpbtn7kdFiR7Pos6ZHN5axtsqGfQJbHuxjxKHKPqIPTj9fjNxGmn8Ry/0x6afwxMXqEjfxr2d8EC5w8ytZ9Oeu97V80i+02sXON59U6CZzxT5R8I2criYXABfvlS95ewPmqz6NDMUdDAXryU+7+z8tdn+WwJ+8IHEosRI79vQL39R3fETj8gwRu8VLgCPytwC0WdwRu8RECZ83alg9oOdDq/CSahLmnyT/JtwL3RqRAJx3SG2Uga0R98sQF9HOqofc6cLdDwHG/A9/hQljgOhqGrIf9SIOqEHcB8FU3Isjjm5r2EsW3qsMa9dkJIMWz63aIhmfXwBLLnl9nafH88tFeepbyAWyAE+QLNqAWb9kAx8m3bEBQH8AGBfunvtuzi3FXXHZ//WUc0wjgLnGHDaibFoR0vlmOT6Smjmdoi3HfuvdZEpXDX/XU1tEA2AJD1n2bnpMHqH6B4g/o8viSuH969CkFNOPhzU/Jnzj/fx3nL11y16oiaBolfgbk6xuq/wDGsyy34F6g1f8AKX6LCsEXr7CDeYsd2OIOdnw19j4cOwj6DXbwbdMUEDV44Oq+Ed0sBb7j42XbD/s2bw9RLX5L5V5qnG959Bb2/y2xzIZhemTT6Dy0L/n6DZNRPC1y0vf64NSe+yT7jqrEHjFyiPo8G75HEPwhI2znj/QpkPQFuXh4ps8AcBaXl271vS57LG0NYfBZlna3O4HKvXBwn/I8vu5PnMG+AKuHRJAFRVE4SVMvWAp9qv9TmQ+UeSzlFbN8bc3PmyDYW90jZJesbrusP71hnrtA954P9Q4APgea59lfcs0ze+QrZDzhkR7FWb1uT8VNseFC3A5D2/wtYCWgLln/klX/Dm6jU/fQ0F1xhfXgbpov68VL9qAA0fcw+XyCBPwQ2EGxxUseId/CDol9uYM731I/3HbFfsB2zQ4pC+NbkPh1dDoVyd8ABYLw/M2O/F8ABXMXKMCTfRYN7MkBWu2pIcj9/nlO/zuOw1Paz8LK9VV3P0UKFq+UyTsQ8bYg/GVB5GvH5oFmbwoCfRhNz7I9Wn8/XGGMfhH0AycPJX4okD298x8a0QiSJC+N6GcM/JQIC/jzdIMJGA5B0e761ux+ehswiA5PaeuVvYE+FAT1r5b18wzgprLZrJ0fNbwBYgwvZei55YQ8AdRX01y4h3+vYbIp0vSm+WsIrFyUVPnNBnjhSsLjO47fS+B8I73fh5DXSPg1VP7Yrvcl8EmrIl8QgnyJkjj6MdLHvMRe9AtNIt8O7JW6fmsC/ASDZ0t/SqmrEve5Ee45yiVz/U/07/j77/kWAXx7Y9RXrPsQ5WPTpjic3jqFLC/dSo8AA/wgm37aCf+KnYCjX157KOhbU+FJ8b0IbjC/bibc5Vrsd1oJb8DoWfp3wOiOKfE9iXtuSWSIMDvNcfyzysd1XJSt1u+f8v2ox4G+VMLYW1vu99kSrzQz/Tqm9eOmxEtYpF47uP/YlHj3VeRLev2JvnrThyDuXQvx6c3/W+8ZsGk/bR9x5XYRwAtgyD9eCtfnN4Xp+dU66wvQbAhfzyLp91zx92Tnh8TlfhjzjrzcpyvygwLzw8z+a+HXt5r1eai0bqMU/MRRHR0SQNjXLPG32uZZ3/6g4rmr4l7pQwThRAm7p1Xmc599gbX+66nOf+XZATBG8o5e+gDVgxNfPc0nXLujefDFXSf1t8XG3s4b+BobU9q2emvqPLfXnwzz/1L87Gex4eeF+ikc9Vyod+xWbJVU+FOydg5/YRDn0v6JLz5aqO+rA/rVuDlGLl7HON7RPR+lFp5I8oyt5GJQzvEbXnlhld61iZ/1+hsUqF8Byhv/7R/at6+BYsziGnDm6UteDPun2n/oEOvXaUXPh1jvIMDiA4JU97tq8aarhKyr2+m/KvrfBPybTAcvRPpDlH92LYZnbwFXwVMFwPm3d8CLf4Qh34WGvw/Jkf/MkgZeP/Ukjb/q5qOvxoXx13bm35rGv98gJd7GXPlnlgq7Vt/w7DPeeoFGt7HH06uxx1ceN70gReweH32NBr0eB/6CILeR3S8U/mrE93FQ91Uqczf1YXD4VU7mnYIXt6fB/TuFoK/SMPpV3lt87DUuVuc46w9Ajk5fittwKtc9jatGXfHjVtzP4/cHQDFBv4RiArkzZnDHz3waT/xwKH7yc///ypk6AcgbnoIbj3b8E/aiz5D3Nt3ge+ALL15X5xeMOuYHARn90dDGP0PffzqggCGvRkAeB03fCxm8yb9Avp8f/7X85ON8oY8a4Phupz0Df73Ni8MD+L8P+z/mTC4YgfoZzPo6dvCDoPg6kHrXKn6cU7K5Cfmf1FsBk27HHewG9cqaqAPuL6DMB0VVkcUr1xY4u2+wlLnn2P42LP3Zsa77E8aeRztcW4fiDLoFhjp+bjbYHQX+NIXrb3X03Xw324z/+vvsDk2S37l1u/OU5U559Lc7N0vgA0I5r8cB3gjAjwwMJMUpaVHmS38r65v58dgtb8RXYuF/90PlJLLAfybc9+MCQlLYzSz7m+DP96LavyIh912ONwLyjMlfu34xJFp/mzBpC2/dvrsQ9bsHnO6MFeFf+vR0x0wXF/htet0742gf0MP4K2MSRe449iT2BcPugeCXj3DvJy2v8UvA5flf8Z9pTXFGLT4NLv2kSYn8nEl51yL6OceYfmuG3W3n/yo8fvflb0XJkTbrdwMo2P1OuYnG3wS83p2X8PdzHN7h6PcmErzsrvd56zuRB4x6sgI+ftbgT5iM3+Ovf4yCDv4fBsET/hYDRYTECPx9DLyjFxcPBuQHoCNBwbjHtwN9NVvvHlgS9Bcc/21geX+2zd8ajT8/geQn5z69ZsF/NBXqH08boBYsxzx5638/PeAX5gV+m5VEkCj+jyyhXx3fZ7CXhtjPDvC/LujrYuIPHOD/UAC8v6Jx8W9YCPeiPj9kJNxvw51h9e/ZvL/At783ikO8irAz5PejLARCfS//L0dZ7lP7rc58BDKSwCFgRg3UWof4BH+AmhtyYJ38eybPc476rgj8+iRL5AtFMi/9vCcL/BdR6yXUfO2D328g3ZvD/lyT3VGJ8dfpsX8+BlmgZuzz+P/BYDQBBjCgFfzy/P/3/aDLN9PrzdxLOKSDsF1XAy55WMf3ji79EXvtH87A/M9Nqrw7gRJIyg1CH2IkUHDuLYIDSUMECNL/dXdE66s9eGcZ3EeYiOhL6FvcMQmpO1bBRyw5vcv41L+hF39IB96t7p0B5O/J84+OHxNvJ8y+sXM+jORvpyL8Nqx5COwhEAD+fGKnm/0Os5E/i0VSD2QIWN2fgPQvA9LpEHWb9kGHftgMPpJcvI7i/usohb+VGb7okzrj3050eOqKorntlvS2rx5vCGk0REAcHi4B017yPzDuCngA49cCQ8T+mKeNNyVYfYlLpDAcYlQLro4b8xLK9TmckcLzbFnf5IWqmEjgk1XgXy9x4zJqQ17UElgvHF+Oe91BchvzzuF2uU9lZloVHBfKTBE63DKWr2iC21OwtS9JwV5XPHHVS/GkSqZvT5yUKdwlOVi5C/KFE7qMC/b8eM8H5c2Rz5xXBYvpJTuq4hKN/GulCmyn8uygbwiNL69z2iQauM4zGT3FB4MKZQPUcI+kCkvpEzOm2+VJb5gpnJgqmJg6lr0h9EkkmWhYl4uat7nKc4Ai13Myd2UAWh4pNpII7UXHzCn0JSTahnWAMUMM6pNM5CVpkgt4V/nq2SmUg+fvxlM8OaezcY7x5UGfRXK1qS6GoF7Ak0PS2I2+Ncn4YO8zHgXlGgVfsLmq7IdYJufVwcTC53UBpaUTiRuP7zcATQyege1/8dzaWbapYo+rgr6AJ3D9kMwP7acnQ2BHHQflTuoLmiUYU0VbDvR89bz+c4zZXSKDew45x7g3BZjXwLaEDgn627rYki2C8iDHjDCvvmHPoWJXr+jSxc0wB5g0hps37fl2z3mkayMN4dYcA9+sk4l5dv+hzq+49vr4ri4UkGIjSyTgm53xmDcDaYbCEY95yLWjjomSP9Ir7MJtysd4zqglmxs8ezUE47y6/bpno3goA/DKPmiute4v9wE2HJKGQePGglJQpxPgSwE8C/lTUEF/fKMr6L9bG0B/jEnjzaBe2FMbIp/sE8zcJ7JLvWgPT5CGw03GxsCN0n2s5zf6rhr7EmDkPvahFF4vIUYPkW9e4oJ8xgdpB/qgBW19bM/jO2WvC7E98pB+ax+Q4LWM1qksVUBC97enb7WDrfuWDmpTrMsr5G7KrWwZnAMp6baAE/qnZ25/yrIKyw5Qi0FTQHm1JGggjVQIuC8GUrRu6iHw0/qBAweIQbSO2eWqCUHPo/tUeMifKvUYWu23cr+VPQHOuACJLl9yEocmzfggZaA8+Bf7NZJtuVpHTQSgxsnx0X2IuYXK07dWg7ce7C6Vr/W6HC+3t8o1EssuvMszX9sH70JJXdVL0Qbt0bFvdAF0qEI/nF/Sza7Dg3EFHJjrM0E/kyzqxtm+jUW+hz89k8jSIYQ9CJAT9EwZNg//PXAsyCMv4fvrpPmWxywDDP69zcNAjutSBUhxKRZP74ix5RG8AwHofIIoDTC+vnHBI2cAyQBcXVffu5/6dRUCFAfoOQFdcKO/sWGLb23/xtEPbXpel+/k25pt4NuXFCDdk8RBaf6GGB0OnqliPBlSDNYRSqsxmvP9MkGeM9A+uO7v68hPW8iHxiZBn5cNkeJ5X6ycr/fmCDwb4stbnrCRTgnmfn1P2NSIDvgoadIuLlAkxtkXbXt1/wR4A0rb+K2fwi7l0SHYLg+RTwANUB8ixfpeGUWM20iMIe/SMGuYS7x5QMIVf5d+JURBcyOen7Xz5X34fKmezfv3v9L/hqYb41l76nOEA04EcrY6gPoCdEonFKIs8k5dvpYF8ICIfLTLGq+CvHa3zMZEk8Oyjg8W0GwQ2VVGra77zPem7Sw+ybLCAbsj/9qDAD1OkDsBJU+x0JbGbM2mYCDvvAHoG7IOcGifoCVAFgxINkAT9xXFQS8dbu/Bv5UDpR3JjTk/m8UIOIzAVNkAemQczIIgVYCxIH0wNpCyQKcUBKrzyO1XFQLQGyPQGeKkl+7JdMCzj+eP+Uf9locd4D394V6ub57n4QRVsBC9VE9mCfGcnYF9BfK6JMz7ol5Sm28cFoVcogN9ZU3sZML6gHNQPgnKBzZWDuoO7xvg+eDZOUwf55UD28SeTIE9Ax1VwnNQHgLqDa7HW/1ubRRudTsBfXiFdTN4mFc8Qf2ql9UJ2ghmwYL8BjgXH/OyMB9ojzHqpXUyZhHmub0TlAfo5d7sRqATc9BzZ2MaJ0AjHLQXB3XEwTO5Cds7PbZXSAi9TABdEqjPMdDWWRdyULaaQ0kA9SVA+yZAv/Ep3ZhBvfgR2FfweRXQMQC0FfOYh33B5sHEDuAZBPQhyF/BvoD1GAANUJVHXrYHXIP06etzgE7wXL/12wjb8nT+QDd+fKjD7dd4pD0L+QAF9D9B6QTtgH2GPPUPaAegU3Wj5eqxX01YJqxTacB0SAsc1Pf2q4qg3A07P5RnwLJvNHh47kaTEfDJk1Q9WJbg7F0X85lH+fNzgO9PJ767OP/rbLiPiFQt6C/Yy7g7cEafJsA9H8TCvzzNFX4xjkX8ui94P479b0Ssfts6ie/uc/s8zPXdjP/7BZTfW//8fHve25YBrPF2MtXnxNSPm5iKvpyV9XWDwufTUu9uC/QRAZu7nHB/7s+/tlzyvUmP96fg/6YZ+N8TmX9rAv7bEbhXvERhPzlvAH+1PJP85Y0B3pvi/2rGKUJ+t15v8n/wlH2BJvSBLztkwhW5O6PqIQ7ubn7xQTNu5D4CwJWdfnS2zOeOLP/Gjix/vl5Ycgeif9e86Lscid5jydeQ/dP7sXzoaqsfAtfvid1/f2O31wtLiNeDjz+8HcvbNSr0q+r89HytjwJD9O1E4v8bG7T94jzF/6tbtn0fP/7LW7a9nE6EvfJIf+Nsw/e3aHt3iP/VuD7ydST/2dmd2UPsYSi8oj+f/nhc9v23M2w/7t2/rU2fkwr+vzXLib6zCux3zR+Y4xbgwLJbzQd8z1uaUbfd3el9/0IQ6Zs99U8mA39vgtNzv/G7UPQ/mwz8t7N9FyT55bVN8qPmzeuyvi2p+Pgtf+4y0n9/Gvl/p6OZnw0dvN6y4od7+bdPRH9VsY+eiH6X5d6dh/53OxT9z6edf09i/g/NOn+17upDzMQUFxMtOJ/SZl6cvdG91H/Of7+R7z/62svnR1yeEj4/4vL5EZcfsFIX74W6n3/Ehb5jpn5IVO4uIvxtnPhnv/8kZBdQIWeIctC7f9z5BNSd7b4/4eMTPj7h4334eDXA9XXn6/8JfORXq0ANFCHIolUoFUUPf6Hflvj9u37uc7a9v10a9bQ/+8uBJIaRJBz/48XulwTzNQGOIcBoIb7443cNI9xF5Ts7394n//3xhg93tNBX0dLF67jJB7lH9GsFif/Nbmi/mP/p81vvDm6Q1Pfy/7L79b3O/1wZ+hkz/a0x08Wr2AJ15zPQvytmepfxiX9Ddfw8SN8JiH5Pnv/tlaHfa8PnytBPQPovrgxlkP8YRL39rMOv7H5pAt/4+md0+vOfbXr5ybX/ZTW6+Ld59L/hkv3k0OOPukPfk85/b0Dqtf+yoBZfmJ8ceXxdForg2Jcfm1n1UZ4I/UuM81Fzql/MGvrp70Z9j2F+dkz7w/lnsXg53ES/nsL8swOaNM784Kdq/qnLTr368MoT275Xszf5iY8d0bw/7vX+BoQfs2/gazH53CDwbc8zr7eu/1nmBgV9QeCHIxckjVHo4mVsinr9jeL/2naB9zn01xaZ/c/Wr/woS344NFKvZniROPEa0f4B/3yfET8IGhcL7FWVv79Q5E29PnihyN0QMvYuMn5Ojf4PTo1+zz35x5NbUJL6gr0SKfoDXJK7TPav7Pn2/iDQDxmMv7TL2y9/+Rt5tahiwfymxWyvMGfxNxPS3uRnPtZ8u/s15L+Nuvz8Yjbj9v3xSxGBf+W2hdYThrDnYQ9YAwYTAe/cYphr1QNV51n+4RImI47xC3Dzt0uQH5N+HHTujee/FKIfh5P3YeNVRO7ex72+rg5+Ee74gC/S3B8Dfd+2/3AN9h4XvZgf+amZflEzYa8Gm18b9D8Qh3nJpH+iv2MhzndH5H8HVjnArZR78HoMEf9soqJ+Z2j2x/jsVGVDsv/jnXDsM/b7AXZ72vnwddT3aePDoslBdeoihjQ9JRHc/fCpNX+xSQJYcTh9gfshfgxI3RlCW7zFqd+1/PXuNxDID1Bi1BvG+MkJu3ffdjf/iz3HQU/1t6lPj2NsL25yEE+ifnrGf/E/ndj331CT7wHgB3Am/mr+/90pXMgdvny9HuHj1Ofbb/u9hYpXG5z+5HjR7XH2CWnuws47O6V2cAopX3jcyh4RTc5bFhym4+5FNwdngQX+4XmeNcAvd924jAozJDVneOIW0uf2P23WCMJWG4KtKpWtq5YtywScKy19HtlajDpEj3CU1vJZA9ltMi2XRCqirSwI+Gm6oPq1H1RP6zDMHEmaLTcb0CvcLi6BwuKycNKq48FrwaMtFllXTU+AzHPKhbmIh4O03/c4KXV+5x4PPn8J0yugCnXupKjfVmxRnLr46mSg2oaN1KLl2cRhhaURnvHnvSYvvVn0gYCgQ3S00wvcAwqT4GsJyfY3Z9/j8ex8Aoa+FNEK4uaEoyxDQQ25fLWZfGtq3LGp6LoyutpYdmtwTws3fPTmHigAOZglgs2YNoPiUXpLmCDRIrMGi0aQks9MPHfhQCMNIwMJ57Td4GrDMCzHhXdNpcphlKuh7DOB8h1E78vNcNnJ0MCWtn1EEMdNTYVk5pKUl0QLowTpXp8sdHwz1xfp0MznXe2llVpicHPcVGoHGTK2HNOn4kwVSS6uYwv1sRrPw6629waoHod3i8O0Uk2FH+D+WNIyGyqKIBqBSZVmhzoL5DgIix0eoUc/g2LJlB2u4I6uUSF3sSfZo86xRFwhQJ33QzXtFFCOBMRLuvqSrnmG2Z6n/tiPVWoxrkme0dqWbN5qV4oCWyZT4IfzljTDLOaExLfyMcaZ82WH9rD6Eb5yO1tP8RnLzHABmbEYGY26esLuEGXNtr9w6906uvhLK14420O3owk8uhw7DRav8QzqykvogdjkhdudleMUenJrX4iA0C84qV/oxAS15ZaItMmvueSDvC5dCnA/Uao9g3/tlo+oVm5C2MXLIUL1NtPia0q7sOaRte3l1HRk6SgA1uJWtNSfzAlVmeN0ZGWlJA5neeHOfuQQi3y5NU0XzbnkjM+4fDgz4wGyeqchnEfOOwW1BWPCelE0NQhPhKxm634ZRS6VDspuI9TEJbt2xpaRF00BMuSA4dfpCZ/K/QmNDHLBppRfYSd6DD1gOHOTLl/ojUM0LLOXNPV82G9qWOtxhHbFoFrXyXBp1tgUaOyG6hXz3cV1k4vqZn+0jOTiSaOX0rvQmsTTUVO9LZOVi9HZy6yvWV5wLKqlMIl96AlXHWhaTpxT124pukOXhSheFruBzeT4uCtRw/QMdW81obxdDKNN7vSrymIS5znucT4uiSW7mfoEP4sqWYlk0CLslUAYZMNRoiXsolniWJVGjDJuAK9xKutgoZCok3Ck8M6pLWGtRi3aTYiiUvVekji+LWu3DG+5FdUe5m574lpx7jeVJqhrI3LCVFet045U1qVoBB05yB4j3XaCNnLRiDsvRQjnkioK66/yqMXFgqvKrhK2V1+pDPmIdnMp1FGgdKcdd5aFgakoXrqchUhtE3aa3LPHiQ5XdWnhtnveGicdyqOLHjqWGq/nTQDZMldtTOSzOGmXeKvpqpmQsN8sVbZXdN4pasIf+cN6PzoSBOR1yolmccjI8IA2K2U+46st3LuaV9h12gki327z7dxeudxgvLguRtm8vYfLVZde2sR169wgkEtXNiEdKv68NStL9VN8DJEgB4QpGAEV5kzZ8S4EZGvi6cKaeSAD2jjQoc1zFRqIjDGq3MoZC2RBQzldMoClZHyfpj7Dcf5FVlztYk0an9O5KBpWHU0Tak17YeeKvJZtss1xxNazil+VvJGQIxQ7gpkKyZ550Ct7LT5K5r7nFMGRMapHZJQ216Aoa2msW7dGR6t1+FGEqnvDTYprWoZrK7zAo2d6jUYeCqHXjXjY/RltoYPSqfiSOu03R51ouKRRTmpucXhsNrp4sscc+DUeMM0kRsLVw7hWyJIRZM081Kf1MTjR8wYAGrksGsNlLar0jiehYa68yVkspTt7A2VLREq4kjzivqhkw5lbIcp4wdduURIsUhkia+F2EhYkAGNbVVnFDihpPEZ4K8M6BrRFK8vmUqWyorvyiC7z2mhZi8xN81jYdWhvXYJnFS6yE00/V41rH1niUCzW+UmmAdbweRoe5xbNge6SeCM3tLHh11mx0WrHZHPOqveIny8oRmq1NK96xCwPKBGmS3BvZcOuNhPNEQ18b7SdYORqxuH7vh0FTN6QOkfsOQvUUGtW6Nq13CJHVIMTfFlCy0kFfOWzLQCDcASSLibsJY62YWzXtuBK63ZseTlv+OWxCDKJVnHjUlJ7pGo06yTojZf21jhwTiRu2THgA08uNaB82YtmoaFYn/Ok5RpZvyjAwjg2MpARqxxFJc/2fBDmAzfKJJEdrIuaHLlcQnsUzQ/cgif4WmudRJCAgMnjNmnZUVzonMXNG9wbLT47XKXd9nQFUi1iyHLHgUIv4jIhDbGfXY4V8lOymldxLJFnvwI9VJzhv1g+efZlZqc9l1ZJf1ldRIB6cqVf7EoV9bD12b255Fo+qXOCYE+7RYfrAFOkY7PW+sUBGjldf6UTWhvSDc1lK4jzU2PZXGfyO8CSe0Nclb7PFojN55Flbx0xrY8QrvPFRWnJRRC0WE+s5p2jslBzYby/D4Yszi2HS0yLmkdGsQi+Zw9ucKAZCepBfVLMI6PK/GWb8dZG3bDcnG+L057c7FVxqhxKwWKZk0oN6+nY3HslXSLZzmGNKk3OqVjUVprrR/i6ZELpPZDDWj14+hpQdcWX1WW/XlTc2C3wEwTQSmD3tWg3XLoUeUveUWJuobm9HPRBFtJ1zsqQDqXDCUK+BSYjF+LKBslVeJqG1wJ1WjGHJlUucHbFE9D9bTe8tWK7sYVq5jRTFvA3sE0nsov2lMimNFZWkI+ulTpAz3DjWuIOKjQhOBW+CssiqkRZ9BREV8fIN/ypFaAVuLYsIr9SA+EGnKwtyWvbzqLGIg6VidAS6E6bk7a2qU6U40AIKh4xLctkZwpCersMCQyu1hqpsRpvIQTuvCYHFzcKTr92VZafB45DBKXHHEVUr3IZNVVO7znfWJwxY6xLWr6yKuIWYq6jmLwNaq1Q6KNwcdZ7yxWlwIscte4FZVeTxSG0VKnhrVywlpix2ReVFygWNNtyaXQlzuhESPtCH/q9IoqI3tE+yzTiwUoAn+FL5Zh3PN8a0M4TLOuo8V2A6km1YUroKxBHBz+CX6HPgTUoqXnAM+llk8UsJppswGhaDkzICGugvlGXGS1Sq1xTUaUrpfV+f0FbXWtFIhjG9aDZIJPVqmt2dd5EqYOIIuRgLj0Li1OijMsEYbyqGdmVz03LXo0swmX38yEyLIwNJa5ZTJ2pWlbFQZ0p59BmcdrFumwYyBYj6Xi8ybJiTAH+KoCNoa9lud1aNHuFmmTdAcRxB06fRfbq8bKqQz5YUjMwUCpebnPV3Mc0QfItO5VeKapWeSICzMdXk20joqbG662GiETayMYBMCGmqITqqYW8Kto4Sho5cIPcBjo1Am6tBDhiPFyr/YrPzAQoHm6zXlzKujhQOc7RGwUgqtWyVDU4+ulswS7ai3zoZHwQOOsqlCfXkgC2SGctq3QrAQ0XlxZwBNSStNg8IaxUoDd+uWZxR5avtClNyzGy8YEXl+hRbO1tmjOqYQCvQzq4qsYZS5E+QSkBomls0rZlq7BeHYf9eixOQWpruSowV3r0fKs3cNbq+GDZqmclkQ6C7PXSGlpy9kq34MIeYPPmJzW0tSvBF43eO23WnvYWIqeWXNjgHWMtWIKB6YMhyN22IyJjuxcLTQD5SWjg1P4+1C070Q9WCFhUVndiNu7PpLofCpNYlVwoCScWEcfSyIt0zxPmMkCxeLhKKwvhyqS29qBFrM6LK6TtiPUpkHPWJYJMlJfLBs/VWeVYyNNsdxUQKcct9aLWIpmA3gd+oneRJZ2xjoqrm9uo3XAYKBfmDk6c7RKpsBSlfFMEKrvCzx5mzofJ5qqRyj1uVIZUPki3sttCrae4OgLWrdWy6QjHOZT91aszkgMcBYGra08EL2+sY+KwLcMellFSJDa1WSw3GU+qyaLzvcXS0Okk2UeCdmHHfeohRSMQBKIvl/wB8jgSLsldxpLL3KWSce0MF9Uy8HkUjcpQzrtTxFcaMUfEkhdE2VThDv+iZC8aVNDO7RVH1gpH6k4VC8AI2+FLTtAFoyDWrG2NSnwEir1zKsaHyKcZwDsuKPXKHnq7U47SFjHWwtbFtCY8E1PNXKjUm6ldHZ5OpHvkeo5QvMYlqrVArDdam5quSLjW2WntSazMbbU7TF3hn9CrsVnNeByJe57aTXZ0YISKsDA8xmvqynJb19j0vqTMtrWIjB3UrGI7OAQsJRZFQdTJC2tuQntnwmp6FwJP3aWAXqAvNiO1153o/UkHhpx2XuztTI4We/k8btEVto/8aneklwQcforREGtWUGPVsYHMzvJ4vW4WJG1GexvqXIpZrWkn2s2mfzzECb1Vwt2hF3eHGDsNOx6YMPB9qHtabGSsPMVWuYEoMp2YxkPTPUU0S1qKTMrtLy7lrTyekTYNVTFQ2SdhRsg1I+TJaeIZdRdQK4Uip+LB9K5m0klNCcorKm0xemyNkdnVnN9li2EFMsiXMPUKTDk3ozMPUOHFTBQ7u07BvWhG/IjB22vpw/TGL6pED02cJDoBq9fdEbrfFzw7SL46O/EtrrFeoAS366bmB+IYL+9tqAsDC1zufGxSkHU53CJIrON6K1sj+UBVn0JfH/5B+nufMiLoL0+BwOdBPpT6guG/Kc539xu8n3G+zzjfZ5zvM873Gef7jPN9xvk+43yfcb7PON9nnO8zzvcZ5/uM833G+T7jfJ9xvs8432ec7+7GFOSXBf1yKt8C//K0FPF/FuqbtLzGLwGX53/Ff6Y1xRm1eHeX11+N9L0zbzzNdtH5NrXznS0i7gb+XsQG3wkFll2W39ZaSAwMHREsazlVuLRzlmMtkQ1YLmdZAd6xxcze7rsQXG9chNVgpJCF2cQFyz4ECMGf6rRpBwqB1xy8DrspBSoZXoPCgbtzBm7wLWsOw4vs+lYOm1CP5X0en8fn8Xl8Hp/H5/F5fB6fx3/6kIAn2ZBd3KRIjBEnVZRO4Xa/jxt7CoDLb7mmrAFzmAtZ4dFOdsGfNSyhUfztelpq9uO1YM7g36XmvLhevbCb1ePD2z/t5s/j8/g8Po/P4/P4PD6Pz+P/xiGoo1GqlCHklDEDoxb8mpsAXAesUbKTUbbjSmhng2fhL2qCaxOauRLLhhZ3ZmWCjW6/CZuy3AVe69AQFkU2ZrmaVXI2uN232CTn9izPshrLcaxosHbOdrAKt3msnKjrzR5JFZbSJ+acTOQ+lJkidMgyhmMVsjTqs3g2eJpdCwyRyFIZYR6iyss6xOrz2tldzgv6oMp1tXbSreki42bLCaG/7xylm0LPpDa1XWbNUK5867CeCZMvr8TqsB8SGa1TWcwzGT3FB4PKBKQIfPsSNC4Fr2PfQwKHLtbldQrlgHIrW1aV/RDL5Lw6LKuwRIpIsZFEaC86nuLpROLGRF6SJrkYm4pcOfRoFPRkFCh8fkjw+pzKEqH75KxOcF7CWoGldGK4NcukqcdUri9xwd3eFvjLS7q1GPXgoZFvUWEjNaYP8gvuEDTiHPLI1fDtve5LlTGLQygYcygkqLHJp7CUCrXgiNi/npO5q4L5eU05NGnGc4wvDzpmlzrmnUIfreODPd8oDVqszwSdNHazquw6wcwp2nJI5DNn9aG++6Bh0BRQK916XSh0TeAHaNDYlemgRbAxK30jXkNZHALfmg0HBfUzK3N2ryth2Ri+WnyjIugF4VndMHLWG6mKsWWtN+Yldhgi2LIXwyFG/ZFi8IlOSBpvn8qeGGztel3eeHkAPDwYM+c+4+WzWVb4mmeIeMtSFr6sYX5blpBwQ9A6fof65f+o5Y1drp1lmyr2uCroS6J4U8w/8E4oG5fQv9aA/86pYsDWXx9bL9vlqjGn0JeQGH/GQZtw+42PYf2flw04E9cPCaAsM4UTfV0B3tRndtIBF+pb8HyBzplPIsE2H8DzpfpEg+baresf8K834+Vr/vLxvDbrZANkLa+WPLgP2kQFD/S/JOX7tH9Md2wxgK0DPfdMUkGKYLW5yrP/+h9f/Pt1AH/af6AOucp+9slnn3z2yWeffPbJZ5989slnn3z2ye1PGIGLmyncPsCGOuWhJW0xSzwBXihcNIeNnMCyicRZgcSxiQx+VW5MVH4v6je7ee+uRXa0Zc6NZDv3ZC6JlSLZy1ekVuwkUAuiWDp5p01jLxfAS5roo1EkR23qn0q3QGlsC0qydP5qHRUuT7SiPf7aHyx9ZFmbZwWJ3bYC66gsuxc59irCdZy0xSrgNmu5cA7bbR7bt0Pk2dGSQDVU3mr1h9yDDW7kosCOKmiuC2gB569JDGupgD6Wzdnq3nBFWUSlPTctr5KgcVUkqiqiXUfbc5AdWxnYcsrzSpP2SSDbbb0sklZzWgT4gfhKqCgLsSW7SlXH7Tae5Pkeug/9xqxCPzxEcn1McBtNDymRbY/CXvXlPSBt0Wmbeun7NVmEzVErO81vOqrojr02D7qPDYuCPJ904Wps5StdLqdB36Dm1keZMsTOSkHAWec1ed61VNZf6csaYXYxDt0auL7wgU6ANqP4RKeHhn+HTiKkEw+yCRzkG5jdWuK33FdR5Bzxyu2XnGul+8Q2CqMVVUnU7DZWON4xZvfCesFBt/l9UJkrsabtG60aQLlOK13MnHNyVZmi7YaKI9aai9qeV6eB73VlKHtNiO276GAi6TbEM6Wm8nYl7oNIKdRGq7qVW4fRtlk2UXtcVV0UHY5aczz1K2SII/ysN9R4WotXwMKTcdCQYe2iabzFzENEnNcVmcUHanU4vqES07ICa3Asq9qgjex6FCi7BQergYPliyogujaAi8RGwbPgJPmi2Z8Yit9lOYdKJp5dJG6Ht2xqb5ImB31WUPA8xc8a+5hnyQg2XEc0sMJRi88MZRxQ+IDDkpKW1SwHs/dXR6JTuC6oZXf2xmhqPGMkhKWKZlFiZMvNDy9x8IsNimAV10XMRVHhCigr2Uzr3hGJBTLZtHrlKLba1Q7JtSyBxdfe5AjELBYYzhw0HEvLA0Msh/OCPmPayc7MZbfwp6pA0CK1kj7KZ80k2OVyeb2OGoLTGqVrPe/Mbt3BZRZ6XJ51cj6kcLmd0G3WMMi2QVyrE4cz4Sl4V4RjMM8MAhc1uskhT7x91O0SZos1fOIt+gvN0mt4c99sYEfYrN7PsQSXB6lndJD0SSBJns1GQtZmxqN3uHuunKHz+iWXLMXqVDrxDleN/aaFa6zjdjdet5vDeO2WRQ3nW8+BHwWbYagCJTOCeJXktNC56Uw4iE/zu6hPSf4YG01KnwxSzrJstzWX12N6RSjQvwi+iWHbeoecGzhB3xi83bbPkEBP/MWMa9NaH+wchkdymUNkONF22V4uhZKuxYnyYmswlWS0T7OzqMKlgHJXtO/gpHfRQ+Ks1yMiDJVqWsrKvGYW6/UOoTxBITRElmVnqdv5aW8d4Jxyzj6tvDEh5XVUT15GdkwwCvW8guxYXyimVIadIl0VpisiaTGly3h/MrxNqMgWD2SbmPNs8OP4yigJdZA4JXR2IRm2rdfopRo5xuoyoR1cGyqaQuPuFy4ZHtLG98YrhARWOVXKSsiTVKuVfWxfE971c++ycpIgKYX2jG7KdSB0yLGWO9wIPUMK4Rp7+MEOqT8mTI+XijvotOtgyboGiWG7g2tULIRb2YBLl7uVxJw1OBebU6tEOjIIU83+wp7XLDWxzXYq9/tSUQafTOEMeXN5YayIPvfJxRit1c4MNMIlXSkhVgJVtFfCRty1tolCsTgBDD15bLzA2M0erpbCtJUPeLDzEp6w3XYF95fgFuiiidcERcMlUGYnBcNSW3qFHLKsE1xui2ucE+7uCinZZrW+bBHBXnHxRdeubmGsVQmjy0M7zlvHx/eCrUd2qPbWHq3EPO22epyf8ZWzs4iuzPiU1a5HHajJ4zHnvZGX4sSgaWK53qKmXh8vR0/ua9pvCHOxo1mOarxsJ/ryfI7PykIhZ5+2a7aoOTxINCsDOudhV4Osh0sWTDiHm5VRpTOGYtqW07U9BYl5tmtcGndWsCY9uCCYsHrLq9LLaWvkqmr0BKtzZR6ORhUS3ZwpKk4HI4X4G+Yay7JpXVRQvTUC1xhtnOOSOtXzaNSL0xZunXCh0p2Lo8u1f3DRFVyTS2upT5KqCw5WAgcrr4kDvsOc3LcJhpk52S+TY9o7R94oq0TumUu85CKd75IjVyZ0aeBwSwF7FcVHr6mZeCmpjuT7q22J2jk0dmqvErqF65uKjoulGo9kdLoum0u2OUC5b+CyxxPjx7qj6rJ1kZbJyTF2l7hxrF01yz3RJe5+1ZnUFi7wYZBGvi1cnC4JIP8a9+alvJ68vRfuhPrYQ7RPM1pZmwxcsyKFFJa4QXsyvWXnbzQa7ZR13EgT4Zayl9ZrMid7c70/omvYGzy9xWF9NgzcEoIGersoJcONKmJhdJh/uPpG6Cy7A7fa4QAqFBpbHi8JJcVbY+EMepoPqJiSGlxCpxqc34Gn2YTu8XPoXVqz54zTccJD7aAKeNULdIFpw7beSxmQK8HIObgfBdVkcOVc00k+7QazJRyW50VYKiMe3SBk2EFU4mhIdtbY+iUXLbm4kHjSFeRtFZZwleYqOgXtFTyr+QsjvQjMcF6KKM1uDjxOw06h2ZDy0ZMeXdGl6y06clssuebcLblcDImw4NB+e3XDvDTGBm4VcmE711xA0rBrzzkJR2fndvi5GvH5CNrH13bJzMlkl3QdeP4W9PFxXB+2F26edxv42Mgd+kDSi2JE3aYOjX7Cjhuf3AdnSb3WW+1Ajwl62NEqGpVwEWG1xaCSgvuAwLUx7Ervw6sDOlIzHTdtPF3ftD1vdDwy70soV7FB7xXitAGQi2jYOWzqyF9w8VFdTB5czGMF/LCRZC341h9DPOvaacCuLslmoOK7yr96R/qk9pgykXJjhXMjhwTslaiS+0wzQqDSKzw9XUsiGTbo4QIXlbLsurnZGgUF1I2t6onVhnwrGxd1Nt/0xqQD4yvzfV9a7A+hvBVwoyyrrDQsbRdSay50JILU94CU4jAvVVh1wYgQTbNWyQHVpEVnSDQ1qMzYuYFSb40KosTSl6NjCci+z3erSybgFxIuHhn5GT0R7XXDeculv1GXureMk/G85atg6k/9rhGa05jJi+2N2JczVFPcYQZodk1jX9uwrSQngPmPvn+wfSPWRfobvQ/UN3JjdeYvBBPlt9WxpHNMQw0gdso8XTKytiIhGje9Vg2Y01HrdDqtJz04opkveguBTPY1JLX2gtQuS07xGQs62sQ6at5Cfs93N7FkJbTzmSMcF5ME1WOvNSlDIjTOxitBF7P1yh2wAK5KU3Qb3JvwfIKro/RshqCLRb10aGmp1kK10+vikdTrKSy0C8F1SdD5HLo0/ViTmM6V6En2Fvu0VW/kBrSheLs5tvTKLwjA5DdJVJUNhBsJPYnuXl5FaX1WBgg5qu4Q+yaKNK64Djg90EwzCiVdaCe0jKBkTKIJdyyBhu3uphFqxZ8df9yU03SCkGzkN7vWmLeLwsFO6mEfHOety4vlWa49Zz0texuoVPmEXzlOT6KdeNIWjYSyJJMvveWa8jod6fQESc82X9QtKHHmi6LI2edG87frIJhCC5LTAA7dIBvAOmQkYPDatsV+0xP3rpv3irxdN6dMSzcbpGV7LD0o8WZrFOGcOBnUhhsc22fr0fWiXkOyzIGL5tBwqei6ki5cbo3bZKQoZIUekfq0NU0HheU31u3d0tbVp9V1QOuaUyNP3ZdoCtefSweXYrTpIuKrxrpxzpa8ocm6JTW0pvi9dDbbZVHUHecg4neaqGRFUS3HJFhveuB1Sarmv21rJ1jXq88RNb2xKCVeSGp9JtmdXuBCeevZcuES1+QQilPVHhfmELoGWYhFPZ6dfL1a+21fugsNqegzdSaCw/UQ0TkHd50wM9eKrjtrW6FYTeNYceMUqcoDaCfhyFGmLwrK7+YeczfTwjozE59kdZafBLjitV8tiauPrOGXfBYpcrkcpeGGyXBrIrgDUDdF146mi+u4BT7GKdGkiluamaUr4hx47rwIjmTnqJdC35hFvqP8w2o1iCe57c5ES3QDdspGvu6kLqIJ5n73i6aaoVpklU0urJAGrYfF7JJLbzwxKFmFM7qdEHk3c/3meAjw9twXdOE5FyyKp/JwTgtY4YXpTdVQkDhowDfB0BiSkxJbO1H7WjgSUxIEBn+hcs+4SOG2sJvtJig6dY1NGLVj0qr3tq3mhDG0PjwjAUUIG5UvziENzWqgV0xcia/YeAjbWb1GlirJFrSV0P5msa2kZhIgXOZwBTtceH2gtx0si1Vau2OT2xwGidWO9E0rwFX8jbNjoGXCMsCPKultv9WhvBMVqvgmXIKP+Cii++uIVZEzej3QgsUtrE35qG9R4kLsaDuqu1XbUGc9x09OmyJbrFJUJmOyq5+O+nEuDm4gFly+3qgt27ziTg5weA0sHMQF/BW0bvXgj/JPnWSf68zOjMrfXPdZXHsNbqtnIrzWEO+Y7DwwCErqxJmg4+22IxH1Jk+KvQ2htVPu2qRZwqYqxmUNtJ07XcPbjg6ofWOw9SBV2dooZuhoJMNtV4kjLSaxfipWi+NGTtCtNMTxrlZnbAUJ7V9cdG1QTXFyrrjDeZ6zlN06M0oJ4gWoVil10CLaYgd69mPDiPJVUs7NYrcwI//hPqg7ykAP2EC6CMFuzqpy7ZQTjwzqoG9zZdjfFfo9hZS7lKFcZA83ArG46obVI7vXfWiTXJDbguiWwqI8L/eOH4x7a143GyU9CT10fLRDp547vakhg2nhwxs8uDIZ2V/cYn9WxrXp2JaQV60FTTKX7y2Z21gsvdevhBHv1OuY0KlNpY62s1aLIYjmY3/cB1wz5yJ9kMJDIfhC25TTFocaOSVDVssJCj1KlsNbvI0sFec0+8zapxC4XDlnlzQqB/nhZrUsJbacdleB6PZwLwRCRJhhCc0Nku75lXA938zR2l0O3rD0dtKgnkNxd1yZ7bXsfIeqFa0KtOq4GzuNNaOtp8P1+hJVWOLGXI42ziqdLR0B0pOyuvTp3LgsufncL1uKmCmEjblMdtDhciJi3pq1a5CQGNljFwvS14JuqcQkcL+bers/QwNrHePsoi2hScW5UDT4S5LyNyvnlp2jtQMDqvIceawjPwbkxNowONA5SNiyFu3ia9PCkQXGlOZaPDRruTSkuVxx2wbKogYjOvtDvKnc1OBOlVe7xytkIjE1BR7VGFg/fC0p5crBmjNjIIwMI0C+y0jHzVgTyKpzys1gsw7lKuKB1vMd2s/n7RYYjF18puyLcYiTTckeb3GIoan8iGv2lTJr4/qynQH4LP0l2WfUzuv1k67tF8se7+FWdOFiHUYTgxL7y7AWdX6F2gJ5NsIlCXd204ZcOfTbBiVpK/aY1X67wbWtHa7NpAfO95Cm7aHIRnTAylNX0KconzQb6fGjlxldMszkQQz81ExwPQQ1puek7aQZ7yJq3otrfLOtLgUMUldp3yMw1AFFApEvfkN5ABqgUJNKBIGNWlgPM7ny4nyBXQS3nrLIBLibzKZM65IsuUY3K4Z70Mpdf/UPByHUpmARG3A7Cn+QgmCeEIclqcyHKRWg/CDbZ2mQSWXDYLcnaW97xJG1QaYOnpRyFyhReAJSv1N70DD4VDDc4iIrQC+yOUqEER3tUlkAoqXrGsYQuu3N8zP8+qEuEE6cHQ3BbO/uiGIdALVmR/6FXcf3zSbTNFdd6fKXYBrEVWqjlTJkAiQOEJehOMoHX/drdDcZY7zYrJZ2nq4rD9QS83tmGcuD3U3nHBTaNcENJux5Y4+q0gRaF+hQtLsVu57V48b0FpOjaaq5siT92M54cGLX6DFINQ5Dzquuz42Oky07B6rA5dfsRUSrdE0725NssI5YUvLpeLEvNz9O3V5gZ265lNpxZJsBVhcFoEi7G86paX9DCYE43fxNSMvS4grXu1kOAq9Ei5sHCv2THcPc7kOoH/fY+ZZjnd8iA3BbRAJhXdOstI3Trmm27nhqV9jX7eBdjlpkE2Z+Cd0QAc7z0ssulkfakNXFIS3OKm4WIxtA6Q7kiwOFULT3kPXqWR5HbziJ2cFp1JmjvQH6zQC8DvR1A42/4tBVVdEOh4Q9WUZVXJb7K4yLzFe3vrXN2Lnbh50e1iSyPyDhbR9G+0gQAqtIq6MGQYNgghtfJAp9rEG38wayp4pSxYPJAPjEWlux0oCFX9l+w/HHmZUugXDZ5ucTtj+l8Xmx60h31s7CsKB3bcR09QZAA70ZhDWnmAlm8I6GFCF9yeZgBDyRryn+dLnxJCtBs2LYujWi1Pk55KbpoEGVly/7+mpKaEApXbgrAiKLSabJVvvGXeB6MGXydrXSoBWSSZx20vuyVCYUzbqdcPHT1bIaap+2lAPcjXC3GJLLOt74SA7JWxi7pYyFfCnQEX30Vm6K7GyEYlbtJa2sQ6D61uhoPM65JAUtA+pm14hV4uhHhlhwKkNd9/jKYIVuy8a1rwRro5QX+zwd1R5Snwiibbzd5dhuEXPKWklwQaoqVzTLPPH6vdqhQj1r3glaXMYiicowXMXTzceCSccb3ptQpiMTSixUUv6th3DmRjXuWD5wcE1kMKhjSDcOhvri9qyk7vjdHthTo39VINu21E2h91hJWnMK55ip+P5gpclgLtGdp/Z+Rpf61VnZZAg8AD0t562caWELUUkW6ZNRXGT6xAntqFVx1qoUes7UXT4Tzs2iVPW1Ozin/JR2sit1LKcoakJO2Gpzk5O+h10qejXX4UVEDAseE71LPAinPWB9VlI2ZtpALL0h0hT0hEc1q5x1kxwqa/GwJvmD0QKDYSURFFERoDRnkIQuKsl+43rcWl7Cb30CZ+i8sPnLYppRc4+sFyOMswzZfqZiRZqiRS3TMvB5OZzdGFCsLxMc1nu0tHZfLa00mfjY3WtHU+6nk+dNRh7vokFxoN62CwRYuJCgsK+b6AYWOCVCZChUJvcfnASWOd11ElgEBcbUzjmI7nXf5HzZpQs9xf2200vpkmT+eb0JStids9eiJaWW3Y4atxwb0Zxso0ptoye3PkWenmA5hNLZePCddw/RcSPHoGnZMwOoJZLU8IuxUnocjl4zLOJOUa/uUlf0K9WOwW46Thenqhb12SHFPSqxbVcJvu1TpbkknR1ubyjgfCBnUr4qfA/D5jD2Vhs40B5wAxqAije+RI9nN+hwKegWHfQbduH6gR9vI1RcdNpd06WG7yF7e23tZm5WhjfF5AQ9T0dJJjAWVO/zZLL+mt8R0OlDt7rGmyhNdla/VM/SjJLHnYXYjKVWtgysHENad1QzwN1H94WDCUtJ7mOE9E34anFtnxBhvbXCTddKx2mpk7vJVI2jUJ616/p0KQ3G9JBbDVn2cGtHQVE8JseRAWNExJr09NzNVxCq4eY9xgOSHinI1QQHcP4WDnF3mx3oGWeVUIfOl0ptsQ8vV+D977ZB0HUbfTHMwbWMMZOVpTYrqTqKgOFT9iYZ4wYzK4rIxVdJ8lw861YbGo3PmitgXcTKxOz5DAK4mV+HmY8ksBZl3/S1bCBNxVyPm2JxDcfmtleavulrPjWr6oSvVW9Tg9amm9avLEpqEXMfl2RwnTb4ElBMiTSDb8vNXqInu1+LXlR5cnzOuc4xmMhwvTYoxES+LHcShCUVj+qcPQBlsluoPdoyGg0crKq8bo/hNicLqoJsIWZ12ghIc7ODDLcHdz3gpFF61huNXXumg7PbdKW62pW8DrFkpSpw5kjGMnY93tk0Ep+uy1VSVZAz/DPZnc25vERpud7KFIYx5pxd2p0/bB3A2Y9Gz8jfjJ7TSaz7eL1NcNrPrn1Mr7EKXRAFHBBQt4meuc5krArW6016WOILOZGyrs1OvYOgGbPqRwEb74Z1hI1P7fabEvwAPYihqJQ4OH816zY0q4NiD1Dbq0UvBdiejvY9ROPtzWpYIdBzcjD9aLuhb0FJnR7KnM3svNSjscikoTRTS3VWRORZ0C63zQiGolMpvBQ878APCHLLwMMN/chN68A5OrtJipySR57HurjmmVvnHSjyZt8I0bncSpWiZckZT7iDFIE8Ctw6mJMyPtlCrdKkB7+GfSfh9QTfjJGXOjDLsFybWU3AWHKxkpxKazOnZZdMVAELTnhwtlnDHe4EyXhzWx/KzWZB7iRp1oZeQ8xRDJfquEGAXpCyoxhSqHfp5PxEhmraVh7v6YdyffEHuaWSvvH8w3y8ErSd1ewF47flhYzDNeUeCwYT+bFerncTNq2767bzLv0w6/QR2xiWKJPzMF3V+UrB+EQH8ThmULKFg7lzuOq16/Ka9yjLZEN6QGtsmA46sz0tgKvErJG1y2xQHh8IPbLNXhx0o2PtjCQplOrbc+ljpc9jzKldXmt87c348oYb2tnv13rc0kDBBGy9czo6APLpdZ4ptB3QTDhVsEEXaRo5J9+NGVqcf1qsVsDbCtXJQWfgHJGrLG23JS1JPjUNyqGqL4zv2FgYLC5RDnUltdb/39quo+lB5Ij+Gl9d5HAEBIic440gchYgxK83w7dbu+vdddkH66aShGbo6e73Xs80FvbE/hvJ0P5NMvRgY3eBbgSEwlYEzJwufvCi0QDtiBxHO20SLsK8FciA5V+P/Pfvr9XwZ0nRyyP4BUGqK+sWrEQWAgU6mTqoDE6Y47J8YaisfyMEFSV+endC668MmGbKuaRq4uj66mFxIwooODXDjiQcU14SbaBY+yTrT/tO74fNZhdtQfmq8S9CDzgy40yiVtI1GwrEEzMo9/7+hQ51tUZhEW9WObU+aA4yI1mJLXqUTuoEHP34aBCTV9NeF04ebeNSuOkR43EahUnqG9DR1VtGG6Koxafv3gzD3T9UGgbxUcQNZLXhyekN/tJow+xx9im+rAQvrAIMH/SM44Fgjw4QSgpXsNQSJMfjU/T6xMc5S6++o4o9pcUAlWdhqEXVickAgR24B3oQVDvfDY4B62IpNYUjTev225/dBqAYS/kaeEkVrye+zjWV6Mjmr04Lqce7beSdiQNGYr34m0Drg7ICsxZh/ZwWZ+csbvlaq/hkz3LZGJTBrvyfPQTK0IM1t6rL0Sc9tYhFKGoHa3M8ZEotlU4+jhhaG8bkhq4gCb/A0IwLzJB1t0xv5CTVCfK/EJWxYn4qF3OwEJsaMdGgmfN92M5jd6C/Fts5sZUxAaipA6V5TIy0yHvdX8RIzViQEOgxmH6hWiBXaOJqzHnzqskt/eo+iGPxVPEZfb59sKrjC51c7mHQtAUYaHv/FRvukrTUgjd4gkVs2vA82dK/RaqpI/r3ckhH6EbOj6r60yA+VmAf1OctD9s798nV7ZRwrkI5xNcEoKtCKFianwmn9cZ1wd54CSHlWArugA0h91PmZ70Z4UWIA8rYkip3kZZyt6Beh4qlTsNwh4ahKXomluMxCg3WvMfU9f0rj9SDkx3F5ys8WrZq2vhuaPm2zWod8z37hsZQ3+niIrVkzNMzFn5lvT/qUpI+X9qyMoi5i+2i8QaIXVeHD4bSHQIfaUCp7SY/KXhgz4YVGQ2oQ8EsLJSlvwqHFIlNTd0SJpeBbsaZpRTrLDd3GMjB2T4Q+sFjIpzhc9a0llrh3LaP76phqeQaHNs0cwJiMcvpkvNdVd81vFUHPGw1AKCKtD7N3G+WNXhzXOwb2Zvdjy6EYaPIZCFyaJMrKSrGM3uYmr6h7dDI2eN5DQ+nfBL5QKatIoWD9V2/4WTqR202kFLk1tZCYECocbZhM5lXjWr6+chUVWXz5KdW0Zk8Wc7U/mRFc6MYjJ7r1azQ677Papiv6S4uruMWsS/xUuKYX/SzGRUzMoAFAHPz6Uum1S39TGJ9oC3EF93aarVhLUVtJChxmNxeDSBOcAHAqEg8PQE6iiuF2kmYPlAtGwZoSUYmqFtBBbWlicu8slAjxnSNFwx2Zrm73ZfslUFcHUjbV7BghHHSII9R7GFxWMsYKIJ18JFxgQ9+E33EsCtNWY/reojygi9eISbgx9hh204M/qxuT1DikQddvX0uBiHcGq74nzAMndjb8AwhOgnaaFh/FYa/fP5rEe/CPAgo4Xk8nynOqXPCA5gy1gr0fRcHStaj5VQ4u9PZ/lDfO1hR1vUwl97JcR7O+4wJiXgXuM+b/NZFogobiFbeJCPBPR3GBV0Dy7WT+3dsr4mjwFMB+jcKYrjKXccNbbN64IjUTqTJqQtZ6Bzb7juPoJ2rV3k4ou4iE815WAiHdJHFkLgnce4uwuUpEk7c5zEx38xReonG54Z3zePJyCSc8EkzKlxphgMBlYV8hk5+axal6FxjbZoybAtMeZnQYtVbpNa8exhiI50bnXTUZJLj9PGYXEm48I2+d8AXsW9A2Ym1tObLLksMByzlMx9JckQhVTWrWl0h+mttL2V7ecNUd5xilJOCIUQa1MUdZE1G8dGZbiaodKhNqOOqZXB8Zl5yLufljM58i4lMQSkXEFjhyVTcbSsWe8l4Il9B/AlR3lvhB8vFcczM0dnCw31dziXcA5SLgGtI8ELabqG40AhzarnC9w+F5OIhCMxy9KS95+WT78ksnFj1XtygW3Fy5f2oDkIWM+xmpRPEKsNRaTSlxx46z+UbKGw/l+uSb9kQ6JXrgOLG9qH2acoNyhGMYQBrFAJ18dLgsMzcVfnJYFxUD1hxxCV8OixN40HGWxn9s4/DvSHuwzGxfMVg954JKMhENzYDM6lAvBbxapNsk8MSrZGS6ZoXgE5KJjAE15fd14yLyxk7HWuh7qnsr+MyjmFdN3FY9fgN6hnWgXSf98LEUmgK4XVXocOphGfSLk06IZD7Q3bB4hAUEn4BDY615FfIHdrny2QUHD7r0AkjZSV6J3swADxOwew98sT9tncf6aAJMBkiE3BP9k2a1F4Un+eQcabooKt/1+WkC6YUFHNF4Sd8L57lgqU7MNR7U3SrxtQczCR8r/Z6ReHhjXed/5rC1ROOBemR9qg2dZVwgZjB9CCrfFpe1XiDLJjBp/tmMLKaBNUYFWUlceaqTzYfyfWix68Cmo+caNJAC/vQu2C221x04F3rNjI5peldQUsubANdbg2C+d77CrubXQMu3ZjQz2oHRDW3Ql4aqiIXbXBKdO/km/MYXJmzruXQPd3dC0700Np1vGymiVIyHu2t6TJzAsP9Z8cLZVW7OPzFYXpZoxXPng1Y9dLcQH7nL/gMNxM5r2im8oFoPYZ+ZLEO6npGIKrz+NyZOxgvY9jO5Xc08u+W2ESFPw2Mj0TnYAkf6P4YsAQQOhAtJmL2hwvA6zjtnXrFpnHcOBSEG3+5hhNeAXYx4rjcvQWZRRxG3dTIx7QnAx3+xRRM3ayHFcFmAjYRXZYgkwLGxfyBldAX6wOFI200Ai5CMZVfUzumKpITZmXjGlr7hOav1LFn3SnKazyvyD3+wT+ivUJytcfgIBFWyEVfXsriz04yCX+F6nG+TKEEPY8r3jKCSM39p00O/+f35o8nxf9A2b/p3PsX/X3/tpkvikH/hHH6txf5h8a+MIL/+QleCEX9RVNf/LrQ/9zUFyhx47j+7jNxSaZKG/MX+Ma/AA== \ No newline at end of file diff --git a/docs/Security-Compliance/File-Transfer-TDRS/diagram.png b/docs/Security-Compliance/File-Transfer-TDRS/diagram.png deleted file mode 100644 index 476b82d33..000000000 Binary files a/docs/Security-Compliance/File-Transfer-TDRS/diagram.png and /dev/null differ diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md b/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md index dbd6920c1..95c1a4604 100644 --- a/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md +++ b/docs/Technical-Documentation/Architecture-Decision-Record/004-configuration-by-environment-variable.md @@ -9,9 +9,11 @@ Accepted Applications need to be configured differently depending on where they are running. For example, the backend running locally will have different configuration then the backend running in production. +Further, environment variables can be designated "secret" or not; the term "secret key" is often used in place of secret environment variables. Secret keys are sometimes (but not always) shared between different deployment environments, which makes it useful to have a central "single source of truth" where a secret key can be kept and copied out to different environments. CircleCI solves this use case for us, allowing secret keys to be managed by the project's Environment Variables, and accessed in the deployment process to write to cloud.gov applications. + ## Decision -We will use environment variables to configure applications. +We will use environment variables to configure applications. We will use Environment Variables in CircleCI to store and manage secret keys. ## Consequences diff --git a/docs/Technical-Documentation/secret-key-rotation-steps.md b/docs/Technical-Documentation/secret-key-rotation-steps.md index bb10ad880..b6f58c39e 100644 --- a/docs/Technical-Documentation/secret-key-rotation-steps.md +++ b/docs/Technical-Documentation/secret-key-rotation-steps.md @@ -6,7 +6,6 @@ To maintain good security, we will periodically rotate the following secret keys - CF deployer keys (_for continuous delivery_) - JWT keys (_external user auth_) - ACF AMS keys (_internal user auth_) -- ACF Titan server keys (_for file transfers between TDP and TDRS_) - Django secret keys ([_cryptographic signing_](https://docs.djangoproject.com/en/4.0/topics/signing/#module-django.core.signing)) This document outlines the process for doing this for each set of keys. @@ -154,61 +153,6 @@ Service requests tickets must be submitted by Government-authorized personnel wi 2. Update environment variables in CircleCI and relevant cloud.gov backend applications after ticket completed by OCIO. [Restage applications](https://cloud.gov/docs/deployment/app-maintenance/#restaging-your-app). -**
ACF Titan Server Keys** -The ACF OCIO Ops team manages these credentials for all environments (dev, staging, and prod), so we will need to submit a service request ticket whenever we need keys rotated. - -Service requests tickets must be submitted by Government-authorized personnel with Government computers and PIV access (e.g. Raft tech lead for lower environments and TDP sys admins for production environment). Please follow the procedures below: - -1. Generate new public/private key pair - -Below is an example of how to generate new titan public/private key pair from _Git BASH for Windows_. Two files called `filename_where_newtitan_keypair_saved` are created: one is the _private_ key and the other is a _public_ key (the latter is saved with a _.pub_ extention). -(note: the info below is not associated with any real keys) - -``` -$ ssh-keygen -t rsa -b 4096 -Generating public/private rsa key pair. - -Enter file in which to save the key (/c/Users/username/.ssh/id_rsa): filename_where_newtitan_keypair_saved - -Enter passphrase (empty for no passphrase): - -Enter same passphrase again: - -Your identification has been saved in filename_where_newtitan_keypair_saved - -Your public key has been saved in filename_where_newtitan_keypair_saved.pub - -The key fingerprint is: -SHA256:BY6Nl0hCjIrI9yZMBGH2vbDFLCTq2DsFQXQTmLydwjI - -The key's randomart image is: -+---[RSA 4096]----+ -| X*B*.. . | -|+ O+=+ * o | -|=oo* *+ = . | -|Eo++B .. . | -|.+=oo. S | -| = o | -| o o | -| . | -| | -+----[SHA256]-----+ -``` - -2. Submit request tickets from government-issued email address and use the email template located on **page 2** of [this document.](https://hhsgov.sharepoint.com/:w:/r/sites/TANFDataPortalOFA/Shared%20Documents/compliance/Authentication%20%26%20Authorization/ACF%20AMS%20docs/OCIO%20OPERATIONS%20REQUEST%20TEMPLATES.docx?d=w5332585c1ecf49a4aeda17674f687154&csf=1&web=1&e=aQyIPz) cc OFA tech lead on lower environment requests. - -The request should include: -- the titan service account name (i.e. `tanfdp` for prod; `tanfdpdev` for dev/staging) -- the newly generated public key from `filename_where_newtitan_keypair_saved.pub` - -3. When OCIO confirms that the change has been made, add the private key from `filename_where_newtitan_keypair_saved` to CircleCI as an environment variable. The variable name is `ACFTITAN_KEY`. **Please note**: the value needs must be edited before adding to CircleCI. It should be a one-line string with underscores ("_") replacing the spaces at the end of every line. See example below: - -``` ------BEGIN OPENSSH PRIVATE KEY-----_somehashvalue_-----END OPENSSH PRIVATE KEY----- -``` - -4. Re-run the deployment workflow from CircleCI and confirm that the updated key value pair has been added to the relevant cloud.gov backend application. -
**
Django secret keys** diff --git a/product-updates/img/error-reports/click-file-name-download.png b/product-updates/img/error-reports/click-file-name-download.png new file mode 100644 index 000000000..7cfd5bb31 Binary files /dev/null and b/product-updates/img/error-reports/click-file-name-download.png differ diff --git a/product-updates/img/error-reports/error-report.png b/product-updates/img/error-reports/error-report.png new file mode 100644 index 000000000..bbb9594e9 Binary files /dev/null and b/product-updates/img/error-reports/error-report.png differ diff --git a/product-updates/img/error-reports/flatfile.png b/product-updates/img/error-reports/flatfile.png new file mode 100644 index 000000000..474479b0e Binary files /dev/null and b/product-updates/img/error-reports/flatfile.png differ diff --git a/product-updates/img/getting-started/data-submitted.png b/product-updates/img/getting-started/data-submitted.png index e65dedfab..07f2c0868 100644 Binary files a/product-updates/img/getting-started/data-submitted.png and b/product-updates/img/getting-started/data-submitted.png differ diff --git a/product-updates/knowledge-center/about-email-notifications.html b/product-updates/knowledge-center/about-email-notifications.html index 1a0fe2987..d94a7aacd 100644 --- a/product-updates/knowledge-center/about-email-notifications.html +++ b/product-updates/knowledge-center/about-email-notifications.html @@ -187,6 +187,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -251,6 +255,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -316,11 +324,13 @@

    Access Request Status

  • Data Submitted by Section

    - Upon successful submission of your data files you will also receive an email confirmation for each section from no-reply@tanfdata.acf.hhs.gov. + When data files you've submitted have been processed by the system you will also receive an email confirmation for each section from no-reply@tanfdata.acf.hhs.gov.
    Data Submitted by Section + + These emails will also highlight whether or not the system found errors in your data. Note that OFA may still reach out to you via email with additional feedback on your data even if TDP detected no errors. Read more about error reports.

  • diff --git a/product-updates/knowledge-center/complete-resubmissions.html b/product-updates/knowledge-center/complete-resubmissions.html index 80899a166..955b726b3 100644 --- a/product-updates/knowledge-center/complete-resubmissions.html +++ b/product-updates/knowledge-center/complete-resubmissions.html @@ -189,6 +189,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -253,6 +257,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/create-new-login.html b/product-updates/knowledge-center/create-new-login.html index a8e20c76f..f63ed3888 100644 --- a/product-updates/knowledge-center/create-new-login.html +++ b/product-updates/knowledge-center/create-new-login.html @@ -189,6 +189,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -252,6 +256,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/existing-login.html b/product-updates/knowledge-center/existing-login.html index b45a5b983..81a9540f9 100644 --- a/product-updates/knowledge-center/existing-login.html +++ b/product-updates/knowledge-center/existing-login.html @@ -185,6 +185,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -250,6 +254,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/exporting-complete-data-using-ftanf.html b/product-updates/knowledge-center/exporting-complete-data-using-ftanf.html index a19e0b65c..d357b3b21 100644 --- a/product-updates/knowledge-center/exporting-complete-data-using-ftanf.html +++ b/product-updates/knowledge-center/exporting-complete-data-using-ftanf.html @@ -188,10 +188,16 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • + +
  • Exporting Complete Data Using FTANF
  • @@ -252,6 +258,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/faq.html b/product-updates/knowledge-center/faq.html index 09420c95c..f5dc4b41d 100644 --- a/product-updates/knowledge-center/faq.html +++ b/product-updates/knowledge-center/faq.html @@ -171,6 +171,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -236,6 +240,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/give-feedback.html b/product-updates/knowledge-center/give-feedback.html index f0db6d203..f47ea6e28 100644 --- a/product-updates/knowledge-center/give-feedback.html +++ b/product-updates/knowledge-center/give-feedback.html @@ -199,6 +199,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -264,6 +268,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/index.html b/product-updates/knowledge-center/index.html index 0850521df..758b76c84 100644 --- a/product-updates/knowledge-center/index.html +++ b/product-updates/knowledge-center/index.html @@ -171,6 +171,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -238,6 +242,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/managing-your-account.html b/product-updates/knowledge-center/managing-your-account.html index 3365d2080..bfef7e8db 100644 --- a/product-updates/knowledge-center/managing-your-account.html +++ b/product-updates/knowledge-center/managing-your-account.html @@ -175,6 +175,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -241,6 +245,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/uploading-data.html b/product-updates/knowledge-center/uploading-data.html index 48730eaeb..cc42c4019 100644 --- a/product-updates/knowledge-center/uploading-data.html +++ b/product-updates/knowledge-center/uploading-data.html @@ -176,6 +176,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -242,6 +246,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • diff --git a/product-updates/knowledge-center/view-submission-history.html b/product-updates/knowledge-center/view-submission-history.html index 9fc7e911c..a825f9142 100644 --- a/product-updates/knowledge-center/view-submission-history.html +++ b/product-updates/knowledge-center/view-submission-history.html @@ -189,6 +189,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -256,6 +260,10 @@ Viewing Submission History +
  • + Understanding Error Reports and File Structure +
  • +
  • Resubmitting Complete Data Files
  • @@ -449,7 +457,7 @@

    Understanding File Statuses and Aggregate Data

    Total Errors - Since section 3 and 4 data contain aggregate values in one record (rather than case data with many records), TDP can only provide the total number of errors detected in the record. Errors here likely relate to values outside those defined in the TANF/SSP Coding Instructions or Tribal TANF Coding Instructions and/or inconsistencies in the values between related elements for a given reporting month. + Since section 3 and 4 data contain aggregate values in one record (rather than case data with many records), TDP can only provide the total number of errors detected in the record. Errors here likely relate to values outside those defined in the TANF/SSP Coding Instructions or Tribal TANF Coding Instructions and/or inconsistencies in the values between related elements for a given reporting month. diff --git a/product-updates/knowledge-center/viewing-error-reports.html b/product-updates/knowledge-center/viewing-error-reports.html new file mode 100644 index 000000000..c8e458ad1 --- /dev/null +++ b/product-updates/knowledge-center/viewing-error-reports.html @@ -0,0 +1,718 @@ + + + + + + TDP — Understanding Error Reports and File Structure + + + + + + + + + + + + + + + + + Skip to main content + + +
    +
    +
    +
    +
    + U.S. flag +
    +
    +

    + An official website of the United States government +

    + +
    + +
    +
    +
    +
    +
    + +
    +

    + Official websites use .gov
    A + .gov website belongs to an official government + organization in the United States. +

    +
    +
    +
    + +
    +

    + Secure .gov websites use HTTPS
    A + lock ( + + + + + ) or https:// means you’ve safely connected to + the .gov website. Share sensitive information only on official, + secure websites. +

    +
    +
    +
    +
    +
    +
    + + + + + + + + + + + + + + + + + + + +
    +
    + +
    + + + +
    +
    +
    + + + + + +
    +

    Understanding Error Reports and File Structure

    +
    + +

    + Error reports are generated for each file you submit when the TDP system detects potential data quality issues in your file. These can range from issues with the file layout (which can prevent TDP from being able to fully process your files) to issues related to specific records and/or cases in your file. +

    +
    +

    This guide provides instruction on how to access, understand, and address issues listed in your error reports. Also Included herein is a brief overview of how data is organized in your files.

    +
    +

    Please note: The error reports were designed to help you to correct a wide variety of data issues. Most of the errors that may be listed in these reports are based on the file record layouts and coding instructions, which are publicly available and accessible from the TDP home page. These reports do not capture every possible data quality issue. The OFA TANF data team may still reach out to you via email with additional feedback.

    +
    +

    Jump to:

    + + +
    + +
    +
    + +

    Download and View Error Reports

    +
    +

    Error reports can be accessed from the Data Files page by navigating to the Submission History tab for a given fiscal year and quarter. Read more about accessing Submission History. + +

    + +

    To access error reports for submitted files, select the link in the ‘Error Report’ column to download error feedback associated with the relevant version of the file.

    +
    + + + + Submission history table with the download error report link emphasized + + +
    + +
    +
    + +

    Data File Structure

    +
    + + + +

    The files that you submit on a quarterly basis for your TANF or SSP program contain large amounts of data in a specific format that enables TDP to process and provide feedback when uploaded and submitted. Please note that these files are designed for system processing and, as such, are not as human-readable. For example, below is an illustration of a short TANF, Section 1 (Active Case Data) file:

    + + + TANF data file opened in a text editor beginning with a Header record and ending with a Trailer record + + + +
    +

    Included below are descriptions of how the data files are classified (or recognized) by the system. These classifications determine the data components to be included in each file and how the components are organized.

    +
    +

    Classifications of Data Files

    +
    +

    Program Type

    +
    +

    Files are first identified by their program type: Tribal TANF, TANF (ACF-199), or Separate State Program Maintenance-of-Effort (ACF-209). Each type of data file has its own unique elements, which are documented in the coding instructions linked below. +

    + + +
    + +

    Sections of Data

    +
    +

    TANF and SSP-MOE data covering each fiscal quarter period are reported in up to four sections of data files:

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Section Definitions +
    Section NameDescription
    Active Case Data + Families currently receiving TANF or SSP-MOE benefits +
    Closed Case Data + Families who are no longer receiving TANF or SSP-MOE benefits. +
    Aggregate Data + Aggregate counts quantifying the total number of families, recipients, and amount of assistance +
    Stratum Data + Aggregate counts quantifying the total number of families by section and stratum +Note: this section of the data report is only required for some jurisdictions. + Read our FAQ for more detail on Stratum data submission requirements +
    + +
    + +

    Components of Data

    +
    +

    Records

    +
    +

    Each data file contains between 3 and 5 different types of records (including the header and trailer).

    +
    +

    While header and trailer records are included in every data file, the other record types included in each file vary by section. These record types (described below) are referenced in the error message associated with each data quality issue included in the error report. Please note that T1, T2,...,T7 record types are relevant to TANF and Tribal TANF data files, whereas M1,M2,...,M7 record types refer to the SSP-MOE data files.

    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Record Definitions +
    Record TypeDescriptionAssociated Section
    T1, M1 +Family-level data for a specific month the family is receiving assistance. + + Section 1 (Active Case Data) +
    T2, M2 + Person-level data for adults and minor heads-of-household for a specific month that the family is receiving assistance. + + Section 1 (Active Case Data) +
    T3, M3 + Person-level data for minor non- heads-of-household for a specific month that the family is receiving assistance. + + Section 1 (Active Case Data) +
    T4, M4 + Family-level data for the last month the family received assistance. + + Section 2 (Closed Case Data) +
    T5, M5 + Person-level data for the last month that the family received assistance. + + Section 2 (Closed Case Data) +
    T6, M6 + Aggregate data of families applying for and receiving assistance, by month. + + Section 3 (Aggregate Data) +
    T7, M7 + Aggregate data of families receiving assistance, by month and stratum. + + Section 4 (Stratum Data) +
    +
    +

    Header and Trailer Records

    +
    +

    The Header and Trailer refer to special records at the beginning and end of every data file. The Header communicates key information to TDP about the file's classification that helps the system correctly process it, including calendar year and quarter, program type, and section. The Trailer contains information about the number of records (excluding the header and trailer records) in the file.

    +
    +

    Examples of Header and Trailer records:

    + +
    + +HEADER20204G02000TAN2 D + +
    + TRAILER0000042 + +
    + +

    Typically, these records will be generated automatically as part of your current process for exporting data files. Please note: TDP does require that the update indicator, one of the items making up the Header record, be set to a value of "D" (meaning "Delete"). The update indicator is an instruction to the system to delete and replace any existing data in the TDP database for the specified quarter, program type, and section with the data included in the submitted file). Read more about how to submit complete data.

    +
    +

    Items

    +
    +

    Items (sometimes referred to as "data elements") are the building blocks of every record. Items are references to the coding instructions. For example, Item 11 of Section 1 of the TANF Data Report is "Number of Family Members". Each data quality issue in the error report will, if applicable, include the associated Item Number and Item Name for reference to the coding instructions, which include all acceptable values for each item, their meanings, and guidelines for their use.

    +
    +

    Some items have a static list of acceptable values, such as the Item 10 (Newly-Approved Applicant), where the acceptable values are either "1" (Yes, a newly-approved applicant) or "2" (No). Others, like Item 32 (Date of Birth), have a broader range of specific formats for which to report values (e.g. YYYYMMDD).

    + +
    + +
    +
    + +

    Overview of the Error Report

    +
    +

    TDP's error reports are designed to provide you with key information you may need to quickly identify records that have been flagged for data quality issues. This includes the following:

    +
    +
      +
    • Case Number, Year, and Month — which can be used to identify which cases have errors and the reporting period those errors are specific to (if applicable).
    • +
    • Error Message — which describes the error and how the related items are logically connected.
    • +
    • Item Number and Item Name — which can be used alongside the error message to cross-reference with the coding instructions.
    • +
    • Internal Variable Name — a value specific to TDP's database for use in support sessions with the TANF data team.
    • +
    • Row Number — identifying which row of the submitted file contains the record associated with a given error.
    • +
    +
    + + + + TDP error report containing some of the error examples below opened in Excel + +
    +

    Examples of Common Errors

    +
    +

    Below are examples of error messages associated with common issues that may be listed in an error report.

    + + + +
    +
    +

    + Some error messages that you may encounter in error reports are still in development and may have been listed incorrectly or need further clarity. Please feel free to reach out to tanfdata@acf.hhs.gov with questions or requests for additional guidance. We continuously update the system based on user feedback and encourage you to reach out if you encounter any confusing errors or believe the system incorrectly identified an issue. +

    +
    + +
    +
    +

    Errors related to header or trailer records:

    +
    +

    Header-related errors are often the result of submitting files for a fiscal period that is not consistent with the time period in the header record (e.g. trying to submit 2022 data for a 2024 submission). Other header or trailer errors may be related to how the file was generated (e.g. the file produced is missing a header or trailer record). Some examples of how these types of error may appear in your error report are included below:

    + + +
    +Submitted reporting year: 2024, quarter: 1 doesn't match file reporting year 2022, quarter: 4. + +
    + Your file does not begin with a HEADER record. + +
    + + +

    Please refer to the Transmission File Header Record definitions to compare your file's header or trailer to the expected layout. + + +

    +

    Errors related to record length:

    +
    +

    Record length-related errors will be raised if the specified record is not aligned with the record layout requirements. For example, this kind of error may appear as follows in the report:

    +
    + + + + + T6 record length is 409 characters but must be 379. + + + +
    +

    Please refer to the Transmission File Layout documents to compare your records against their expected layouts.

    + +
    +

    Errors related to invalid values for a specific item/data element:

    +
    +

    Invalid value errors can come up when a specific item/data element has an unexpected value (e.g. a letter or a symbol was reported for the zip code field, such as: "462$1"):

    + +
    + Item 17 (ZIP code) must be numeric. + +
    + +

    To remedy these type of issues, you can compare the value reported in the file for a given item number to the allowable values in the coding instructions associated to the relevant item number. The coding instructions are linked below:

    + + + +
    +

    Errors related to inconsistent values for related items/data elements in the same record:

    +
    +

    Some errors may require review of the coding instructions for multiple items (and their respective values) to determine the proper correction. In the example below, the error is communicating that the value reported for Item 49 is in a conflict with the value for Item 30 in the same record. This message suggests a problem with either the value of Item 49 or the value of Item 30. Refer to the coding instructions and your own data to determine which value needs to be corrected.

    + + + +
    + If Item 30 (Family Affiliation) is 1 then Item 49 (Work Participation Status) must be in set of values [01, 02, 05, 07, 09, 15, 17, 18, 19, 99]. + +
    + +
    +

    Errors related to inconsistent values across related records:

    +
    +

    Errors with inconsistent values across related records may require review of the coding instructions to determine the proper correction. In the example below, the error is communicating that a T1 (family) record was found in the file that did not have a corresponding T2 (adult) or T3 (child) record, which effectively means that person-level records associated with this family are missing from the file.

    + + + +
    + Every T1 record should have at least one corresponding T2 or T3 record with the same Report Month & Year and Case Number. + +
    + +
    + +
    +
    + +

    New Coding Instructions Guidance for States and Territories

    +
    +

    Some errors flagged in error reports may be associated with values and/or fields that have been deprecated by the updated coding instructions for states and territories. You can refer to the public guidance for a timeline to correct these.

    +
    +

    If you submit files based on the old instructions, this will not negatively impact your submission, work participation rates, or published data. You will, however, begin to see errors in the TDP error reports that reflect the new instructions, but these errors will not impact whether your data is accepted.

    +
    +

    Please contact the data team at tanfdata@acf.hhs.gov with any questions.

    +
    + + + +
    +
    +
    + +
    diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index 7a742ad79..24bef90d9 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -42,9 +42,6 @@ echo backend_app_name: "$backend_app_name" set_cf_envs() { var_list=( - "ACFTITAN_HOST" - "ACFTITAN_KEY" - "ACFTITAN_USERNAME" "AMS_CLIENT_ID" "AMS_CLIENT_SECRET" "AMS_CONFIGURATION_ENDPOINT" @@ -62,6 +59,7 @@ set_cf_envs() "REDIS_URI" "JWT_KEY" "STAGING_JWT_KEY" + "SENDGRID_API_KEY" ) echo "Setting environment variables for $CGAPPNAME_BACKEND" diff --git a/tdrs-backend/.env.example b/tdrs-backend/.env.example index af513c929..5ffe271c1 100644 --- a/tdrs-backend/.env.example +++ b/tdrs-backend/.env.example @@ -86,6 +86,3 @@ ELASTIC_HOST=elastic:9200 # testing CYPRESS_TOKEN=local-cypress-token - -# sftp -ACFTITAN_SFTP_PYTEST=local-acftitan-key diff --git a/tdrs-backend/Pipfile b/tdrs-backend/Pipfile index 7ab59800e..117e86c75 100644 --- a/tdrs-backend/Pipfile +++ b/tdrs-backend/Pipfile @@ -51,8 +51,6 @@ celery = "==5.2.6" redis = "==4.1.2" flower = "==1.1.0" django-celery-beat = "==2.2.1" -paramiko = "==2.11.0" -pytest_sftpserver = "==1.3.0" elasticsearch = "==7.13.4" # REQUIRED - v7.14.0 introduces breaking changes django-elasticsearch-dsl = "==7.3" django-elasticsearch-dsl-drf = "==0.22.5" diff --git a/tdrs-backend/Pipfile.lock b/tdrs-backend/Pipfile.lock index d62cb708b..c2cdbfdc0 100644 --- a/tdrs-backend/Pipfile.lock +++ b/tdrs-backend/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e0b5d173936dd0ae24d434e543a96d139eed6ab962f1f92b1fe354d74d9a0599" + "sha256": "a082fb8d3118128843dec21e83b70a4ee5d9743a2e869918452d1b8c47533edc" }, "pipfile-spec": 6, "requires": { @@ -39,39 +39,6 @@ ], "version": "==2.4.1" }, - "bcrypt": { - "hashes": [ - "sha256:01746eb2c4299dd0ae1670234bf77704f581dd72cc180f444bfe74eb80495b64", - "sha256:037c5bf7c196a63dcce75545c8874610c600809d5d82c305dd327cd4969995bf", - "sha256:094fd31e08c2b102a14880ee5b3d09913ecf334cd604af27e1013c76831f7b05", - "sha256:0d4cf6ef1525f79255ef048b3489602868c47aea61f375377f0d00514fe4a78c", - "sha256:193bb49eeeb9c1e2db9ba65d09dc6384edd5608d9d672b4125e9320af9153a15", - "sha256:2505b54afb074627111b5a8dc9b6ae69d0f01fea65c2fcaea403448c503d3991", - "sha256:2ee15dd749f5952fe3f0430d0ff6b74082e159c50332a1413d51b5689cf06623", - "sha256:31adb9cbb8737a581a843e13df22ffb7c84638342de3708a98d5c986770f2834", - "sha256:3a5be252fef513363fe281bafc596c31b552cf81d04c5085bc5dac29670faa08", - "sha256:3d3b317050a9a711a5c7214bf04e28333cf528e0ed0ec9a4e55ba628d0f07c1a", - "sha256:48429c83292b57bf4af6ab75809f8f4daf52aa5d480632e53707805cc1ce9b74", - "sha256:4a8bea4c152b91fd8319fef4c6a790da5c07840421c2b785084989bf8bbb7455", - "sha256:4fb253d65da30d9269e0a6f4b0de32bd657a0208a6f4e43d3e645774fb5457f3", - "sha256:551b320396e1d05e49cc18dd77d970accd52b322441628aca04801bbd1d52a73", - "sha256:5f7cd3399fbc4ec290378b541b0cf3d4398e4737a65d0f938c7c0f9d5e686611", - "sha256:6004f5229b50f8493c49232b8e75726b568535fd300e5039e255d919fc3a07f2", - "sha256:6717543d2c110a155e6821ce5670c1f512f602eabb77dba95717ca76af79867d", - "sha256:6cac78a8d42f9d120b3987f82252bdbeb7e6e900a5e1ba37f6be6fe4e3848286", - "sha256:8a893d192dfb7c8e883c4576813bf18bb9d59e2cfd88b68b725990f033f1b978", - "sha256:8cbb119267068c2581ae38790e0d1fbae65d0725247a930fc9900c285d95725d", - "sha256:9f8ea645eb94fb6e7bea0cf4ba121c07a3a182ac52876493870033141aa687bc", - "sha256:c4c8d9b3e97209dd7111bf726e79f638ad9224b4691d1c7cfefa571a09b1b2d6", - "sha256:cb9c707c10bddaf9e5ba7cdb769f3e889e60b7d4fea22834b261f51ca2b89fed", - "sha256:d84702adb8f2798d813b17d8187d27076cca3cd52fe3686bb07a9083930ce650", - "sha256:ec3c2e1ca3e5c4b9edb94290b356d082b721f3f50758bce7cce11d8a7c89ce84", - "sha256:f44a97780677e7ac0ca393bd7982b19dbbd8d7228c1afe10b128fd9550eef5f1", - "sha256:f5698ce5292a4e4b9e5861f7e53b1d89242ad39d54c3da451a93cac17b61921a" - ], - "markers": "python_version >= '3.7'", - "version": "==4.1.3" - }, "billiard": { "hashes": [ "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", @@ -694,14 +661,6 @@ "markers": "python_version >= '3.7'", "version": "==24.0" }, - "paramiko": { - "hashes": [ - "sha256:003e6bee7c034c21fbb051bf83dc0a9ee4106204dd3c53054c71452cc4ec3938", - "sha256:655f25dc8baf763277b933dfcea101d636581df8d6b9774d1fb653426b72c270" - ], - "index": "pypi", - "version": "==2.11.0" - }, "parso": { "hashes": [ "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", @@ -868,36 +827,11 @@ "markers": "python_version >= '3.6'", "version": "==2.4.0" }, - "pynacl": { - "hashes": [ - "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", - "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", - "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", - "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", - "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", - "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", - "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", - "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", - "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", - "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" - ], - "markers": "python_version >= '3.6'", - "version": "==1.5.0" - }, - "pytest-sftpserver": { - "hashes": [ - "sha256:b7ac34a23f63d77e27f67b6a81c9418243733f027eeb8a3061d965b2da7e5cab", - "sha256:c5e8a37049866d4eabc711db9f1c09e1c02ab72ba290f5fd244939c9a188042f" - ], - "index": "pypi", - "version": "==1.3.0" - }, "python-crontab": { "hashes": [ - "sha256:6d5ba3c190ec76e4d252989a1644fcb233dbf53fbc8fceeb9febe1657b9fb1d4", - "sha256:79fb7465039ddfd4fb93d072d6ee0d45c1ac8bf1597f0686ea14fd4361dba379" + "sha256:f4ea1605d24533b67fa7a634ef26cb59a5f2e7954f6e677d2d7a2229959a2fc8" ], - "version": "==3.0.0" + "version": "==3.1.0" }, "python-dateutil": { "hashes": [ @@ -1040,11 +974,11 @@ }, "setuptools": { "hashes": [ - "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987", - "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32" + "sha256:54faa7f2e8d2d11bcd2c07bed282eef1046b5c080d1c32add737d7b5817b1ad4", + "sha256:f211a66637b8fa059bb28183da127d4e86396c991a942b028c6650d4319c3fd0" ], "markers": "python_version >= '3.8'", - "version": "==69.5.1" + "version": "==70.0.0" }, "six": { "hashes": [ @@ -1381,11 +1315,11 @@ }, "faker": { "hashes": [ - "sha256:6737cc6d591cd83421fdc5e494f6e2c1a6e32266214f158b745aa9fa15687c98", - "sha256:c153505618801f1704807b258a6010ea8cabf91f66f4788939bfdba83b887e76" + "sha256:45b84f47ff1ef86e3d1a8d11583ca871ecf6730fad0660edadc02576583a2423", + "sha256:cfe97c4857c4c36ee32ea4aaabef884895992e209bae4cbd26807cf3e05c6918" ], "markers": "python_version >= '3.8'", - "version": "==25.0.1" + "version": "==25.2.0" }, "flake8": { "hashes": [ diff --git a/tdrs-backend/docker-compose.local.yml b/tdrs-backend/docker-compose.local.yml index d2cd5289c..2de355c9c 100644 --- a/tdrs-backend/docker-compose.local.yml +++ b/tdrs-backend/docker-compose.local.yml @@ -68,12 +68,8 @@ services: - AMS_CLIENT_ID - AMS_CLIENT_SECRET - AMS_CONFIGURATION_ENDPOINT - - ACFTITAN_HOST - - ACFTITAN_KEY - - ACFTITAN_USERNAME - REDIS_URI=redis://redis-server:6379 - REDIS_SERVER_LOCAL=TRUE - - ACFTITAN_SFTP_PYTEST - SENDGRID_API_KEY volumes: - .:/tdpapp diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index dba1be5de..07d014fb5 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -91,12 +91,8 @@ services: - AMS_CLIENT_ID - AMS_CLIENT_SECRET - AMS_CONFIGURATION_ENDPOINT - - ACFTITAN_HOST - - ACFTITAN_KEY - - ACFTITAN_USERNAME - REDIS_URI=redis://redis-server:6379 - REDIS_SERVER_LOCAL=TRUE - - ACFTITAN_SFTP_PYTEST - CYPRESS_TOKEN - DJANGO_DEBUG - SENDGRID_API_KEY diff --git a/tdrs-backend/docs/api/authentication.md b/tdrs-backend/docs/api/authentication.md index e51f2174d..177c23213 100644 --- a/tdrs-backend/docs/api/authentication.md +++ b/tdrs-backend/docs/api/authentication.md @@ -4,3 +4,14 @@ For clients to authenticate, they have to authenticate with Login.gov via the ba This will allow the backend to identify the browser which requested access and authorize them based on the cookie they provide in their API calls. The secured portion of this authorization is due to the httpOnly cookie being inaccessible to the client's local browser. + +# Generating API token + +In order to use APIs, an activated `OFA Sys Admin` user has to generate a new token and use it in the API request following these steps: +1. User has to first login using frontend and going through the normal login process +2. After user is logged in, user can grab a token at `/v1/security/get-token` +3. The token then can be used in authorization header. As an example: + +```curl --location 'http://{host}/v1/users/' --header 'Authorization: Token {token}'``` + +Note: the authentication token is available for 24 hours by default but this can be overridden using the `TOKEN_EXPIRATION_HOURS` environment variable. diff --git a/tdrs-backend/tdpservice/data_files/test/test_api.py b/tdrs-backend/tdpservice/data_files/test/test_api.py index 57808de8f..bc643d908 100644 --- a/tdrs-backend/tdpservice/data_files/test/test_api.py +++ b/tdrs-backend/tdpservice/data_files/test/test_api.py @@ -96,11 +96,11 @@ def assert_error_report_tanf_file_content_matches_with_friendly_names(response): """Assert the error report file contents match expected with friendly names.""" ws = DataFileAPITestBase.get_spreadsheet(response) - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == "if cash amount :873 validator1 passed" \ + " then number of months T1: 0 is not larger than 0." @staticmethod @@ -108,11 +108,11 @@ def assert_error_report_ssp_file_content_matches_with_friendly_names(response): """Assert the error report file contents match expected with friendly names.""" ws = DataFileAPITestBase.get_spreadsheet(response) - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=4, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=7, column=COL_ERROR_MESSAGE).value == "TRAILER record length is 15 characters " + \ "but must be 23." @staticmethod @@ -128,11 +128,11 @@ def assert_error_report_file_content_matches_without_friendly_names(response): wb = openpyxl.load_workbook('mycls.xlsx') ws = wb.get_sheet_by_name('Sheet1') - COL_ERROR_MESSAGE = 5 + COL_ERROR_MESSAGE = 4 - assert ws.cell(row=1, column=1).value == "Error reporting in TDP is still in development.We'll" \ - + " be in touch when it's ready to use!For now please refer to the reports you receive via email" - assert ws.cell(row=5, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " + assert ws.cell(row=1, column=1).value == "Please refer to the most recent versions of the coding " \ + + "instructions (linked below) when looking up items and allowable values during the data revision process" + assert ws.cell(row=8, column=COL_ERROR_MESSAGE).value == ("if CASH_AMOUNT :873 validator1 passed then " "NBR_MONTHS T1: 0 is not larger than 0.") @staticmethod diff --git a/tdrs-backend/tdpservice/data_files/util.py b/tdrs-backend/tdpservice/data_files/util.py index 3c5470c13..17beb90aa 100644 --- a/tdrs-backend/tdpservice/data_files/util.py +++ b/tdrs-backend/tdpservice/data_files/util.py @@ -36,27 +36,51 @@ def format_error_msg(x): output = BytesIO() workbook = xlsxwriter.Workbook(output) worksheet = workbook.add_worksheet() + report_columns = [ ('case_number', lambda x: x['case_number']), ('year', lambda x: str(x['rpt_month_year'])[0:4] if x['rpt_month_year'] else None), ('month', lambda x: calendar.month_name[ int(str(x['rpt_month_year'])[4:]) ] if x['rpt_month_year'] else None), - ('error_type', lambda x: x['error_type']), ('error_message', lambda x: format_error_msg(chk(x))), ('item_number', lambda x: x['item_number']), ('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])), ('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])), ('row_number', lambda x: x['row_number']), - ('column_number', lambda x: x['column_number']) ] # write beta banner - worksheet.write(row, col, - "Error reporting in TDP is still in development." + - "We'll be in touch when it's ready to use!" + - "For now please refer to the reports you receive via email") + worksheet.write( + row, col, + "Please refer to the most recent versions of the coding " + + "instructions (linked below) when looking up items " + + "and allowable values during the data revision process" + ) + + row, col = 1, 0 + worksheet.write_url( + row, col, + 'https://www.acf.hhs.gov/ofa/policy-guidance/tribal-tanf-data-coding-instructions', + string='For Tribal TANF data reports: Tribal TANF Instructions', + ) + row, col = 2, 0 + worksheet.write_url( + row, col, + 'https://www.acf.hhs.gov/ofa/policy-guidance/acf-ofa-pi-23-04', + string='For TANF and SSP-MOE data reports: TANF / SSP-MOE (ACF-199 / ACF-209) Instructions' + ) + + row, col = 3, 0 + worksheet.write_url( + row, col, + 'https://tdp-project-updates.app.cloud.gov/knowledge-center/viewing-error-reports.html', + string='Visit the Knowledge Center for further guidance on reviewing error reports' + ) + + row, col = 5, 0 + # write csv header bold = workbook.add_format({'bold': True}) @@ -68,7 +92,7 @@ def format_header(header_list: list): [worksheet.write(row, col, format_header(key[0]), bold) for col, key in enumerate(report_columns)] [ - worksheet.write(row + 3, col, key[1](data_i)) for col, key in enumerate(report_columns) + worksheet.write(row + 6, col, key[1](data_i)) for col, key in enumerate(report_columns) for row, data_i in enumerate(data) ] diff --git a/tdrs-backend/tdpservice/data_files/views.py b/tdrs-backend/tdpservice/data_files/views.py index dfcc71416..3f67d7cb3 100644 --- a/tdrs-backend/tdpservice/data_files/views.py +++ b/tdrs-backend/tdpservice/data_files/views.py @@ -18,7 +18,7 @@ from tdpservice.data_files.util import get_xls_serialized_file from tdpservice.data_files.models import DataFile, get_s3_upload_path from tdpservice.users.permissions import DataFilePermissions, IsApprovedPermission -from tdpservice.scheduling import sftp_task, parser_task +from tdpservice.scheduling import parser_task from tdpservice.data_files.s3_client import S3Client from tdpservice.parsers.models import ParserError from tdpservice.parsers.serializers import ParsingErrorSerializer @@ -59,7 +59,6 @@ def create(self, request, *args, **kwargs): # only if file is passed the virus scan and created successfully will we perform side-effects: # * Send to parsing - # * Upload to ACF-TITAN # * Send email to user logger.debug(f"{self.__class__.__name__}: status: {response.status_code}") @@ -74,15 +73,6 @@ def create(self, request, *args, **kwargs): parser_task.parse.delay(data_file_id) logger.info("Submitted parse task to queue for datafile %s.", data_file_id) - sftp_task.upload.delay( - data_file_pk=data_file_id, - server_address=settings.ACFTITAN_SERVER_ADDRESS, - local_key=settings.ACFTITAN_LOCAL_KEY, - username=settings.ACFTITAN_USERNAME, - port=22 - ) - logger.info("Submitted upload task to redis for datafile %s.", data_file_id) - app_name = settings.APP_NAME + '/' key = app_name + get_s3_upload_path(data_file, '') version_id = self.get_s3_versioning_id(response.data.get('original_filename'), key) diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py index 5c2ac878c..338a97216 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m1.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M1DataSubmissionDocument @@ -133,15 +133,16 @@ required=True, validators=[validators.notEmpty()] ), - Field( + TransformField( + zero_pad(3), item="2", - name='COUNTY_FIPS_CODE', + name="COUNTY_FIPS_CODE", friendly_name="county fips code", - type='string', + type="string", startIndex=19, endIndex=22, required=True, - validators=[validators.isNumber(),] + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py index 0f72a48e1..c44bfc76a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m2.py @@ -370,14 +370,14 @@ item="37", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=55, endIndex=57, required=False, validators=[ validators.or_validators( - validators.isInLimits(0, 16), validators.isInLimits(98, 99) - ) + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) + ), ] ), Field( @@ -388,7 +388,7 @@ startIndex=57, endIndex=58, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="39", @@ -398,7 +398,7 @@ startIndex=58, endIndex=59, required=False, - validators=[validators.oneOf([0, 1, 2, 9])] + validators=[validators.oneOf([1, 2, 9])] ), Field( item="40", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py index 19069089b..12376bbfc 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m3.py @@ -279,15 +279,15 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=49, endIndex=51, required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) - ) + ), ] ), Field( @@ -298,7 +298,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", @@ -593,13 +593,13 @@ item="68", name='EDUCATION_LEVEL', friendly_name="education level", - type='number', + type='string', startIndex=90, endIndex=92, required=True, validators=[ validators.or_validators( - validators.isInStringRange(0, 16), + validators.isInStringRange(1, 16), validators.isInStringRange(98, 99) ) ] @@ -612,7 +612,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 3, 9])] + validators=[validators.oneOf([1, 2, 3, 9])] ), Field( item="70A", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py index 792ac2ba6..705e2592a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m4.py @@ -1,7 +1,7 @@ """Schema for SSP M1 record type.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M4DataSubmissionDocument @@ -54,7 +54,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +63,7 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(0, 999)], + validators=[validators.isNumber()], ), Field( item="4", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py index 7ba6aa2c6..080716cb0 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m5.py @@ -347,7 +347,8 @@ validators.or_validators( validators.isInStringRange(0, 16), validators.isInStringRange(98, 99), - ) + ), + validators.notMatches("00") ], ), Field( @@ -359,10 +360,7 @@ endIndex=57, required=False, validators=[ - validators.or_validators( - validators.isInLimits(0, 3), - validators.matches(9) - ) + validators.oneOf([1, 2, 3, 9]), ], ), Field( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py index e85bc8fb1..69d1bda7a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m6.py @@ -1,10 +1,10 @@ """Schema for HEADER row of all submission types.""" -from ...transforms import calendar_quarter_to_rpt_month_year -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M6DataSubmissionDocument s1 = RowSchema( diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py index 8d6664a43..39ecf8f84 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/ssp/m7.py @@ -1,9 +1,9 @@ """Schema for TANF T7 Row.""" -from ...fields import Field, TransformField -from ...row_schema import RowSchema, SchemaManager -from ...transforms import calendar_quarter_to_rpt_month_year -from ... import validators +from tdpservice.parsers.transforms import calendar_quarter_to_rpt_month_year +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.ssp import SSP_M7DataSubmissionDocument schemas = [] diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py index dc5d153cb..371b29df6 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t1.py @@ -1,6 +1,7 @@ """Schema for t1 record types.""" -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T1DataSubmissionDocument @@ -155,7 +156,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -163,9 +165,7 @@ startIndex=19, endIndex=22, required=True, - validators=[ - validators.isNumber(), - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py index 47b6e9144..683649a9b 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t2.py @@ -98,7 +98,6 @@ result_field_name="COOPERATION_CHILD_SUPPORT", result_function=validators.oneOf((1, 2, 9)), ), - validators.validate__FAM_AFF__HOH__Fed_Time(), validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 3), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py index 4bc9f6195..5cf53bc6a 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t3.py @@ -295,7 +295,7 @@ startIndex=51, endIndex=52, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", @@ -608,7 +608,7 @@ startIndex=92, endIndex=93, required=False, - validators=[validators.oneOf([0, 1, 2, 9])], + validators=[validators.oneOf([1, 2, 9])], ), Field( item="77A", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py index 2828e2a8f..2de7ea71c 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t4.py @@ -1,7 +1,7 @@ """Schema for HEADER row of all submission types.""" - -from tdpservice.parsers.fields import Field +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField from tdpservice.parsers.row_schema import RowSchema, SchemaManager from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tanf import TANF_T4DataSubmissionDocument @@ -55,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -63,7 +64,7 @@ startIndex=19, endIndex=22, required=True, - validators=[validators.isInStringRange(1, 999)], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py index fa0e1792c..df9bf9ce2 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tanf/t5.py @@ -92,7 +92,6 @@ result_field_name="CITIZENSHIP_STATUS", result_function=validators.isInLimits(1, 2), ), - validators.validate__FAM_AFF__HOH__Count_Fed_Time(), validators.if_then_validator( condition_field_name="DATE_OF_BIRTH", condition_function=validators.olderThan(18), diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py index 221739dd9..a5d4a45a7 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t1.py @@ -1,8 +1,9 @@ """Schema for Tribal TANF T1 record types.""" -from ...fields import Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T1DataSubmissionDocument t1 = SchemaManager( @@ -156,7 +157,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -164,9 +166,7 @@ startIndex=19, endIndex=22, required=False, - validators=[ - validators.isNumber(), - ], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py index 396418cba..e815ac849 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t2.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T2 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func, zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T2DataSubmissionDocument @@ -98,7 +98,7 @@ result_field_name="COOPERATION_CHILD_SUPPORT", result_function=validators.oneOf((1, 2, 9)), ), - validators.validate__FAM_AFF__HOH__Fed_Time(), + validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.isInLimits(1, 3), @@ -347,9 +347,9 @@ type="string", startIndex=51, endIndex=53, - required=False, + required=True, validators=[ - validators.isInStringRange(0, 10), + validators.isInStringRange(1, 10), ], ), Field( @@ -622,7 +622,8 @@ validators.isInStringRange(0, 99), ], ), - Field( + TransformField( + zero_pad(2), item="61", name="ADD_WORK_ACTIVITIES", friendly_name="additional work activities", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py index 7acbdadaa..4e03bbe61 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t3.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T3 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T3DataSubmissionDocument FIRST_CHILD = 1 diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py index e59cecade..9f1f53802 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t4.py @@ -1,8 +1,9 @@ """Schema for Tribal TANF T4 record types.""" -from ...fields import Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import zero_pad +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T4DataSubmissionDocument @@ -54,7 +55,8 @@ required=True, validators=[validators.notEmpty()], ), - Field( + TransformField( + zero_pad(3), item="2", name="COUNTY_FIPS_CODE", friendly_name="county fips code", @@ -62,7 +64,7 @@ startIndex=19, endIndex=22, required=False, - validators=[validators.matches("000")], + validators=[validators.isNumber()], ), Field( item="5", diff --git a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py index 553197447..22ea004a8 100644 --- a/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py +++ b/tdrs-backend/tdpservice/parsers/schema_defs/tribal_tanf/t5.py @@ -1,10 +1,10 @@ """Schema for Tribal TANF T5 row of all submission types.""" -from ...transforms import tanf_ssn_decryption_func -from ...fields import TransformField, Field -from ...row_schema import RowSchema, SchemaManager -from ... import validators +from tdpservice.parsers.transforms import tanf_ssn_decryption_func +from tdpservice.parsers.fields import Field, TransformField +from tdpservice.parsers.row_schema import RowSchema, SchemaManager +from tdpservice.parsers import validators from tdpservice.search_indexes.documents.tribal import Tribal_TANF_T5DataSubmissionDocument @@ -92,7 +92,7 @@ result_field_name="CITIZENSHIP_STATUS", result_function=validators.isInLimits(1, 2), ), - validators.validate__FAM_AFF__HOH__Count_Fed_Time(), + validators.if_then_validator( condition_field_name="FAMILY_AFFILIATION", condition_function=validators.matches(1), diff --git a/tdrs-backend/tdpservice/parsers/test/conftest.py b/tdrs-backend/tdpservice/parsers/test/conftest.py new file mode 100644 index 000000000..1754e66e3 --- /dev/null +++ b/tdrs-backend/tdpservice/parsers/test/conftest.py @@ -0,0 +1,64 @@ +"""Fixtures for parsing integration tests.""" +import pytest +from .factories import ParsingFileFactory + +@pytest.fixture +def t3_cat2_invalid_citizenship_file(): + """Fixture for T3 file with an invalid CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2021, + quarter='Q1', + file__name='t3_invalid_citizenship_file.txt', + file__section='Active Case Data', + file__data=(b'HEADER20204A06 TAN1ED\n' + b'T320201011111111112420190127WTTTT90W022212222204398000000000\n' + b'T320201011111111112420190127WTTTT90W0222122222043981000000004201001013333333330000000' + b'1100000099998888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m2_cat2_invalid_37_38_39_file(): + """Fixture for M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m2_cat2_invalid_37_38_39_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M2202310111111111275219811103WTTT#PW@W22212222222250122000010119350000000000000000000000000000000' + b'00000000000000000000000000000225300000000000000000000\n' + b'TRAILER0000001 ') + ) + return parsing_file + +@pytest.fixture +def m3_cat2_invalid_68_69_file(): + """Fixture for M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m3_cat2_invalid_68_69_file.txt', + section='SSP Active Case Data', + file__data=(b'HEADER20234A24 SSP1ED\n' + b'M320231011111111127420110615WTTTP99B#22212222204300000000000\n' + b'M320231011111111127120110615WTTTP99B#222122222043011000000004201001013333333330000000110000009999' + b'8888\n' + b'TRAILER0000002 ') + ) + return parsing_file + +@pytest.fixture +def m5_cat2_invalid_23_24_file(): + """Fixture for M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + parsing_file = ParsingFileFactory( + year=2024, + quarter='Q1', + file__name='m5_cat2_invalid_23_24_file.txt', + section='SSP Closed Case Data', + file__data=(b'HEADER20184C24 SSP1ED\n' + b'M520181011111111161519791106WTTTY0ZB922212222222210112000112970000\n' + b'TRAILER0000001 ') + ) + return parsing_file diff --git a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt index 7ef4b46b0..4078f53ee 100644 --- a/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt +++ b/tdrs-backend/tdpservice/parsers/test/data/ADS.E2J.FTP2.TS142.txt @@ -1,21 +1,21 @@ HEADER20194C00142TAN1ED -T420191011111111762255 0402451153123 +T420191011111111762 1 0402451153123 T520191011111111762120160102WTTTTT@YB2122222222221 822981 0 03 0 0 T520191011111111762120170526WTTTTTZPW2122221222221 822981 0 03 0 0 T520191011111111762319880112WTTTTTTY#2222212222222 122161 1591 0 0 T520191011111111762319610502WTTTTTT##2222212222222 222161 0601 0 0 -T420191011111112343255 0402451 91113 +T420191011111112343 1 0402451 91113 T520191011111112343119860308WTTTTTTTY2122222222221 122111 44162 0 0 -T420191011111112970255 0403561 91112 +T420191011111112970 1 0403561 91112 T520191011111112970119940807WTTTTT@#Z2122221222221 122121 10501 0 0 -T420191111111111339255 0403561 83113 +T420191111111111339 1 0403561 83113 T520191111111111339119880402WTTTTTZ#B2122221222223 221121 3571 0 0 T520191111111111339119970502WTTTTTTYB2122222222221 111111 8522 0 0 -T420191111111112073255 0403561151123 +T420191111111112073 1 0403561151123 T520191111111112073319900312WTTTTTT0@2122222222222 122121 0601 0 0 T520191111111112073319920507WTTTTT@B02122221222222 222121 0601 0 0 T520191111111112073120090514WTTTTT@@@2122222222221 822 11 0 03 0 0 -T420191111111112472255 0403561183113 +T420191111111112472 1 0403561183113 T520191111111112472120140814WTTTTTZZ02222212222221 422981 0 03 0 0 T520191111111112472119840305WTTTTT90W2122221222221 122101 40202 0 0 TRAILER 19 diff --git a/tdrs-backend/tdpservice/parsers/test/test_parse.py b/tdrs-backend/tdpservice/parsers/test/test_parse.py index 8a3d688d6..8bce2a7a0 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_parse.py +++ b/tdrs-backend/tdpservice/parsers/test/test_parse.py @@ -1083,7 +1083,7 @@ def test_parse_tanf_section1_blanks_file(tanf_section1_file_with_blanks, dfs): parser_errors = ParserError.objects.filter(file=tanf_section1_file_with_blanks) - assert parser_errors.count() == 23 + assert parser_errors.count() == 22 # Should only be cat3 validator errors for error in parser_errors: @@ -1966,3 +1966,85 @@ def test_parse_tribal_section_4_bad_quarter(tribal_section_4_bad_quarter, dfs): "representing the Calendar Year and Quarter formatted as YYYYQ" Tribal_TANF_T7.objects.count() == 0 + +@pytest.mark.django_db() +def test_parse_t3_cat2_invalid_citizenship(t3_cat2_invalid_citizenship_file, dfs): + """Test parsing a TANF T3 record with an invalid CITIZENSHIP_STATUS.""" + dfs.datafile = t3_cat2_invalid_citizenship_file + t3_cat2_invalid_citizenship_file.year = 2021 + t3_cat2_invalid_citizenship_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(t3_cat2_invalid_citizenship_file, dfs) + + parser_errors = ParserError.objects.filter(file=t3_cat2_invalid_citizenship_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 2 + + for e in parser_errors: + assert e.error_message == "T3: 0 is not in [1, 2, 9]." + + +@pytest.mark.django_db() +def test_parse_m2_cat2_invalid_37_38_39_file(m2_cat2_invalid_37_38_39_file, dfs): + """Test parsing an SSP M2 file with an invalid EDUCATION_LEVEL, CITIZENSHIP_STATUS, COOPERATION_CHILD_SUPPORT.""" + dfs.datafile = m2_cat2_invalid_37_38_39_file + m2_cat2_invalid_37_38_39_file.year = 2024 + m2_cat2_invalid_37_38_39_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m2_cat2_invalid_37_38_39_file, dfs) + + parser_errors = ParserError.objects.filter(file=m2_cat2_invalid_37_38_39_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 3 + + error_msgs = {"M2: 00 is not in range [1, 16]. or M2: 00 is not in range [98, 99].", + "M2: 0 is not in [1, 2, 3, 9].", + "M2: 0 is not in [1, 2, 9]."} + for e in parser_errors: + assert e.error_message in error_msgs + +@pytest.mark.django_db() +def test_parse_m3_cat2_invalid_68_69_file(m3_cat2_invalid_68_69_file, dfs): + """Test parsing an SSP M3 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m3_cat2_invalid_68_69_file + m3_cat2_invalid_68_69_file.year = 2024 + m3_cat2_invalid_68_69_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m3_cat2_invalid_68_69_file, dfs) + + parser_errors = ParserError.objects.filter(file=m3_cat2_invalid_68_69_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 4 + + error_msgs = {"M3: 00 is not in range [1, 16]. or M3: 00 is not in range [98, 99].", + "M3: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs + +@pytest.mark.django_db() +def test_parse_m5_cat2_invalid_23_24_file(m5_cat2_invalid_23_24_file, dfs): + """Test parsing an SSP M5 file with an invalid EDUCATION_LEVEL and CITIZENSHIP_STATUS.""" + dfs.datafile = m5_cat2_invalid_23_24_file + m5_cat2_invalid_23_24_file.year = 2019 + m5_cat2_invalid_23_24_file.quarter = 'Q1' + dfs.save() + + parse.parse_datafile(m5_cat2_invalid_23_24_file, dfs) + + parser_errors = ParserError.objects.filter(file=m5_cat2_invalid_23_24_file).exclude( + error_type=ParserErrorCategoryChoices.CASE_CONSISTENCY).order_by("pk") + + assert parser_errors.count() == 2 + + error_msgs = {"M5: 00 matches 00.", + "M5: 0 is not in [1, 2, 3, 9]."} + + for e in parser_errors: + assert e.error_message in error_msgs diff --git a/tdrs-backend/tdpservice/parsers/test/test_transforms.py b/tdrs-backend/tdpservice/parsers/test/test_transforms.py index ff1188785..1aabe69db 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_transforms.py +++ b/tdrs-backend/tdpservice/parsers/test/test_transforms.py @@ -1,11 +1,31 @@ -"""Tests for the transforms module.""" +"""Test for Transforms.""" +import pytest +import tdpservice.parsers.transforms as transforms from tdpservice.parsers.transforms import ( tanf_ssn_decryption_func, ssp_ssn_decryption_func, ) +@pytest.mark.parametrize("value,digits,expected", [ + ("1", 3, "001"), + ("10", 3, "010"), + ("100", 3, "100"), + ("1000", 3, "1000"), + ("1 ", 3, "01 "), + ("1 ", 3, "1 "), + ("1", 0, "1"), + ("1", -1, "1") +]) +def test_zero_pad(value, digits, expected): + """Test zero_pad returns valid value.""" + transform = transforms.zero_pad(digits) + result = transform(value) + + assert result == expected + + def test_tanf_ssn_decryption_func(): """Test the TANF SSN decryption function.""" assert tanf_ssn_decryption_func(None) is None diff --git a/tdrs-backend/tdpservice/parsers/test/test_validators.py b/tdrs-backend/tdpservice/parsers/test/test_validators.py index 8cc594e14..e9d5ba2bc 100644 --- a/tdrs-backend/tdpservice/parsers/test/test_validators.py +++ b/tdrs-backend/tdpservice/parsers/test/test_validators.py @@ -869,19 +869,6 @@ def test_validate_cooperation_with_child_support(self, record): result = val(record, RowSchema()) assert result[0] is False - def test_validate_months_federal_time_limit(self, record): - """Test cat3 validator for federal time limit.""" - val = validators.validate__FAM_AFF__HOH__Fed_Time() - record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH', 'MONTHS_FED_TIME_LIMIT']) - - record.FAMILY_AFFILIATION = 1 - record.MONTHS_FED_TIME_LIMIT = "000" - record.RELATIONSHIP_HOH = "01" - result = val(record, RowSchema()) - assert result[0] is False - def test_validate_employment_status(self, record): """Test cat3 validator for employment status.""" val = validators.if_then_validator( @@ -1180,21 +1167,6 @@ def test_validate_citizenship_status(self, record): result = val(record, RowSchema()) assert result[0] is False - def test_validate_hoh_fed_time(self, record): - """Test cat3 validator for federal disability.""" - val = validators.validate__FAM_AFF__HOH__Count_Fed_Time() - - record.FAMILY_AFFILIATION = 0 - result = val(record, RowSchema()) - assert result == (True, None, ['FAMILY_AFFILIATION', 'RELATIONSHIP_HOH', 'COUNTABLE_MONTH_FED_TIME']) - - record.FAMILY_AFFILIATION = 1 - record.RELATIONSHIP_HOH = 1 - record.COUNTABLE_MONTH_FED_TIME = 0 - - result = val(record, RowSchema()) - assert result[0] is False - def test_validate_oasdi_insurance(self, record): """Test cat3 validator for OASDI insurance.""" val = validators.if_then_validator( diff --git a/tdrs-backend/tdpservice/parsers/transforms.py b/tdrs-backend/tdpservice/parsers/transforms.py index 78d15dcbb..cd51e2012 100644 --- a/tdrs-backend/tdpservice/parsers/transforms.py +++ b/tdrs-backend/tdpservice/parsers/transforms.py @@ -36,3 +36,9 @@ def ssp_ssn_decryption_func(value, **kwargs): decryption_table = str.maketrans(decryption_dict) return value.translate(decryption_table) return value + +def zero_pad(digits): + """Zero pad a string.""" + def transform(value, **kwargs): + return value.lstrip().zfill(digits) + return transform diff --git a/tdrs-backend/tdpservice/parsers/validators.py b/tdrs-backend/tdpservice/parsers/validators.py index 3eeeeb02b..1d5abd7cb 100644 --- a/tdrs-backend/tdpservice/parsers/validators.py +++ b/tdrs-backend/tdpservice/parsers/validators.py @@ -297,6 +297,17 @@ def recordHasLengthBetween(lower, upper, error_func=None): f"{row_schema.record_type} record length of {len(value)} characters is not in the range [{lower}, {upper}].", ) +def fieldHasLength(length): + """Validate that the field value (string or array) has a length matching length param.""" + return make_validator( + lambda value: len(value) == length, + lambda value, + row_schema, + friendly_name, + item_num: f"{row_schema.record_type} field length is {len(value)} characters but must be {length}.", + ) + + def hasLengthGreaterThan(val, error_func=None): """Validate that value (string or array) has a length greater than val.""" return make_validator( @@ -473,7 +484,7 @@ def isSmallerThanOrEqualTo(UpperBound): def isInLimits(LowerBound, UpperBound): """Validate that value is in a range including the limits.""" return make_validator( - lambda value: value >= LowerBound and value <= UpperBound, + lambda value: int(value) >= LowerBound and int(value) <= UpperBound, lambda value, row_schema, friendly_name, item_num: f"{row_schema.record_type}: {value} is not larger or equal to {LowerBound} and " f"smaller or equal to {UpperBound}." @@ -595,104 +606,6 @@ def validate(instance, row_schema): return validate - -def validate__FAM_AFF__HOH__Fed_Time(): - """If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then MONTHS_FED_TIME_LIMIT >= 1.""" - # value is instance - def validate(instance, row_schema): - false_case = (False, - f"{row_schema.record_type}: If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then " - + "MONTHS_FED_TIME_LIMIT >= 1.", - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "MONTHS_FED_TIME_LIMIT",], - ) - true_case = (True, - None, - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "MONTHS_FED_TIME_LIMIT",], - ) - try: - FAMILY_AFFILIATION = ( - instance["FAMILY_AFFILIATION"] - if type(instance) is dict - else getattr(instance, "FAMILY_AFFILIATION") - ) - RELATIONSHIP_HOH = ( - instance["RELATIONSHIP_HOH"] - if type(instance) is dict - else getattr(instance, "RELATIONSHIP_HOH") - ) - RELATIONSHIP_HOH = int(RELATIONSHIP_HOH) - MONTHS_FED_TIME_LIMIT = ( - instance["MONTHS_FED_TIME_LIMIT"] - if type(instance) is dict - else getattr(instance, "MONTHS_FED_TIME_LIMIT") - ) - if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if MONTHS_FED_TIME_LIMIT is None or int(MONTHS_FED_TIME_LIMIT) < 1: - return false_case - else: - return true_case - else: - return true_case - except Exception: - vals = {"FAMILY_AFFILIATION": FAMILY_AFFILIATION, - "RELATIONSHIP_HOH": RELATIONSHIP_HOH, - "MONTHS_FED_TIME_LIMIT": MONTHS_FED_TIME_LIMIT} - logger.debug("Caught exception in validator: validate__FAM_AFF__HOH__Fed_Time. With field values: " + - f"{vals}.") - return false_case - - return validate - - -def validate__FAM_AFF__HOH__Count_Fed_Time(): - """If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then COUNTABLE_MONTH_FED_TIME >= 1.""" - # value is instance - def validate(instance, row_schema): - false_case = (False, - f"{row_schema.record_type}: If FAMILY_AFFILIATION == 1 and RELATIONSHIP_HOH == 1 or 2, then " - + "COUNTABLE_MONTH_FED_TIME >= 1.", - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "COUNTABLE_MONTH_FED_TIME",], - ) - true_case = (True, - None, - ["FAMILY_AFFILIATION", "RELATIONSHIP_HOH", "COUNTABLE_MONTH_FED_TIME",], - ) - try: - FAMILY_AFFILIATION = ( - instance["FAMILY_AFFILIATION"] - if type(instance) is dict - else getattr(instance, "FAMILY_AFFILIATION") - ) - RELATIONSHIP_HOH = ( - instance["RELATIONSHIP_HOH"] - if type(instance) is dict - else getattr(instance, "RELATIONSHIP_HOH") - ) - RELATIONSHIP_HOH = int(RELATIONSHIP_HOH) - COUNTABLE_MONTH_FED_TIME = ( - instance["COUNTABLE_MONTH_FED_TIME"] - if type(instance) is dict - else getattr(instance, "COUNTABLE_MONTH_FED_TIME") - ) - if FAMILY_AFFILIATION == 1 and (RELATIONSHIP_HOH == 1 or RELATIONSHIP_HOH == 2): - if int(COUNTABLE_MONTH_FED_TIME) < 1: - return false_case - else: - return true_case - else: - return true_case - except Exception: - vals = {"FAMILY_AFFILIATION": FAMILY_AFFILIATION, - "RELATIONSHIP_HOH": RELATIONSHIP_HOH, - "COUNTABLE_MONTH_FED_TIME": COUNTABLE_MONTH_FED_TIME - } - logger.debug("Caught exception in validator: validate__FAM_AFF__HOH__Count_Fed_Time. With field values: " + - f"{vals}.") - return false_case - - return validate - - def validate_header_section_matches_submission(datafile, section, generate_error): """Validate header section matches submission section.""" is_valid = datafile.section == section diff --git a/tdrs-backend/tdpservice/scheduling/sftp_task.py b/tdrs-backend/tdpservice/scheduling/sftp_task.py deleted file mode 100644 index d4807ac88..000000000 --- a/tdrs-backend/tdpservice/scheduling/sftp_task.py +++ /dev/null @@ -1,135 +0,0 @@ -"""schedule tasks.""" - -from __future__ import absolute_import - -# The tasks - -import hashlib -import os - -from celery import shared_task -from django.conf import settings -import datetime -import paramiko -import logging -from tdpservice.data_files.models import DataFile, LegacyFileTransfer - -logger = logging.getLogger(__name__) - - -@shared_task(acks_late=True, worker_prefetch_multiplier=1) -def upload( - data_file_pk, - server_address=settings.ACFTITAN_SERVER_ADDRESS, - local_key=settings.ACFTITAN_LOCAL_KEY, - username=settings.ACFTITAN_USERNAME, - port=22, -): - """ - Upload to SFTP server. - - This task uploads the file in DataFile object with pk = data_file_pk - to sftp server as defined in Settings file - """ - # Upload file - data_file = DataFile.objects.get(id=data_file_pk) - file_transfer_record = LegacyFileTransfer( - data_file=data_file, - uploaded_by=data_file.user, - file_name=data_file.filename if data_file.filename is not None else "None", - ) - - def write_key_to_file(private_key): - """Paramiko require the key in file object format.""" - with open("temp_key_file", "w") as f: - f.write(private_key) - f.close() - return "temp_key_file" - - def create_dir(directory_name, sftp_server): - """Code snippet to create directory in SFTP server.""" - try: - sftp_server.chdir(directory_name) # Test if remote_path exists - except IOError: - sftp_server.mkdir(directory_name) # Create remote_path - sftp_server.chdir(directory_name) - - for attempt in range(1, 4): - logger.info("Attempt {} to upload file {}".format(attempt, data_file.filename)) - try: - # Create directory names for ACF titan - destination = str(data_file.filename) - today_date = datetime.datetime.today() - upper_directory_name = today_date.strftime("%Y%m%d") - lower_directory_name = today_date.strftime( - str(data_file.year) + "-" + str(data_file.quarter) - ) - - # Paramiko need local file - paramiko_local_file = data_file.file.read() - with open(destination, "wb") as f1: - f1.write(paramiko_local_file) - file_transfer_record.file_size = f1.tell() - file_transfer_record.file_shasum = hashlib.sha256( - paramiko_local_file - ).hexdigest() - f1.close() - - # Paramiko SSH connection requires private key as file - temp_key_file = write_key_to_file(local_key) - os.chmod(temp_key_file, 0o600) - - # Create SFTP/SSH connection - transport = paramiko.SSHClient() - transport.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - pkey = paramiko.RSAKey.from_private_key_file(temp_key_file) - transport.connect( - server_address, - pkey=pkey, - username=username, - port=port, - look_for_keys=False, - banner_timeout=30, - disabled_algorithms={"pubkeys": ["rsa-sha2-512", "rsa-sha2-256"]}, - ) - # remove temp key file - os.remove(temp_key_file) - sftp = transport.open_sftp() - - # Create remote directory - create_dir(settings.ACFTITAN_DIRECTORY, sftp_server=sftp) - create_dir(upper_directory_name, sftp_server=sftp) - create_dir(lower_directory_name, sftp_server=sftp) - - # Put the file in SFTP server - sftp.put(destination, destination) - - # Delete temp file - os.remove(destination) - logger.info( - "File {} has been successfully uploaded to {}".format( - destination, server_address - ) - ) - - # Add the log LegacyFileTransfer - file_transfer_record.result = LegacyFileTransfer.Result.COMPLETED - file_transfer_record.save() - transport.close() - return True - - except Exception as e: - logger.error( - "Attempt {} failed to upload {} with error:{}".format( - attempt, destination, e - ) - ) - transport.close() - else: - # All attempts failed - logger.error("Failed to upload {} after 3 attempts".format(destination)) - file_transfer_record.file_size = 0 - file_transfer_record.result = LegacyFileTransfer.Result.ERROR - file_transfer_record.save() - transport.close() - return False diff --git a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py b/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py deleted file mode 100644 index 140ec6d64..000000000 --- a/tdrs-backend/tdpservice/scheduling/test/test_file_upload.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Scheduling tests.""" - -from datetime import datetime - -import pytest -from paramiko import Transport -from paramiko.sftp_client import SFTPClient -from tdpservice.scheduling.sftp_task import upload -from tdpservice.data_files.test.factories import DataFileFactory -from tdpservice.stts.models import STT -from django.conf import settings - -""" -To mock sftp server, pytest_sftpserver (https://github.com/ulope/pytest-sftpserver) is used. -The package provides two main fixtures for testing: sftpserver and sftpclient. -""" - -@pytest.fixture -def stt_instance(region): - """Return an STT.""" - stt, _ = STT.objects.get_or_create( - name="first", - region=region, - postal_code="AR", - stt_code='234', - filenames={ - 'Aggregate Data': 'ADS.E2J.NDM3.TS22', - 'Active Case Data': 'test', - 'Closed Case Data': 'ADS.E2J.NDM2.TS22'} - ) - return stt - -@pytest.fixture -def data_file_instance(stt_instance): - """Prepare data file fixture instance for testing datafile.""" - return DataFileFactory.create( - created_at=datetime.now(), - stt=stt_instance - ) - - -@pytest.fixture -def sftp_connection_values(sftpserver): - """SFTP connection values for local sftp server.""" - server_address = sftpserver.host - local_key = settings.ACFTITAN_SFTP_PYTEST - username = "user" - port = sftpserver.port - return { - 'server_address': server_address, - 'username': username, - 'local_key': local_key, - 'port': port - } - - -@pytest.fixture(scope="session") -def sftpclient(sftpserver): - """SFTP client for local sftp server.""" - transport = Transport((sftpserver.host, sftpserver.port)) - transport.connect(username="a", password="b") - sftpclient = SFTPClient.from_transport(transport) - yield sftpclient - sftpclient.close() - transport.close() - - -@pytest.mark.django_db -def test_new_data_file(sftpserver, data_file_instance, sftp_connection_values, sftpclient): - """Datafile object for testing the file.""" - data_file_instance.save() - - """ - Need .serve_content to keep the communication alive - Here we put a dummy file somefile.txt in a_dir to keep the port open - """ - with sftpserver.serve_content({'a_dir': {'somefile.txt': "File content"}}): - upload(data_file_instance.pk, - server_address=sftp_connection_values['server_address'], - local_key=sftp_connection_values['local_key'], - username=sftp_connection_values['username'], - port=sftp_connection_values['port']) - - # Create directory structure as needed for ACF_TITAN to assert correct directory name - today_date = datetime.today() - upper_directory_name = today_date.strftime('%Y%m%d') - lower_directory_name = today_date.strftime(str(data_file_instance.year) + - '-' + - str(data_file_instance.quarter)) - assert sftpclient.listdir(upper_directory_name+'/'+lower_directory_name)[0] == \ - data_file_instance.filename diff --git a/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py new file mode 100644 index 000000000..4a7cf36b4 --- /dev/null +++ b/tdrs-backend/tdpservice/search_indexes/migrations/0028_education_level_to_string.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.15 on 2024-04-29 18:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search_indexes', '0027_tribal_ssp_tanf_dob_to_string'), + ] + + operations = [ + migrations.AlterField( + model_name='ssp_m2', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + migrations.AlterField( + model_name='ssp_m3', + name='EDUCATION_LEVEL', + field=models.CharField(max_length=2, null=True), + ), + ] diff --git a/tdrs-backend/tdpservice/search_indexes/models/ssp.py b/tdrs-backend/tdpservice/search_indexes/models/ssp.py index b11e6fff5..bb5840323 100644 --- a/tdrs-backend/tdpservice/search_indexes/models/ssp.py +++ b/tdrs-backend/tdpservice/search_indexes/models/ssp.py @@ -113,7 +113,7 @@ class SSP_M2(models.Model): RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) NEEDS_PREGNANT_WOMAN = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) COOPERATION_CHILD_SUPPORT = models.IntegerField(null=True, blank=False) EMPLOYMENT_STATUS = models.IntegerField(null=True, blank=False) @@ -194,7 +194,7 @@ class SSP_M3(models.Model): RECEIVE_SSI = models.IntegerField(null=True, blank=False) RELATIONSHIP_HOH = models.IntegerField(null=True, blank=False) PARENT_MINOR_CHILD = models.IntegerField(null=True, blank=False) - EDUCATION_LEVEL = models.IntegerField(null=True, blank=False) + EDUCATION_LEVEL = models.CharField(max_length=2, null=True, blank=False) CITIZENSHIP_STATUS = models.IntegerField(null=True, blank=False) UNEARNED_SSI = models.IntegerField(null=True, blank=False) OTHER_UNEARNED_INCOME = models.IntegerField(null=True, blank=False) diff --git a/tdrs-backend/tdpservice/security/test/test_views.py b/tdrs-backend/tdpservice/security/test/test_views.py new file mode 100644 index 000000000..b602ca577 --- /dev/null +++ b/tdrs-backend/tdpservice/security/test/test_views.py @@ -0,0 +1,66 @@ +"""Tests for the views in the security app.""" + +import pytest +import logging +from rest_framework.authtoken.models import Token +from tdpservice.users.models import User, AccountApprovalStatusChoices +from tdpservice.security.views import token_is_valid +from django.test import Client +from django.urls import reverse +from django.contrib.auth.models import Group + +client = Client() + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def token(): + """Return a DRF token.""" + user = User.objects.create(username="testuser") + token = Token.objects.create(user=user) + return token + + +@pytest.mark.django_db +def test_token_is_valid(token): + """Test token_is_valid function.""" + logger.info(token.__dict__) + assert token_is_valid(token) is True + token.created = token.created.replace(year=2000) + # token.save() + assert token_is_valid(token) is False + + +@pytest.mark.django_db +def test_generate_new_token(client): + """Test generate_new_token function.""" + url = reverse("get-new-token") + # assert if user is not authenticated + response = client.get(url) + assert response.status_code == 302 + + # assert if user is not ofa_sys_admin + user = User.objects.create_user(username="testuser", password="testpassword") + user.save() + client.login(username="testuser", password="testpassword") + response = client.get(url) + assert response.status_code == 302 + + # assert if user is not approved + user.account_approval_status = AccountApprovalStatusChoices.PENDING + user.groups.add(Group.objects.get(name="OFA System Admin")) + user.save() + client.login(username="testuser", password="testpassword") + response = client.get(url) + assert response.status_code == 302 + + # assert if token is valid + user.account_approval_status = AccountApprovalStatusChoices.APPROVED + user.save() + + client.login(username="testuser", password="testpassword") + url = reverse("get-new-token") + response = client.get(url) + assert response.status_code == 200 + assert response.data == str(Token.objects.get(user=user)) diff --git a/tdrs-backend/tdpservice/security/urls.py b/tdrs-backend/tdpservice/security/urls.py new file mode 100644 index 000000000..ef62d5e44 --- /dev/null +++ b/tdrs-backend/tdpservice/security/urls.py @@ -0,0 +1,12 @@ +"""URL patterns for the security app.""" + +from . import views +from django.urls import path + +urlpatterns = [ + path( + "get-token", + views.generate_new_token, + name="get-new-token", + ), +] diff --git a/tdrs-backend/tdpservice/security/utils.py b/tdrs-backend/tdpservice/security/utils.py new file mode 100644 index 000000000..873dc3aa4 --- /dev/null +++ b/tdrs-backend/tdpservice/security/utils.py @@ -0,0 +1,43 @@ +"""Utility classes and functions for security.""" + +from rest_framework import exceptions +from rest_framework.authentication import TokenAuthentication +from django.utils.translation import gettext_lazy as _ +from datetime import datetime +import pytz +from datetime import timedelta +from django.conf import settings +import logging + +logger = logging.getLogger(__name__) + +def token_is_valid(token): + """Check if token is valid.""" + utc_now = datetime.now() + utc_now = utc_now.replace(tzinfo=pytz.utc) + if token.created < (utc_now - timedelta(hours=settings.TOKEN_EXPIRATION_HOURS)): + logger.info("API auth Token expired") + return False + return token is not None + +# have to use ExpTokenAuthentication in settings.py instead of TokenAuthentication +class ExpTokenAuthentication(TokenAuthentication): + """Custom token authentication class that checks if token is expired.""" + + # see https://github.com/encode/django-rest-framework/blob/master/rest_framework/authentication.py + + def authenticate_credentials(self, key): + """Authenticate the credentials.""" + model = self.get_model() + try: + token = model.objects.select_related("user").get(key=key) + except model.DoesNotExist: + raise exceptions.AuthenticationFailed(_("Invalid token.")) + + if not token.user.is_active: + raise exceptions.AuthenticationFailed(_("User inactive or deleted.")) + + if not token_is_valid(token): + raise exceptions.AuthenticationFailed(_("Token expired.")) + + return (token.user, token) diff --git a/tdrs-backend/tdpservice/security/views.py b/tdrs-backend/tdpservice/security/views.py new file mode 100644 index 000000000..64377332c --- /dev/null +++ b/tdrs-backend/tdpservice/security/views.py @@ -0,0 +1,36 @@ +"""Views for the security app.""" + +from rest_framework.decorators import api_view +from rest_framework.response import Response +from django.contrib.auth.decorators import user_passes_test +from tdpservice.users.models import User, AccountApprovalStatusChoices +from rest_framework.authtoken.models import Token +from tdpservice.security.utils import token_is_valid + +import logging + +logger = logging.getLogger(__name__) + + +def can_get_new_token(user): + """Check if user can get a new token.""" + return ( + user.is_authenticated + and user.is_ofa_sys_admin + and user.account_approval_status == AccountApprovalStatusChoices.APPROVED + ) + + +@user_passes_test(can_get_new_token, login_url="/login/") +@api_view(["GET"]) +def generate_new_token(request): + """Generate new token for the API user.""" + if request.method == "GET": + user = User.objects.get(username=request.user) + token, created = Token.objects.get_or_create(user=user) + if token_is_valid(token): + return Response(str(token)) + else: + token.delete() + token = Token.objects.create(user=user) + return Response(str(token)) diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index 50bc6cb90..d4f6e0c13 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -297,7 +297,7 @@ class Common(Configuration): "DEFAULT_AUTHENTICATION_CLASSES": ( "tdpservice.users.authentication.CustomAuthentication", "rest_framework.authentication.SessionAuthentication", - "rest_framework.authentication.TokenAuthentication", + "tdpservice.security.utils.ExpTokenAuthentication", ), "DEFAULT_FILTER_BACKENDS": [ "django_filters.rest_framework.DjangoFilterBackend", @@ -311,6 +311,8 @@ class Common(Configuration): "django.contrib.auth.backends.ModelBackend", ) + TOKEN_EXPIRATION_HOURS = int(os.getenv("TOKEN_EXPIRATION_HOURS", 24)) + # CORS CORS_ALLOW_CREDENTIALS = True @@ -441,17 +443,6 @@ class Common(Configuration): '' ) - # ------- SFTP CONFIG - ACFTITAN_SERVER_ADDRESS = os.getenv('ACFTITAN_HOST', '') - """ - To be able to fit the PRIVATE KEY in one line as environment variable, we replace the EOL - with an underscore char. - The next line replaces the _ with EOL before using the PRIVATE KEY - """ - ACFTITAN_LOCAL_KEY = os.getenv('ACFTITAN_KEY', '').replace('_', '\n') - ACFTITAN_USERNAME = os.getenv('ACFTITAN_USERNAME', '') - ACFTITAN_DIRECTORY = os.getenv('ACFTITAN_DIRECTORY', '') - # -------- CELERY CONFIG REDIS_URI = os.getenv( 'REDIS_URI', diff --git a/tdrs-backend/tdpservice/settings/local.py b/tdrs-backend/tdpservice/settings/local.py index ae22cbcab..171608fe5 100644 --- a/tdrs-backend/tdpservice/settings/local.py +++ b/tdrs-backend/tdpservice/settings/local.py @@ -43,12 +43,3 @@ class Local(Common): } REDIS_SERVER_LOCAL = bool(strtobool(os.getenv("REDIS_SERVER_LOCAL", "TRUE"))) - - # SFTP TEST KEY - """ - To be able to fit the PRIVATE KEY in one line as environment variable, we replace the EOL - with an underscore char. - The next line replaces the _ with EOL before using the PRIVATE KEY - """ - ACFTITAN_SFTP_PYTEST = os.getenv("ACFTITAN_SFTP_PYTEST").replace('_', '\n') - APP_NAME = "tdrs-backend-local" diff --git a/tdrs-backend/tdpservice/urls.py b/tdrs-backend/tdpservice/urls.py index f19e26b98..d40e64651 100755 --- a/tdrs-backend/tdpservice/urls.py +++ b/tdrs-backend/tdpservice/urls.py @@ -38,6 +38,7 @@ path("stts/", include("tdpservice.stts.urls")), path("data_files/", include("tdpservice.data_files.urls")), path("logs/", write_logs), + path("security/", include("tdpservice.security.urls")), ] if settings.DEBUG: diff --git a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx index 9800206c6..3ddfe7365 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/CaseAggregatesTable.jsx @@ -121,7 +121,7 @@ export const CaseAggregatesTable = ({ files }) => ( Status - Error Reports (In development) + Error Reports diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx index d61ce85ec..654339ed6 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.jsx @@ -72,17 +72,31 @@ const SubmissionHistory = ({ filterValues }) => { }, [hasFetchedFiles, files, dispatch, filterValues]) return ( -
    - {fileUploadSections.map((section, index) => ( - f.section.includes(section))} - /> - ))} -
    + <> +
    + + Visit the Knowledge Center for further guidance on reviewing error + reports + +
    +
    + {fileUploadSections.map((section, index) => ( + f.section.includes(section))} + /> + ))} +
    + ) } diff --git a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js index 06baeb730..325c7d898 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js +++ b/tdrs-frontend/src/components/SubmissionHistory/SubmissionHistory.test.js @@ -241,9 +241,7 @@ describe('SubmissionHistory', () => { expect(screen.queryByText('test5.txt')).not.toBeInTheDocument() expect(screen.queryByText('test6.txt')).toBeInTheDocument() - expect( - screen.queryByText('Error Reports (In development)') - ).toBeInTheDocument() + expect(screen.queryByText('Error Reports')).toBeInTheDocument() }) it('Shows SSP results when SSP-MOE file type selected', () => { diff --git a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx index 8a8e6fd5c..3f4ba24a4 100644 --- a/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx +++ b/tdrs-frontend/src/components/SubmissionHistory/TotalAggregatesTable.jsx @@ -109,7 +109,7 @@ export const TotalAggregatesTable = ({ files }) => ( Status - Error Reports (In development) + Error Reports