diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index 54bd7d108..43adb60e3 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -37,16 +37,40 @@ backend-appname: <> frontend-appname: <> cf-space: <> - - deploy-clamav: - backend-appname: <> - cf-org: <> - cf-space: <> - deploy-frontend: environment: <> backend-appname: <> frontend-appname: <> cf-space: <> + clamav-cloud-dot-gov: + parameters: + backend-appname: + default: tdp-backend + type: string + cf-password: + default: CF_PASSWORD_DEV + type: env_var_name + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + cf-username: + default: CF_USERNAME_DEV + type: env_var_name + steps: + - checkout + - sudo-check + - cf-check + - login-cloud-dot-gov: + cf-password: <> + cf-org: <> + cf-space: <> + cf-username: <> + - deploy-clamav + deploy-backend: parameters: backend-appname: @@ -71,30 +95,12 @@ <> deploy-clamav: - parameters: - backend-appname: - default: tdp-backend - type: string - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string steps: - run: name: Deploy ClamAV REST application command: | cf push clamav-rest -f tdrs-backend/manifest.clamav.yml \ - --var cf-space=<> \ - - run: - name: Enable internal route between backend and clamav-rest app - command: | - cf add-network-policy <> clamav-rest \ - -s <> \ - -o ${<>} \ - --protocol tcp \ - --port 9000 + --var cf-space=tanf-prod deploy-frontend: parameters: diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index 542f33136..9aa40dfa7 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -151,3 +151,12 @@ cf-password: CF_PASSWORD_PROD cf-space: tanf-prod cf-username: CF_USERNAME_PROD + prod-deploy-clamav: + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - clamav-cloud-dot-gov: + backend-appname: tdp-backend-prod + cf-password: CF_PASSWORD_PROD + cf-space: tanf-prod + cf-username: CF_USERNAME_PROD diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 6ecaae41f..5689b83bc 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -118,6 +118,13 @@ branches: only: - master + - prod-deploy-clamav: + requires: + - deploy-infrastructure-production + filters: + branches: + only: + - master - make_erd: # from ../util folder filters: branches: diff --git a/docs/Security-Compliance/diagram.drawio b/docs/Security-Compliance/diagram.drawio index adfd23414..8ead466d8 100644 --- a/docs/Security-Compliance/diagram.drawio +++ b/docs/Security-Compliance/diagram.drawio @@ -1 +1,241 @@ -7V1pc6LK1/80qXqeF7FYXV4irhnQRE2MvpliC6IgXkARP/3/NIuytEsMmZl7azIzFehuuk+f5XdOdx+YB5q39l1H2ixEW9XMB4pQ9w9064GiGIZtwC9UEkQlJEPSUYnuGGpcdioYGwctLiTi0q2ham6moWfbpmdssoWKvV5ripcpkxzH9rPNPmwzO+pG0rVCwViRzGLp1FC9RVxaI4hTRU8z9EUyNJnUWFLSOi5wF5Jq+6kiuv1A845te9GVtec1E7EvYUz0XOdM7ZEyR1t7tzzgi2+dasdTve2HatKzD/+lWn2kq1E3O8ncxlPmpmMo6No73rS3aky7FyQc2djG2gu5yjbhL4zJEw8s1PDorkKxuYL8fS1bQBbvUB/Zgvx9LVtA5rsnc+OTeQJTBYW7TPdEbnwiRSD8pZv21jONtcYf9Y+AQt2RVAPEwtum7UDZ2l4D95oLzzLhjoRLf2F42ngjKYirPhgPlH3Yay+2AJJK7mPGo15BgTbo2trryNoqku8yFd2xt5twyD7YALb2J1z+VJAwf0qmhzryHHulJcQ9UDT86SClaX4Yppkjeqc5ngEGwZmGjvr3bDScFN85keo3XZiKsdYF7QPxgCZi8nFjqJK70NR4TkUljvUaDavtU0WxUnc129I8J4AmcS3FxAYWYwxdj+/9k8GySZtFylbrdFwoxSChH/s+mRFcxJaEtyrn7f1nnaM+JvJ4Lx+Gz8ps7D+SbMGqhh0OwRSSQkW3d3A9dHRpbRwkzwC5/bWyf72VOZqOJIkxL5KptZvc95rXcYzSzauWNS+GLpoXU8eYV60M83o5/HjnZ/Xu7Iepu+0BM1jU2Ue2aF4FAwKRrNWQD0gRJEdJZF5P84jEievjg4AfvLjO6Bzq8FnyPM1BUqsDKcyx5yRuYIpCI4h4JMveSXJIOqLI0VzAhdS97Ule6h6iLC19r6lG+jaOhVIlJagBzWbVoMoU1YDGoSxLlKAGBvcofCy4mv9h7D5+/hT/USVAWQajBtXQwyE+h1FhwurqP1sUZzVP4k0VVXX0u+/apuSBWlDEs2OrWyUC5gxmR1gSDyI7yaNJCUwjGjkp/ovq/3ZU320UbMTE1Oskcw+kmyGG34DoHNesNTNo9QuQIgtZbEnYUauFupWGj1oRPiicF6HY74IPpl6AD962LAMBBw8r2YL1aiosDeNb2/EWtm6vJbN9Km1mnc6pjWAjFQgLl5rnBbGmSlvPzqp2Qc+qfL3d7FwSg2tvHUW74C2pGCY9ydE17xJD4qU5muctYgVjr7G16BlHA+w0dtlVM05kcW/PCAlTTeyPDxeIy6xfkzbxcI90g6pAmAMOpVatVmm2Xs2oFJnQn/QZcSbuJacsx9ncH4VQRffT0naaaW80xy0oDxbrzi2RzmBgGmvSzbNakwpJjqiRQJIgyZr5bLtG6Nvolmx7nm1dxSwFaNGcrKpeQ1zJ3UQT/TD2iI5m6Pw0p73TIh9InoPlrYsYWArykCSZ1RGCKcBOrY6JXevfFLpSN4Su2lrl0NYVYrwpua6hXAEJguB54mK4XyJINLAgAU86muRx7hicWjIRAi+b9LocGzB+DVIS0ecCVrqWcyRn4KHYEZ3tiM2vayKeFToCGUpBqlkc/N1McOIOT+oW9VgqiCVjfjKGJghFycbQKQVOClEHj24IEWjvgyQ3+2LUnYwG8dA6KXsejiZoCYUA/RhYpxtAZW8yeR7fGncDWnhZG0oHTkQCTsfIvIXDvjxEWoaqhl7fRKDalJSVHvr/zEoS/VxY92VBs2C9lyEkj4LHXfB4XuctMPGoRIVgEg+emAlZjvU1srhLVuoscfqhcq666P5LU/DiBvdXFonHtd9NGl87o/GShbzrWnY32cWjjLeKL4yUEBsVcIrihEuAaGl7ftyCvZU+6yYyFQmp0EUqPmnW2EgIs0S73bhxi81TmI8J4o62XUL4Qldzmy5kcdVENnCrphIWTdrTNFCr+57s6OJ80ay+srrwSBasKdrY5lTLWLvFnRGO74QykwAG/4bEf0RITGR1qk4Vdeq7QmKsSlFnAfrOmKC0cLpg2anyC14bE3NfsqVbl9uhtyay2yjH+O3XxNS5CLVevTukzoYH1XxHZYXUZ8bBhtS3bErcEZVgRV8MSsQQRneG9IDO3m20hgJU3XoLUCtAIQ+UkIKxiOf+G4zIc3x0i4qJsVjU+j/DN5aCV1mVZ4laQeVxLrAMvMKugFmywOxfsDMIDHSC95it4c0M3VTY5La1T1e2gvTds+YYMHkkzWg3/Ow24znIuwnl8Kc0yf7AtZ2FBFyu4uHNIPa106ViqJM+CTJtSYVfsmRKawUYm1eJq+FFSrY3RhpXDTgUWLPdoXBhxGHraBVE9c+E5p+6tgbFUM4EImXErwxVyVkvJoKla0kjnMsqf9+/CL7Hff+eba+KQWw69jgXZPzOs4F7seF+o0622tNG/cG9t+2e2nrsvHyM+V2DGO/sR7pWtlHjPX09lwREsUeluhJT3OHSL7IkpVZdw+tt5YKuZJYh2EVQSuoFFDBzgFJw059c0OSBwtdkEzTTreiGt0ioLzWDhMKtODAIUGPPa8HXEKBWEFVL25h28Kea/snATzY9y5h0Kc5f2xteahS4myUEwPVpDHTzKQy5CA3XjxzYGzHkeC5JVBNr/Oo2Jslk/RdNfnbJU+pi4iJ/0g4tFalwz/2CzqZ0K4NGYUTv5lIrclss9RrbpnB6dNztzqe5VAgiTFypVOlcQkucs5IrbWBLo9yXXMvGmY5r4dNQj+mEzJVR9VzbcP8/j4urraw5a7Ajt2KE2SLNTZI2Im2M26O4+/G7BCg+JuolyXzEbfsHzLctpqi/i6nPxF2NGzGTvHVz6XMA+dkNmPyOERP79HMbQ/n2LM3mVOxrZ6AXmZrCT8HWjXWEn+eR87b1WK3Rqt5j9sddlhtxJb/5jA0s46yzSWgnj9WijnbCHwz8AV2aJW1gBQmcKQeMGjndwGSUkhRuZ+e7VoZ3H4bjzwrT2wWvIwEZGwgF7RXcly2K8YBJiudVJ4dtFwY3/PF3qqbOsheqwpqkCaa/+qkmdKUl7IXkT04K6n/LUYpiuIpNNipO2NfJf8diKRhvh0N/8EcELFGj74H4282DrVJhXHNl9+TSdv9XLAQfsxcMJKXk+bWTjJjmhAnVo1Zx3YQFqO8+osOcrtEVR3UxcW67Rofpt2dOHkuQMJ2LxkgCszJmqQpFYURcr5SxPsZKuZgYe5uUx/QfLGSXLsq4TbAUQ5+XMcbua5F7LCMWr6KF0emHrF1XBqZeoelvUwZ8utF5p/jJ89m85pR0XFutcc1GEm5fP5b9QuLi6SCWYZP3mH/RwWuDyjqCe09eoaMKC3rUqLF1qkrWsjpXI9hKPUfcp49iv3kLBP/CZa2oR59YRhL37shLjpco6zW3kF7b4eeAOSi75IS/oMjfu+hjcntmDZbA60XSnqheav/lRR+e20UnF0MYy9BUPlsN/JKnO1oxrR8JIfRcV3btz56hX09ELWrURRP4elooUamyjWzgmSTpfBHGqFynuV2lUvABazK4rPsrGZbyMaH3MV71oQQNR5f/j0LLGyLKv8hd///lVeApVirkyaFNWoLbbEyU7JHNzsx60VsCrE8m0f1xeXHYHDiwlBBCo0UbMhzcW3tQ5EnAEOcndo/6GMBhEl/KiOnILPRVGyeHmooUqphIIe/RS4vmq7/DNd7kBrHkYk6FLpn0rYdCDE3m30GslfFm8qU5/Aq4iTYbCIQBj4k6hVnYqBl7Lxx1HDAjiMT/YtJvxiR3LW0mduRGS0vLwews/QlARRfNhjccxdT44gFmIg3DCj/gVBRXXNFSJU8Ci4huQW93+gPV3IMaUPxzq8HIU19XrbdAocydvCQMccz4faNpytZgN++a2/mBMN7eRl1hohv93oCYTdnVbLrfydZro2+xu/4SYpgmv/QXwpjQR9Tbdv7+tFC7jWBoNJvzbsOYj5tPcndPKvQomL2PdorB7Yc8sxeWbbffGUxHQbOj9Zo7Zf2iv0K7eUA+yQa3jeum0N9Bmja2Q4OjhCXn99tPpDTdr/otbtPnOU+YMD/45f6gWsoPuNe1LunKa7E674pA4YJQe1xVCBq++v7kClYjmAeN1SxomHL3zZtPWUIJ6oiWXV+39T7fBI7st8phs5zBzKXeiFBa9k6gBsF82iGk97k5oxqeDPQoAbtTLGUHYy1zzwbz7iw9Nq3SylY9iFuZfloLhzY7nKx2Yqu/gyc9xRpZwvuAldejhcaT0K9o8Aan93sLT+6yh+F6QM3TtEBvasDSYjy+CDwR+Qaaf+a55/GTrfZG/tCo7+AJWlgrh2j+9UBscb5AQ79BP8MzhWqspPcmSH6Vpv8gU6ON0oW6MXuQ6bdgRr1ZaC7zMQvyftmNOqM29Ic0xkdthQm3nfdGqxxfNrLlHWZUx59PCvM51Y1jvlodb/4+8GfTgakEjVR9RHNOa/fxWJt5izAm3Q4LevMhxm01KBN7TSZuwz6P+77S02N+zTfzd5WXab3RX3K6yHN7sSVuh+Hv161oRH2Arixm1t4Upk+LGeWtFatBytYLsgJTDUAvW/As0s9WH+Rx4ivIL5wDyMNXrLcD0EUlc5CmrKNQg4XSfa1m5sMzrDhuBuJEpMXla0znib9Da7SbUexCniIr3O/mVN2TpoOdbLApPVA3IAMb5hrPJx6z+7aZUwsiKg/nBxb83CVNtdtZgYUuwqdD6tDsTuVAjfG83CPtrr6uRl24BivZvIMmOMkz4b/e02q+3AC3GqQKnO8vmTpYY3UO2ieDFT1bpjebqmakgR7CoLpAjZZDaw6SJxdqK2qv9kx//mKf+j31HYBm7MCil1lNapKK5UdWBv2hf/LUJLT3pimQAwJQwx1PycWcejX6fD2cNYy6Hm3U7t58Xvq7cNSuScjdV1TLN47zQ7XIUofmU3sE8xGoE1+AD6v5dH7I8m1kztfiHjRQFw5MPWVZ1VCzpyNKmr7RyTNKt7OeIwkCcoJklnMr+hNpLLTpPqHxTcU6tRksZxT6V2zTQBq3UXtgxcu2kYwhU0//wBgEoLOLUBow3gy1INYMsAzQanN1qV6dmqs5oDigZwC+IOS/OOGM09xPGh3NKU3LhXbvA3s2He1UQLrE4pA1nxBjQ8MzK5lWPJVCNCJrFf3BAd8ntNmC96GF6cKUpqqN9FCcKGS6b4QUaVkMx8e6gwTPzumnsM3c6rgK9XocZ26ZhAB6pFjqRjZIQqa5zNxy9S7oBrI2/ySn+UblSW/2/rSWpgx4AHMt9V4u9WHI9IiQKeIsDzWrsZMnERIOeSz/lggFB5P2NjXPbD16ftnfDvD1R/6HaDoRU/MxtxINmgh2NlwDvYBOakAilCXO0HLsC/CAkabkRrPeVkjXsH1aA1JZP5ny+gU8G0L2fqO/2i+06VvwfmgnttxrQtyhHyUI6OEi7QROunLLXoqHl8OgJRJnRgB/w5ozGsUn5BKQhQLLBjR5zXEcpLQOx6FP/SBrJ3TxoG8Hhg8axlD9rgh+xPcGBsP2AWOh3BMniLPgUwyGFHgi/N1vzUAaPviMdiAsX93BGJ6Nr+P2vhC24TxUJ0R1ujBJt2m2+q0XQlj23cES4Tl3gPgK2r6yqG2Gro6tT8YcibREAH/1EnDBANED19A/C/1DjKUD7ahehOdnqWtU7h+GYzQnzh20uC34qCW6hv4IoBvu/ZC+cI6tkDYX/OEe0SbyqG3bRf5VWK5cFCMMDA7ai3DdjttyqB3MR/SF5YsrHtqoTTgm9Af8eg3jRvCJOkhuKwZ+ADyiYb400EjDM/oAzTeI59tSGGGpAF8U5M8pmOtBaOnQd19HlgD0MjC/APjnJ+XiAejifYiv0PN94OMMeNvWZR7JgtNnAefBMwTIENqvkCwQHR7wgOzzRHY+cA/lwfE54BO6FkK5+WguyXXEN96PaAh/izHvOaQHJPDfRdYJ80AyIxL5wDyAT6uQl8NYrgPUJ6JpKaJyxAsa6A1/99vQ74Q7RP2JqO+QB9FzIU980JPEqqLIEq7OrjJTi8r7c/vwaYLYjwocU3TK2K+q1StUdvcd1qNJ5nz6bIuuMJiXQlnm62tB/G7279i0+rb854uf3k3vdF1s+OtfjNKI1mFs/eM/rnT/WTaW9g9ngXkFPHrJm8O8hfg3W+67suXYetFIvy1bDqsH1JdMtPSXoD6difW7cmiLp2JkVrTV/Ncbbj3cp3MvQR2/elzya9X5LN2E4N+V1duqM4LHLzdEQPe6my3ZX8szzGcFuo4E1o85xP37IYrf8SGKx/yHhTH5v7gvQtZLADSsxpBFlSlC3N0fkyj1nYOboo5LZvEf+s5b7hs5ZJIq9emPUuQ6YvJZtCWhZ+5zcrlXIr4HDMli4u6/4zNvZ7MG/9sffruMT3/yh98obKe/IAOwuCy5euaeO2gnjkfrqStMRg+39ow3w9m6D/HLlVfzXcsb+9vm9PeU/7+VeVTHvCryXaf5B9kGHHjaDA9resG//BBNe4NNufsNWzqneC3D65vDmvuSmNhilHUBtUpfUeYzcGtsMa/71rAo39fpvYTyP6yBVaQ/LLX7K5rz7YJu3Lt1kH8x/GYpf3tyeI6wspPDsSp3Njf82ndAfkkq+M3xX1YDzxvX788Ez728dE+YCLen/0svan76Pwnp9v8A \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/Security-Compliance/diagram.png b/docs/Security-Compliance/diagram.png index 1bad6a52d..306ec5845 100644 Binary files a/docs/Security-Compliance/diagram.png and b/docs/Security-Compliance/diagram.png differ diff --git a/docs/Sprint-Review/sprint-82-summary.md b/docs/Sprint-Review/sprint-82-summary.md new file mode 100644 index 000000000..cba846de1 --- /dev/null +++ b/docs/Sprint-Review/sprint-82-summary.md @@ -0,0 +1,49 @@ + +# Sprint 82 Summary + +09/13/23 - 09/26/23 + +Velocity: Dev (13) + +## Sprint Goal +* Continue parsing engine development for TANF Section (04) and SSP (01), close out subsmission history and metadata workflows (1613/12/10). +* UX to continue regional staff and in-app messaging research, errors audit approach, and bridge onboarding to >95% of total users +* DevOps to investigate singluar ClamAV (2429), resolve utlity images for CircleCI and evaluate CI/CD pipeline. + +## Tickets +### Completed/Merged +* [#1613 As a developer, I need parsed file meta data (TANF Section 1)](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/board) +* [#2700 Deployment/migration issue](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2700) + +### Ready to Merge +* N/A + +### Submitted (QASP Review, OCIO Review) +* [#1612 Detailed case level metadata](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1612) + +### Closed (not merged) +* N/A + +## Moved to Next Sprint (Blocked, Raft Review, In Progress, Current Sprint Backlog) +### In Progress + +* [#2429 Singular ClamAV scanner](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2429) +* [#2695 space-filled values update](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2695) +* [#2411 As system admin, I need to view metadata on parsed datafiles](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2411) +* [#2536 [spike] Cat 4 validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2536) +* [#2709 SSP (Section 1) validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2709) + +### Blocked +* N/A + +### Raft Review +* [#1610 As a user, I need information about the acceptance of my data and a link for the error report](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1610) +* [#1111 TANF (04) Parsing and Validation](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/1111) +* [#2664 (bug) file extension](https://app.zenhub.com/workspaces/sprint-board-5f18ab06dfd91c000f7e682e/issues/gh/raft-tech/tanf-app/2664) + +### Demo +* Internal: + * 1613 + + + diff --git a/docs/Technical-Documentation/README.md b/docs/Technical-Documentation/README.md index 8d7d2b1dd..e328771cd 100644 --- a/docs/Technical-Documentation/README.md +++ b/docs/Technical-Documentation/README.md @@ -12,6 +12,7 @@ This directory contains system and architecture documentation including diagrams - [buildpack-changelog.md](./buildpack-changelog.md) : A running log of updates to our Cloud.gov buildpacks in use. - [circle-ci-audit-template.md](./circle-ci-audit-template.md) : This is a checklist document used during audits of our continuous intergration pipeline tool. - [circle-ci.md](./circle-ci.md) : Overview of our CI/CD platform jobs. +- [clamav.md](./clamav.md) : How to access ClamAV from different apps/spaces. - [cypress-integration-tests.md](./cypress-integration-tests.md) : Shows how we use Cypress to manage our end to end integration testing. - [data-file-downloads.md](./data-file-downloads.md) : Provides an architecture-level view of data file storage and downloading. - [django-admin-logging.md](./django-admin-logging.md) : Outlines sections of the Django Administrator Console and details what should be logged. diff --git a/docs/Technical-Documentation/clamav.md b/docs/Technical-Documentation/clamav.md new file mode 100644 index 000000000..894853348 --- /dev/null +++ b/docs/Technical-Documentation/clamav.md @@ -0,0 +1,48 @@ +# CLAMAV + +In order to have one CLAMAV instance, existing in prod, the Nginx router is created +for CLAMAV to forward the traffic from 'dev' and 'staging' spaces into +prod space, where the CLAMAV service exists. + +## Deploy Nginx instance +To route the clamav traffic to clamav in prod, each space needs to have one instance of _Nginx Router_ which routes traffic to clamav. + +In order to deploy the nginx router instance, change your directory to `tdrs-backend/clamav-router/` and run the following command while logged into the target space: + + +>`cf push tdp-clamav-nginx-${cf-shortened-space} -f manifest.yml --no-route` + +, where _cf-shortened-space_ can be : _dev_, _staging_, or _prod_. + +The instance name then will be set as an environment variable to redirect each instance traffic. This will deploy the nginx instance to the target environment. + +## Further communication configurations + +### Setup Individual Instances + +First, set the environment variable __AV_SCAN_URL__ as follows: +``` +Environment variable name: AV_SCAN_URL +Environment variable value: http://{nginx_instance}.apps.internal:9000/scan +``` + +### Add network policy from _{tdp-clamav-nginx}_ to clamav in prod +To enable traffic between the "__nginx instance__" and "__clamav instance in production__", we need to add the network policiy and route between the two: + +>`cf add-network-policy {nginx_instance} "clamav-rest" -s "tanf-prod" --protocol tcp --port 9000` +e.g: `{nginx_instance_name} = tdp-clamav-nginx-dev` + +### Add network policy from _{backend_instance}_ to _tdp-clamav-nginx_ + +>`cf add-network-policy {backend_instance} {nginx_instance} --protocol tcp --port 9000` + +where e.g: `backend_instance = tdp-backend-develop` + +### Add route for _tdp-clamav-nginx_ + + Note: Make sure to delete (if existing) routes that are not being used. In some rare cases, a mal-assigned network policy can interfere with outgoing traffic. As an example, a policy like + >`cf delete-route app.cloud.gov --hostname tdp-frontend-staging` + +Add route: + +>`cf map-route {nginx_instance} apps.internal --hostname {nginx_instance}` \ No newline at end of file diff --git a/scripts/deploy-backend.sh b/scripts/deploy-backend.sh index f3ed5941d..f50152891 100755 --- a/scripts/deploy-backend.sh +++ b/scripts/deploy-backend.sh @@ -40,7 +40,6 @@ set_cf_envs() "AMS_CLIENT_ID" "AMS_CLIENT_SECRET" "AMS_CONFIGURATION_ENDPOINT" - "AV_SCAN_URL" "BASE_URL" "CLAMAV_NEEDED" "CYPRESS_TOKEN" @@ -84,6 +83,15 @@ generate_jwt_cert() update_backend() { cd tdrs-backend || exit + cf unset-env "$CGAPPNAME_BACKEND" "AV_SCAN_URL" + + if ["$CF_SPACE" = "tanf-prod" ]; then + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tanf-prod-clamav-rest.apps.internal:9000/scan" + else + # Add environment varilables for clamav + cf set-env "$CGAPPNAME_BACKEND" AV_SCAN_URL "http://tdp-clamav-nginx-$env.apps.internal:9000/scan" + fi + if [ "$1" = "rolling" ] ; then set_cf_envs @@ -101,11 +109,18 @@ update_backend() fi set_cf_envs - + cf map-route "$CGAPPNAME_BACKEND" apps.internal --hostname "$CGAPPNAME_BACKEND" # Add network policy to allow frontend to access backend cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080 + + if ["$CF_SPACE" = "tanf-prod" ]; then + # Add network policy to allow backend to access tanf-prod services + cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000 + else + cf add-network-policy "$CGAPPNAME_BACKEND" tdp-clamav-nginx-$env --protocol tcp --port 9000 + fi cd .. } diff --git a/tdrs-backend/clamav-router/manifest.yml b/tdrs-backend/clamav-router/manifest.yml new file mode 100644 index 000000000..0df08de81 --- /dev/null +++ b/tdrs-backend/clamav-router/manifest.yml @@ -0,0 +1,9 @@ +version: 1 +applications: +- name: tdp-clamav-nginx + buildpacks: + - https://github.com/cloudfoundry/nginx-buildpack.git#v1.2.6 + memory: 32M + instances: 1 + disk_quota: 64M + timeout: 180 diff --git a/tdrs-backend/clamav-router/nginx.conf b/tdrs-backend/clamav-router/nginx.conf new file mode 100644 index 000000000..50cc6395b --- /dev/null +++ b/tdrs-backend/clamav-router/nginx.conf @@ -0,0 +1,20 @@ +events { worker_connections 1024; +} + +# This opens a route to clamav prod +http{ + server { + listen {{port}}; + location /scan { + proxy_pass http://tanf-prod-clamav-rest.apps.internal:9000/scan; + proxy_pass_request_headers on; + } + } + server { + listen 9000; + location /scan { + proxy_pass http://tanf-prod-clamav-rest.apps.internal:9000/scan; + proxy_pass_request_headers on; + } + } +} diff --git a/tdrs-backend/manifest.clamav.yml b/tdrs-backend/manifest.clamav.yml index 6883e83e5..157064ea8 100644 --- a/tdrs-backend/manifest.clamav.yml +++ b/tdrs-backend/manifest.clamav.yml @@ -9,4 +9,4 @@ applications: env: MAX_FILE_SIZE: 200M routes: - - route: ((cf-space))-clamav-rest.apps.internal + - route: tanf-prod-clamav-rest.apps.internal diff --git a/tdrs-backend/manifest.yml b/tdrs-backend/manifest.yml index dcf688000..bc059b375 100755 --- a/tdrs-backend/manifest.yml +++ b/tdrs-backend/manifest.yml @@ -6,5 +6,3 @@ applications: disk_quota: 2G docker: image: ((docker-backend)) - env: - AV_SCAN_URL: http://((cf-space))-clamav-rest.apps.internal:9000/scan diff --git a/tdrs-backend/tdpservice/data_files/admin.py b/tdrs-backend/tdpservice/data_files/admin.py index 1ac5b2b2c..1a049dad3 100644 --- a/tdrs-backend/tdpservice/data_files/admin.py +++ b/tdrs-backend/tdpservice/data_files/admin.py @@ -3,12 +3,64 @@ from ..core.utils import ReadOnlyAdminMixin from .models import DataFile, LegacyFileTransfer +from tdpservice.parsers.models import DataFileSummary, ParserError +from django.conf import settings +from django.utils.html import format_html +DOMAIN = settings.FRONTEND_BASE_URL + +class DataFileSummaryPrgTypeFilter(admin.SimpleListFilter): + """Admin class filter for Program Type on datafile model.""" + + title = 'Program Type' + parameter_name = 'program_type' + + def lookups(self, request, model_admin): + """Return a list of tuples.""" + return [ + ('TAN', 'TAN'), + ('SSP', 'SSP'), + ] + + def queryset(self, request, queryset): + """Return a queryset.""" + if self.value(): + query_set_ids = [df.id for df in queryset if df.prog_type == self.value()] + return queryset.filter(id__in=query_set_ids) + else: + return queryset @admin.register(DataFile) class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): """Admin class for DataFile models.""" + def status(self, obj): + """Return the status of the data file summary.""" + return DataFileSummary.objects.get(datafile=obj).status + + def case_totals(self, obj): + """Return the case totals.""" + return DataFileSummary.objects.get(datafile=obj).case_aggregates + + def error_report_link(self, obj): + """Return the link to the error report.""" + pe_len = ParserError.objects.filter(file=obj).count() + + filtered_parserror_list_url = f'{DOMAIN}/admin/parsers/parsererror/?file=' + str(obj.id) + # have to find the error id from obj + return format_html("{field}", + field="Parser Errors: " + str(pe_len), + url=filtered_parserror_list_url) + + error_report_link.allow_tags = True + + def data_file_summary(self, obj): + """Return the data file summary.""" + df = DataFileSummary.objects.get(datafile=obj) + return format_html("{field}", + field=f'{df.id}' + ":" + df.get_status(), + url=f"{DOMAIN}/admin/parsers/datafilesummary/{df.id}/change/") + list_display = [ 'id', 'stt', @@ -16,6 +68,8 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): 'quarter', 'section', 'version', + 'data_file_summary', + 'error_report_link', ] list_filter = [ @@ -25,6 +79,8 @@ class DataFileAdmin(ReadOnlyAdminMixin, admin.ModelAdmin): 'user', 'year', 'version', + 'summary__status', + DataFileSummaryPrgTypeFilter ] @admin.register(LegacyFileTransfer) diff --git a/tdrs-backend/tdpservice/data_files/models.py b/tdrs-backend/tdpservice/data_files/models.py index b4248a9cd..abfcce8ab 100644 --- a/tdrs-backend/tdpservice/data_files/models.py +++ b/tdrs-backend/tdpservice/data_files/models.py @@ -152,6 +152,20 @@ class Meta: null=True ) + @property + def prog_type(self): + """Return the program type for a given section.""" + # e.g., 'SSP Closed Case Data' + if self.section.startswith('SSP'): + return 'SSP' + elif self.section.startswith('Tribal'): + return 'TAN' # problematic, do we need to infer tribal entirely from tribe/fips code? + else: + return 'TAN' + + # TODO: if given a datafile (section), we can reverse back to the program b/c the + # section string has "tribal/ssp" in it, then process of elimination we have tanf + @property def filename(self): """Return the correct filename for this data file.""" diff --git a/tdrs-backend/tdpservice/data_files/test/test_admin.py b/tdrs-backend/tdpservice/data_files/test/test_admin.py new file mode 100644 index 000000000..02701fe82 --- /dev/null +++ b/tdrs-backend/tdpservice/data_files/test/test_admin.py @@ -0,0 +1,22 @@ +"""Test DataFileAdmin methods.""" +import pytest +from django.contrib.admin.sites import AdminSite + +from tdpservice.data_files.admin import DataFileAdmin +from tdpservice.data_files.models import DataFile +from tdpservice.data_files.test.factories import DataFileFactory +from tdpservice.parsers.test.factories import DataFileSummaryFactory +from django.conf import settings + +@pytest.mark.django_db +def test_DataFileAdmin_status(): + """Test DataFileAdmin status method.""" + data_file = DataFileFactory() + data_file_summary = DataFileSummaryFactory(datafile=data_file) + data_file_admin = DataFileAdmin(DataFile, AdminSite()) + + assert data_file_admin.status(data_file) == data_file_summary.status + assert data_file_admin.case_totals(data_file) == data_file_summary.case_aggregates + DOMAIN = settings.FRONTEND_BASE_URL + assert data_file_admin.error_report_link(data_file) == \ + f"Parser Errors: 0" diff --git a/tdrs-backend/tdpservice/settings/cloudgov.py b/tdrs-backend/tdpservice/settings/cloudgov.py index 541d98cc0..6f7c7342b 100644 --- a/tdrs-backend/tdpservice/settings/cloudgov.py +++ b/tdrs-backend/tdpservice/settings/cloudgov.py @@ -43,7 +43,6 @@ class CloudGov(Common): cloudgov_space = cloudgov_app.get('space_name', 'tanf-dev') cloudgov_space_suffix = cloudgov_space.strip('tanf-') - AV_SCAN_URL = f'http://tanf-{cloudgov_space_suffix}-clamav-rest.apps.internal:9000/scan' cloudgov_name = cloudgov_app.get('name').split("-")[-1] # converting "tdp-backend-name" to just "name" services_basename = cloudgov_name if ( cloudgov_name == "develop" and cloudgov_space_suffix == "staging" diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index e69a0b8c8..9abbb2c15 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -326,7 +326,7 @@ class Common(Configuration): logger.debug("RAW_CLAMAV: " + str(RAW_CLAMAV)) CLAMAV_NEEDED = bool(strtobool(RAW_CLAMAV)) - # The URL endpoint to send AV scan requests to (clamav-rest) + # The URL endpoint to send AV scan requests to (clamav-rest/clamav-nginx-proxy) AV_SCAN_URL = os.getenv('AV_SCAN_URL') # The factor used to determine how long to wait before retrying failed scans diff --git a/tdrs-frontend/cypress/e2e/accounts/accounts.js b/tdrs-frontend/cypress/e2e/accounts/accounts.js index 3c6158204..a1feaedf3 100644 --- a/tdrs-frontend/cypress/e2e/accounts/accounts.js +++ b/tdrs-frontend/cypress/e2e/accounts/accounts.js @@ -8,7 +8,9 @@ Then('{string} sees a Request Access form', (username) => { Then('{string} can see the hompage', (username) => { cy.visit('/home') - cy.contains('You have been approved for access to TDP.').should('exist') + cy.wait(2000).then(() => { + cy.contains('You have been approved for access to TDP.').should('exist') + }) }) When('{string} is in begin state', (username) => {