Merge branch 'develop' into 2990-stuck-files-notification

.circleci/base_config.yml

@@ -13,11 +13,11 @@ executors:
   machine-executor:
     machine:
       docker_layer_caching: false
-      image: ubuntu-2204:2024.01.1
+      image: ubuntu-2204:2024.05.1
   large-machine-executor:
     machine:
       docker_layer_caching: false
-      image: ubuntu-2204:2024.01.1
+      image: ubuntu-2204:2024.05.1
     resource_class: large
 
 parameters:

.circleci/build-and-test/commands.yml

@@ -49,6 +49,13 @@
       - run:
           name: Disable npm audit warnings in CI
           command: npm set audit false
-          
+
   # This allows us to use the node orb to install packages within other commands
   install-nodejs-packages: node/install-packages
+
+  docker-login:
+    steps:
+      - run:
+          name: Docker login
+          command: |
+            echo "$CIRCI_DOCKER_LOGIN" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-circi --password-stdin

.circleci/build-and-test/jobs.yml

@@ -3,6 +3,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - run:
           name: Run Unit Tests And Create Code Coverage Report
@@ -46,6 +47,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - docker-compose-up-frontend
       - install-nodejs-machine

.circleci/deployment/commands.yml

@@ -1,4 +1,33 @@
 # commands:
+  init-deploy:
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+
+  build-and-tag-images:
+    parameters:
+      backend-appname:
+        default: tdp-backend
+        type: string
+      frontend-appname:
+        default: tdp-frontend
+        type: string
+    steps:
+      - run:
+          name: Update Docker daemon
+          command: |
+            sudo echo '{"max-concurrent-uploads": 1}' | sudo tee /etc/docker/daemon.json
+            sudo service docker restart
+      - run:
+          name: Create builder
+          command: |
+            docker buildx create --name container-builder --driver docker-container --use --bootstrap
+      - run:
+          name: Build and tag images
+          command: |
+            ./scripts/build-and-tag-images.sh <<parameters.backend-appname>> <<parameters.frontend-appname>> ./tdrs-backend ./tdrs-frontend $CIRCLE_BUILD_NUM $CIRCLE_SHA1 "$CIRCI_DOCKER_LOGIN" tdp-circi
+
   deploy-cloud-dot-gov:
     parameters:
       environment:
@@ -25,9 +54,6 @@
         default: tdp-frontend
         type: string
     steps:
-      - checkout
-      - sudo-check
-      - cf-check
       - login-cloud-dot-gov:
           cf-password: <<parameters.cf-password>>
           cf-org: <<parameters.cf-org>>

.circleci/deployment/jobs.yml

@@ -1,10 +1,41 @@
+  build-and-tag-develop:
+    executor: large-machine-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - build-and-tag-images:
+          backend-appname: tdp-backend-develop
+          frontend-appname: tdp-frontend-develop
+
+  build-and-tag-staging:
+    executor: large-machine-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - build-and-tag-images:
+          backend-appname: tdp-backend-staging
+          frontend-appname: tdp-frontend-staging
+
+  build-and-tag-production:
+    executor: large-machine-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - build-and-tag-images:
+          backend-appname: tdp-backend-production
+          frontend-appname: tdp-frontend-production
+
   deploy-dev:
     parameters:
       target_env:
         type: string
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-<< parameters.target_env >>
           frontend-appname: tdp-frontend-<< parameters.target_env >>
@@ -13,6 +44,7 @@
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-staging
           frontend-appname: tdp-frontend-staging
@@ -24,6 +56,7 @@
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           backend-appname: tdp-backend-develop
           frontend-appname: tdp-frontend-develop
@@ -133,6 +166,7 @@
     executor: docker-executor
     working_directory: ~/tdp-deploy
     steps:
+      - init-deploy
       - deploy-cloud-dot-gov:
           environment: production
           backend-appname: tdp-backend-prod

.circleci/deployment/workflows.yml

@@ -93,27 +93,48 @@
                 - develop
                 - main
                 - master
-      - deploy-develop:
+      - build-and-tag-develop:
           requires:
             - deploy-infrastructure-staging
           filters:
             branches:
               only:
                 - develop
-      - deploy-staging:
+      - deploy-develop:
+          requires:
+            - build-and-tag-develop
+          filters:
+            branches:
+              only:
+                - develop
+      - build-and-tag-staging:
           requires:
             - deploy-infrastructure-staging
           filters:
             branches:
               only:
                 - main
-      - deploy-production:
+      - deploy-staging:
+          requires:
+            - build-and-tag-staging
+          filters:
+            branches:
+              only:
+                - main
+      - build-and-tag-production:
           requires:
             - deploy-infrastructure-production
           filters:
             branches:
               only:
                 - master
+      - deploy-production:
+          requires:
+            - build-and-tag-production
+          filters:
+            branches:
+              only:
+                - master
       - test-deployment-e2e:
           requires:
             - deploy-develop

.circleci/owasp/jobs.yml

@@ -4,6 +4,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - docker-compose-up-frontend
       - run:
@@ -26,6 +27,7 @@
     steps:
       - checkout
       - docker-compose-check
+      - docker-login
       - docker-compose-up-backend
       - docker-compose-up-frontend
       - run:
@@ -66,6 +68,7 @@
       - sudo-check
       - cf-check
       - docker-compose-check
+      - docker-login
       - login-cloud-dot-gov:
           cf-password: <<parameters.cf_password>>
           cf-space: <<parameters.cf_space>>

Taskfile.yml

@@ -2,6 +2,11 @@ version: '3'
 
 tasks:
 
+  upload-kibana-objs:
+    desc: Upload dashboards to Kibana server
+    cmds:
+      - curl -X POST localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson
+
   create-network:
     desc: Create the external network
     cmds:
@@ -29,7 +34,7 @@ tasks:
     desc: Create Sentry service
     dir: sentry
     cmds:
-      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0 
+      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0
       - (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true
       - docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true"
 
@@ -155,7 +160,7 @@ tasks:
       - docker rm $(docker ps -aq) || true
       - docker rmi $(docker images -q) || true
       - docker volume rm $(docker volume ls -q) || true
-  
+
   clamav-up:
     desc: Start clamav service
     dir: tdrs-backend
@@ -187,7 +192,7 @@ tasks:
       - task: frontend-up
       - task: clamav-up
 
-  
+
   # need more work
   frontend-init:
     desc: Initialize the frontend project

docs/Technical-Documentation/nexus-repo.md

@@ -40,29 +40,94 @@ After logging in as root for the first time, you will be taken to a page to set
 
 In order to use Nexus as a Docker repository, the DNS for the repo needs to be able to terminate https. We are currently using cloudflare to do this.
 
-When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port. 
+When creating the repository (must be signed in with admin privileges), since the nexus server isn't actually terminating the https, select the HTTP repository connector. The port can be anything you assign, as long as the tool used to terminate the https connection forwards the traffic to that port.
 
 In order to allow [Docker client login and connections](https://help.sonatype.com/repomanager3/nexus-repository-administration/formats/docker-registry/docker-authentication) you must set up the Docker Bearer Token Realm in Settings -> Security -> Realms -> and move the Docker Bearer Token Realm over to Active.
 Also, any users will need nx-repository-view-docker-#{RepoName}-(browse && read) at a minimum and (add and edit) in order to push images.
 
 We have a separate endpoint to connect specifically to the docker repository.
 [https://tdp-docker.dev.raftlabs.tech](tdp-docker.dev.raftlabs.tech)
 
-e.g. `docker login https://tdp-docker.dev.raftlabs.tech`
+e.g.
+```
+docker login https://tdp-docker.dev.raftlabs.tech
+```
 
 ### Pushing Images
 
 Before an image can be pushed to the nexus repository, it must be tagged for that repo:
 
-`docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}`
+```
+docker image tag ${ImageId} tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}
+```
 
 then you can push:
 
-`docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}`
+```
+docker push tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}
+```
 
 ### Pulling Images
 
-We have set up a proxy mirror to dockerhub that can pull and cache DockerHub images.
-Then we have created a group docker repository that can be pulled from. If the container is in our hosted repo, the group will return that container. If not, it will see if we have a cached version of that container in our proxy repo and, if not, pull that from dockerhub, cache it and allow the docker pull to happen.
+We do not allow anonymous access on our Nexus instance. With that said, if you have not [logged in with Docker](#docker-login) you will not be able to pull. If you are logged in:
+
+```
+docker pull tdp-docker.dev.raftlabs.tech/${ImageName}:${Version}
+```
+
+## Nexus Administration
+
+### UI Admin Login
+To administer Nexus via the UI, you will need to access the service key in our dev cloud.gov environment.
+
+Log in with CloudFoundry
+```
+cf login --sso
+```
+Be sure to specify the space as `tanf-dev`
 
-`docker pull https://tdp-docker-store.dev.raftlabs.tech/${ImageName}:${Version}`
+After you've authenticated you can grab the password from the key:
+```
+cf service-key tanf-keys nexus-dev-admin
+```
+
+The key returns a username and a password:
+```
+{
+  "credentials": {
+    "password": REDACTED,
+    "username": REDACTED
+  }
+}
+```
+Copy the `password` to your clipboard and login into the Nexus UI with the `tdp-dev-admin` user. See below:
+
+![Nexus Dev Admin Login](./images/nexus-dev-admin-login.png)
+
+### VM Login
+To access the VM running Nexus, you will need to gain access to the Raft internal network. To do this, you will need to install CloudFlare's WARP zero trust VPN. Follow the instructions [here](https://gorafttech-my.sharepoint.com/:w:/g/personal/tradin_teamraft_com/EZePOTv0dbdBguHITcoXQF0Bd5JAcqeLsJTlEOktTfIXHA?e=34WqB4) to get setup. From there, reach out to Eric Lipe or Connor Meehan for the IP, username, and password to access the VM. Once you have the credentials, you can login with SSH:
+```
+ssh username@IP_Address
+```
+
+Once logged in, you can run `docker ps` or other docker commands to view and administer the Nexus container as necessary. You should also consider generating an ssh key to avoid having to enter the password each time you login. To do so, run the following commands on your local machine.
+```
+ssh-keygen
+```
+
+```
+ssh-copy-id username@IP_Address
+```
+Now you will no longer have to enter the password when logging in.
+
+## Local Docker Login
+After logging into the `tanf-dev` space with the `cf` cli, execute the following commands to authenticate your local docker daemon
+```
+export NEXUS_DOCKER_PASSWORD=`cf service-key tanf-keys nexus-dev | tail -n +2 | jq .credentials.password`
+echo "$NEXUS_DOCKER_PASSWORD" | docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev --password-stdin
+```
+
+Sometimes the `docker login...` command above doesn't work. If that happens, just copy the content of `NEXUS_DOCKER_PASSWORD` to your clipboard and paste it when prompted for the password after executing the command below.
+```
+docker login https://tdp-docker.dev.raftlabs.tech -u tdp-dev
+```

scripts/apply-remote-migrations.sh

@@ -5,7 +5,7 @@ app=${1}
 cd ./tdrs-backend
 
 echo "Install dependencies..."
-sudo apt install -y gcc
+sudo apt-get install -y gcc && sudo apt-get install -y graphviz && sudo apt-get install -y graphviz-dev
 sudo apt install -y libpq-dev python3-dev
 
 python -m venv ./env