Merge branch 'develop' into 2458-nexus-mvp

Taskfile.yml

@@ -2,6 +2,11 @@ version: '3'
 
 tasks:
 
+  upload-kibana-objs:
+    desc: Upload dashboards to Kibana server
+    cmds:
+      - curl -X POST localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson
+
   create-network:
     desc: Create the external network
     cmds:
@@ -29,7 +34,7 @@ tasks:
     desc: Create Sentry service
     dir: sentry
     cmds:
-      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0 
+      # limiting the memory to 2GB and CPU to only one cpu @0, for faster response, you can remove the limittask :  --cpuset-cpus 0
       - (docker run --privileged -p 9001:9000 -d --memory="8g" --memory-swap="8g" --name sentry docker:dind) || true
       - docker exec sentry sh -c "git clone https://github.com/getsentry/self-hosted.git || true"
 
@@ -155,7 +160,7 @@ tasks:
       - docker rm $(docker ps -aq) || true
       - docker rmi $(docker images -q) || true
       - docker volume rm $(docker volume ls -q) || true
-  
+
   clamav-up:
     desc: Start clamav service
     dir: tdrs-backend
@@ -187,7 +192,7 @@ tasks:
       - task: frontend-up
       - task: clamav-up
 
-  
+
   # need more work
   frontend-init:
     desc: Initialize the frontend project

scripts/apply-remote-migrations.sh

@@ -5,7 +5,7 @@ app=${1}
 cd ./tdrs-backend
 
 echo "Install dependencies..."
-sudo apt install -y gcc
+sudo apt-get install -y gcc && sudo apt-get install -y graphviz && sudo apt-get install -y graphviz-dev
 sudo apt install -y libpq-dev python3-dev
 
 python -m venv ./env

scripts/deploy-backend.sh

@@ -99,6 +99,10 @@ update_kibana()
   cf add-network-policy "$CGAPPNAME_BACKEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
   cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_KIBANA" --protocol tcp --port 5601
   cf add-network-policy "$CGAPPNAME_KIBANA" "$CGAPPNAME_FRONTEND" --protocol tcp --port 80
+
+  # Upload dashboards to Kibana
+  CMD="curl -X POST $CGAPPNAME_KIBANA.apps.internal:5601/api/saved_objects/_import -H 'kbn-xsrf: true' --form file=@/home/vcap/app/tdpservice/search_indexes/kibana_saved_objs.ndjson"
+  cf run-task $CGAPPNAME_BACKEND --command "$CMD" --name kibana-obj-upload
 }
 
 update_backend()

tdrs-backend/docker-compose.yml

@@ -55,6 +55,8 @@ services:
       - SERVER_BASEPATH=/kibana
       - SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY=no-referrer
       - CSP_WARNLEGACYBROWSERS=false
+    volumes:
+      - ./search_indexes/kibana_saved_objs.ndjson:/usr/share/kibana/kibana_saved_objs.ndjson
     depends_on:
       - elastic
 

tdrs-backend/manifest.kibana.yml

@@ -10,7 +10,7 @@ applications:
       SERVER_SECURITYRESPONSEHEADERS_REFERRERPOLICY: no-referrer
       CSP_WARNLEGACYBROWSERS: false
     docker:
-      image: docker.elastic.co/kibana/kibana-oss:7.4.2
+      image: docker.elastic.co/kibana/kibana-oss:7.10.2
     command: |
       export ELASTICSEARCH_HOSTS=http://$CGAPPNAME_PROXY.apps.internal:8080 &&
       /usr/local/bin/dumb-init -- /usr/local/bin/kibana-docker

tdrs-backend/tdpservice/data_files/util.py

@@ -3,6 +3,7 @@
 from io import BytesIO
 import xlsxwriter
 import calendar
+from tdpservice.parsers.models import ParserErrorCategoryChoices
 
 
 def get_xls_serialized_file(data):
@@ -48,6 +49,7 @@ def format_error_msg(x):
         ('item_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].values()])),
         ('internal_variable_name', lambda x: ','.join([i for i in chk(x)['fields_json']['friendly_name'].keys()])),
         ('row_number', lambda x: x['row_number']),
+        ('error_type', lambda x: str(ParserErrorCategoryChoices(x['error_type']).label)),
     ]
 
     # write beta banner

tdrs-backend/tdpservice/parsers/test/test_parse.py

@@ -1300,7 +1300,7 @@ def test_parse_tribal_section_2_file(tribal_section_2_file, dfs):
     t4 = t4_objs.first()
     t5 = t5_objs.last()
 
-    assert t4.CLOSURE_REASON == 8
+    assert t4.CLOSURE_REASON == '15'
     assert t5.COUNTABLE_MONTH_FED_TIME == '  8'
 
 

tdrs-backend/tdpservice/parsers/transforms.py

@@ -12,7 +12,7 @@ def transform(value, **kwargs):
             month = transform_to_months(quarter)[month_index]
         except ValueError:
             return None
-        return f"{year}{month_to_int(month)}"
+        return int(f"{year}{month_to_int(month)}")
     return transform
 
 def tanf_ssn_decryption_func(value, **kwargs):

tdrs-backend/tdpservice/search_indexes/migrations/0031_alter_tribal_tanf_t4_closure_reason.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.15 on 2024-08-29 19:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('search_indexes', '0030_reparse_meta_model'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tribal_tanf_t4',
+            name='CLOSURE_REASON',
+            field=models.CharField(max_length=2, null=True),
+        ),
+    ]

tdrs-backend/tdpservice/search_indexes/models/tribal.py

@@ -233,7 +233,7 @@ class Meta:
     STRATUM = models.CharField(max_length=2, null=True, blank=False)
     ZIP_CODE = models.CharField(max_length=5, null=True, blank=False)
     DISPOSITION = models.IntegerField(null=True, blank=False)
-    CLOSURE_REASON = models.IntegerField(null=True, blank=False)
+    CLOSURE_REASON = models.CharField(max_length=2, null=True, blank=False)
     REC_SUB_HOUSING = models.IntegerField(null=True, blank=False)
     REC_MED_ASSIST = models.IntegerField(null=True, blank=False)
     REC_FOOD_STAMPS = models.IntegerField(null=True, blank=False)

terraform/dev/main.tf

@@ -106,7 +106,14 @@ data "cloudfoundry_service" "elasticsearch" {
 }
 
 resource "cloudfoundry_service_instance" "elasticsearch" {
-  name         = "es-dev"
-  space        = data.cloudfoundry_space.space.id
-  service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"]
+  name                     = "es-dev"
+  space                    = data.cloudfoundry_space.space.id
+  service_plan             = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"]
+  replace_on_params_change = true
+  json_params              = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}"
+  timeouts {
+    create = "60m"
+    update = "60m"
+    delete = "2h"
+  }
 }

terraform/production/main.tf

@@ -90,4 +90,11 @@ resource "cloudfoundry_service_instance" "elasticsearch" {
   name         = "es-prod"
   space        = data.cloudfoundry_space.space.id
   service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-medium"]
+  replace_on_params_change = true
+  json_params  = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}"
+  timeouts {
+    create = "60m"
+    update = "60m"
+    delete = "2h"
+  }
 }

terraform/staging/main.tf

@@ -90,4 +90,11 @@ resource "cloudfoundry_service_instance" "elasticsearch" {
   name         = "es-staging"
   space        = data.cloudfoundry_space.space.id
   service_plan = data.cloudfoundry_service.elasticsearch.service_plans["es-dev"]
+  replace_on_params_change = true
+  json_params  = "{\"ElasticsearchVersion\": \"Elasticsearch_7.10\"}"
+  timeouts {
+    create = "60m"
+    update = "60m"
+    delete = "2h"
+  }
 }