From 7257fb5d253ad1d23c9e8a0f5eb46d94a942ec24 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Tue, 13 Aug 2019 19:05:55 +0300 Subject: [PATCH 01/12] Add expectedIndicators to IndicatorQuery to support 0 for grouped results - Add exectedIndicators property for Multi-Result Indicator queries - Add migrations to add column for exected_indicators in indicator_queries table - Add sample multi-result query with expected_indicators - Enable reading the expected_indicators fields and saving it - Enable processing multi-result in case expectedIndicators have been defined and providing the expected output only Fixes #32 --- .../reporting/ReportingLibrary.java | 4 +- .../reporting/dao/ReportIndicatorDaoImpl.java | 72 ++++++++++++++++++- .../domain/CompositeIndicatorTally.java | 14 ++++ .../reporting/domain/IndicatorQuery.java | 16 ++++- .../domain/IndicatorYamlConfigItem.java | 12 ++++ .../DailyIndicatorCountRepository.java | 2 +- .../repository/IndicatorQueryRepository.java | 35 ++++++++- .../reporting/util/Constants.java | 1 + .../smartregister/reporting/dao/DaoTest.java | 2 +- .../dao/ReportIndicatorDaoImplTest.java | 38 ++++++++++ .../assets/config/indicator-definitions.yml | 8 ++- .../sample/repository/SampleRepository.java | 4 +- 12 files changed, 196 insertions(+), 12 deletions(-) create mode 100644 opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/ReportingLibrary.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/ReportingLibrary.java index 657752c7..1d9008e3 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/ReportingLibrary.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/ReportingLibrary.java @@ -223,9 +223,11 @@ private void readConfigFile(String configFilePath, SQLiteDatabase sqLiteDatabase indicatorQuery = new IndicatorQuery(null, indicatorYamlConfigItem.getKey() , indicatorYamlConfigItem.getIndicatorQuery() , 0 - , indicatorYamlConfigItem.isMultiResult()); + , indicatorYamlConfigItem.isMultiResult() + , indicatorYamlConfigItem.getExpectedIndicators()); indicatorQueries.add(indicatorQuery); } + reportIndicators.add(indicator); } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java index 69ea1d96..3ccc50e4 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java @@ -2,6 +2,7 @@ import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.annotation.VisibleForTesting; import com.google.gson.Gson; @@ -13,6 +14,7 @@ import org.smartregister.reporting.domain.IndicatorQuery; import org.smartregister.reporting.domain.IndicatorTally; import org.smartregister.reporting.domain.ReportIndicator; +import org.smartregister.reporting.processor.MultiResultProcessor; import org.smartregister.reporting.repository.DailyIndicatorCountRepository; import org.smartregister.reporting.repository.IndicatorQueryRepository; import org.smartregister.reporting.repository.IndicatorRepository; @@ -95,10 +97,34 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { ArrayList result = executeQueryAndReturnMultiResult(indicatorQuery.getQuery(), dates.getKey(), database); // If the size contains actual result other than the column names which are at index 0 - if (result.size() > 1) { + if (result.size() > 1 || (indicatorQuery.getExpectedIndicators() != null && indicatorQuery.getExpectedIndicators().size() > 0)) { tally = new CompositeIndicatorTally(); - tally.setValueSet(new Gson().toJson(result)); tally.setValueSetFlag(true); + tally.setValueSet(new Gson().toJson(result)); + tally.setExpectedIndicators(indicatorQuery.getExpectedIndicators()); + + if (indicatorQuery.getExpectedIndicators() != null && indicatorQuery.getExpectedIndicators().size() > 0) { + HashMap tallies = extractIndicatorTallies(result + , ReportingLibrary.getInstance().getDefaultMultiResultProcessor() + , ReportingLibrary.getInstance().getMultiResultProcessors() + , tally); + + result = new ArrayList<>(); + + List expectedIndicators = indicatorQuery.getExpectedIndicators(); + + for (String expectedIndicator: expectedIndicators) { + Float indicatorValue = 0F; + if (tallies.get(expectedIndicator) != null) { + indicatorValue = tallies.get(expectedIndicator); + } + + result.add(new Object[]{expectedIndicator, indicatorValue}); + } + + tally.setValueSet(new Gson().toJson(result)); + } + } } else { count = executeQueryAndReturnCount(indicatorQuery.getQuery(), dates.getKey(), database); @@ -267,4 +293,46 @@ public void setDailyIndicatorCountRepository(DailyIndicatorCountRepository daily public void setIndicatorRepository(IndicatorRepository indicatorRepository) { this.indicatorRepository = indicatorRepository; } + + @VisibleForTesting + public HashMap extractIndicatorTallies(ArrayList compositeTallies, @NonNull MultiResultProcessor defaultMultiResultProcessor + , @NonNull ArrayList multiResultProcessors, @NonNull CompositeIndicatorTally compositeIndicatorTally) { + + HashMap tallies = new HashMap<>(); + List uncondensedTallies = null; + + if (compositeTallies.size() > 1) { + Object[] objectFieldNames = (Object[]) compositeTallies.get(0); + String[] fieldNames = new String[objectFieldNames.length]; + + for (int i = 0; i < objectFieldNames.length; i++) { + fieldNames[i] = (String) objectFieldNames[i]; + } + + if (defaultMultiResultProcessor.canProcess(fieldNames.length,fieldNames)) { + uncondensedTallies = DailyIndicatorCountRepository.processMultipleTallies(defaultMultiResultProcessor, compositeIndicatorTally); + } else { + for (MultiResultProcessor multiResultProcessor: multiResultProcessors) { + if (multiResultProcessor.canProcess(fieldNames.length, fieldNames)) { + uncondensedTallies = DailyIndicatorCountRepository.processMultipleTallies(multiResultProcessor, compositeIndicatorTally); + } + } + } + } + + if (uncondensedTallies != null) { + for (IndicatorTally indicatorTally: uncondensedTallies) { + Float value = tallies.get(indicatorTally.getIndicatorCode()); + if (value != null) { + value = value + indicatorTally.getFloatCount(); + } else { + value = indicatorTally.getFloatCount(); + } + + tallies.put(indicatorTally.getIndicatorCode(), value); + } + } + + return tallies; + } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/CompositeIndicatorTally.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/CompositeIndicatorTally.java index ba9675c4..a7ea7f4d 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/CompositeIndicatorTally.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/CompositeIndicatorTally.java @@ -1,8 +1,10 @@ package org.smartregister.reporting.domain; import android.support.annotation.NonNull; +import android.support.annotation.Nullable; import java.util.Date; +import java.util.List; /** * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-07-10 @@ -13,6 +15,9 @@ public class CompositeIndicatorTally extends IndicatorTally { private String valueSet; private boolean isValueSet; + @Nullable + private List expectedIndicators; + public CompositeIndicatorTally() { } @@ -42,4 +47,13 @@ public boolean isValueSet() { public void setValueSetFlag(boolean valueSet) { isValueSet = valueSet; } + + @Nullable + public List getExpectedIndicators() { + return expectedIndicators; + } + + public void setExpectedIndicators(@Nullable List expectedIndicators) { + this.expectedIndicators = expectedIndicators; + } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorQuery.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorQuery.java index 6409e5db..7d58fc15 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorQuery.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorQuery.java @@ -1,18 +1,24 @@ package org.smartregister.reporting.domain; +import android.support.annotation.Nullable; + +import java.util.List; + public class IndicatorQuery { private Long id; private String indicatorCode; private String query; private int dbVersion; private boolean isMultiResult; + private List expectedIndicators; - public IndicatorQuery(Long id, String indicatorCode, String query, int dbVersion, boolean isMultiResult) { + public IndicatorQuery(Long id, String indicatorCode, String query, int dbVersion, boolean isMultiResult, @Nullable List expectedIndicators) { this.id = id; this.indicatorCode = indicatorCode; this.query = query; this.dbVersion = dbVersion; this.isMultiResult = isMultiResult; + this.expectedIndicators = expectedIndicators; } public IndicatorQuery() { @@ -57,4 +63,12 @@ public boolean isMultiResult() { public void setMultiResult(boolean multiResult) { isMultiResult = multiResult; } + + public List getExpectedIndicators() { + return expectedIndicators; + } + + public void setExpectedIndicators(List expectedIndicators) { + this.expectedIndicators = expectedIndicators; + } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorYamlConfigItem.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorYamlConfigItem.java index f87f8188..7096a421 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorYamlConfigItem.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/domain/IndicatorYamlConfigItem.java @@ -1,5 +1,7 @@ package org.smartregister.reporting.domain; +import java.util.List; + public class IndicatorYamlConfigItem { public static String INDICATOR_PROPERTY = "indicatorData"; @@ -8,6 +10,8 @@ public class IndicatorYamlConfigItem { private String description; private String indicatorQuery; private boolean isMultiResult; + private List expectedIndicators; + public String getKey() { return key; @@ -40,4 +44,12 @@ public boolean isMultiResult() { public void setMultiResult(boolean multiResult) { isMultiResult = multiResult; } + + public List getExpectedIndicators() { + return expectedIndicators; + } + + public void setExpectedIndicators(List expectedIndicators) { + this.expectedIndicators = expectedIndicators; + } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index f305cc4d..9c253e47 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -169,7 +169,7 @@ private void extractIndicatorTalliesFromMultiResult(@NonNull Map processMultipleTallies(@NonNull MultiResultProcessor defaultMultiResultProcessor + public static List processMultipleTallies(@NonNull MultiResultProcessor defaultMultiResultProcessor , @NonNull CompositeIndicatorTally compositeIndicatorTally) { try { return defaultMultiResultProcessor.processMultiResultTally(compositeIndicatorTally); diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java index ca9de882..0e185e5a 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java @@ -3,6 +3,10 @@ import android.content.ContentValues; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.text.TextUtils; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import net.sqlcipher.Cursor; import net.sqlcipher.database.SQLiteDatabase; @@ -14,6 +18,7 @@ import org.smartregister.repository.Repository; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -28,6 +33,7 @@ public class IndicatorQueryRepository extends BaseRepository { + "(" + Constants.IndicatorQueryRepository.ID + " INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " + Constants.IndicatorQueryRepository.QUERY + " TEXT NOT NULL, " + Constants.IndicatorQueryRepository.INDICATOR_CODE + " TEXT NOT NULL, " + Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT + " BOOLEAN NOT NULL DEFAULT 0, " + + Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS + " TEXT, " + Constants.IndicatorQueryRepository.DB_VERSION + " INTEGER)"; public IndicatorQueryRepository(Repository repository) { @@ -41,6 +47,12 @@ public static void performMigrations(@NonNull SQLiteDatabase database) { addMultiResultFlagField(database); aggregateDailyTallies(database); } + + if (Utils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) + && !Utils.isColumnExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS)) { + addExpectedIndicatorColumn(database); + } } public static void createTable(SQLiteDatabase database) { @@ -73,7 +85,7 @@ public Map getAllIndicatorQueries() { SQLiteDatabase database = getReadableDatabase(); String[] columns = {Constants.IndicatorQueryRepository.ID, Constants.IndicatorQueryRepository.INDICATOR_CODE , Constants.IndicatorQueryRepository.QUERY, Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT - , Constants.IndicatorQueryRepository.DB_VERSION}; + , Constants.IndicatorQueryRepository.DB_VERSION, Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS}; Cursor cursor = database.query(Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, columns , null, null, null, null, null, null); if (cursor != null && cursor.getCount() > 0 && cursor.moveToFirst()) { @@ -92,7 +104,7 @@ public IndicatorQuery findQueryByIndicatorCode(String indicatorCode) { IndicatorQuery indicatorQuery = null; String[] columns = {Constants.IndicatorQueryRepository.ID, Constants.IndicatorQueryRepository.INDICATOR_CODE , Constants.IndicatorQueryRepository.QUERY, Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT - , Constants.IndicatorQueryRepository.DB_VERSION}; + , Constants.IndicatorQueryRepository.DB_VERSION, Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS}; String selection = Constants.IndicatorQueryRepository.QUERY + " = ?"; String[] selectionArgs = {indicatorCode}; @@ -116,6 +128,9 @@ public ContentValues createContentValues(IndicatorQuery indicatorQuery) { values.put(Constants.IndicatorQueryRepository.INDICATOR_CODE, indicatorQuery.getIndicatorCode()); values.put(Constants.IndicatorQueryRepository.DB_VERSION, indicatorQuery.getDbVersion()); values.put(Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT, indicatorQuery.isMultiResult()); + values.put(Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS, + indicatorQuery.getExpectedIndicators() != null ? + new Gson().toJson(indicatorQuery.getExpectedIndicators()) : null); return values; } @@ -127,6 +142,11 @@ private IndicatorQuery processCursorRow(@NonNull Cursor cursor) { indicatorQuery.setMultiResult(cursor.getInt(cursor.getColumnIndex(Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT)) == 1); indicatorQuery.setDbVersion(cursor.getInt(cursor.getColumnIndex(Constants.IndicatorQueryRepository.DB_VERSION))); + String expectedResults = cursor.getString(cursor.getColumnIndex(Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS)); + + indicatorQuery.setExpectedIndicators(TextUtils.isEmpty(expectedResults) ? null : + (List) new Gson().fromJson(expectedResults, new TypeToken>() {}.getType())); + return indicatorQuery; } @@ -152,4 +172,15 @@ public static void aggregateDailyTallies(@NonNull SQLiteDatabase database) { // Code to migrate the code over from incremental tallies should be written here } + public static void addExpectedIndicatorColumn(@NonNull SQLiteDatabase database) { + database.execSQL("PRAGMA foreign_keys=off"); + database.beginTransaction(); + + database.execSQL("ALTER TABLE " + Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE + + " ADD COLUMN " + Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS + " TEXT"); + database.setTransactionSuccessful(); + database.endTransaction(); + database.execSQL("PRAGMA foreign_keys=on;"); + } + } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Constants.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Constants.java index da589a5d..d82a0101 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Constants.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Constants.java @@ -31,5 +31,6 @@ interface IndicatorQueryRepository { String DB_VERSION = "db_version"; String INDICATOR_QUERY_TABLE = "indicator_queries"; String INDICATOR_QUERY_IS_MULTI_RESULT = "indicator_is_multi_result"; + String INDICATOR_QUERY_EXPECTED_INDICATORS = "expected_indicators"; } } diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java index 0e0846cf..43def7ef 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java @@ -103,7 +103,7 @@ public void generateDailyIndicatorTalliesSetsLastProcessedDatePreference() throw reportEventDates.put("20190513", new SimpleDateFormat("yyyy-MM-dd hh:mm:ss", Locale.getDefault()).parse("2019-05-13 12:19:37")); Map indicatorQueries = new HashMap<>(); - indicatorQueries.put("INDI-100", new IndicatorQuery(1L, "INDI-100", "select count(*) from table", 4, false)); + indicatorQueries.put("INDI-100", new IndicatorQuery(1L, "INDI-100", "select count(*) from table", 4, false, null)); PowerMockito.mockStatic(ReportingLibrary.class); diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java new file mode 100644 index 00000000..d511205c --- /dev/null +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java @@ -0,0 +1,38 @@ +package org.smartregister.reporting.dao; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.MockitoJUnitRunner; +import org.smartregister.reporting.domain.CompositeIndicatorTally; +import org.smartregister.reporting.processor.DefaultMultiResultProcessor; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; + +/** + * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-08-13 + */ + +@RunWith(MockitoJUnitRunner.class) +public class ReportIndicatorDaoImplTest { + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void extractIndicatorTalliesShouldReturnEmptyMapIfCompositeTalliesContainsOnlyColumnName() { + ReportIndicatorDaoImpl reportIndicatorDao = new ReportIndicatorDaoImpl(); + + ArrayList tallies = new ArrayList<>(); + tallies.add(new Object[]{"gender", "counter"}); + + HashMap actualResult = reportIndicatorDao.extractIndicatorTallies(tallies, new DefaultMultiResultProcessor(), null, new CompositeIndicatorTally(9L, 5, "ISN", new Date())); + Assert.assertEquals(0, actualResult.size()); + } +} diff --git a/sample/src/main/assets/config/indicator-definitions.yml b/sample/src/main/assets/config/indicator-definitions.yml index 60ca42b7..56f2e4b6 100644 --- a/sample/src/main/assets/config/indicator-definitions.yml +++ b/sample/src/main/assets/config/indicator-definitions.yml @@ -14,4 +14,10 @@ indicators: - key: "S_IND_004" description: "A simple multi-result query" indicatorQuery: "select indicator_code, count(*) as count from indicator_queries group by indicator_code UNION ALL select 'TOTAL', count(*) from indicator_queries" - isMultiResult: true \ No newline at end of file + isMultiResult: true + + - key: "S_IND" + description: "A simple multi-result query where some results are ignored and no-results are prefilled with 0" + indicatorQuery: "select indicator_code, count(*) as count from indicator_queries group by indicator_code UNION ALL select 'TOTAL', count(*) from indicator_queries" + isMultiResult: true + expectedIndicators: ["S_IND_MALE", "S_IND_FEMALE", "S_IND_S_IND_001"] \ No newline at end of file diff --git a/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java b/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java index 93966e04..401b739c 100644 --- a/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java +++ b/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java @@ -50,9 +50,7 @@ public void onCreate(SQLiteDatabase database) { } public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) { - if (oldVersion == 1 && newVersion == 2) { - ReportingLibrary.getInstance().performMigrations(db); - } + ReportingLibrary.getInstance().performMigrations(db); } @Override From 41a38a7d37cd76f56e83ac41e58a7fe73e89ebc4 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Wed, 14 Aug 2019 11:30:36 +0300 Subject: [PATCH 02/12] Add more tests for extracting expectedIndicators from multi-result --- .../reporting/dao/ReportIndicatorDaoImpl.java | 54 ++++++++------- .../dao/ReportIndicatorDaoImplTest.java | 66 ++++++++++++++++++- .../IndicatorQueryRepositoryTest.java | 2 +- 3 files changed, 96 insertions(+), 26 deletions(-) diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java index 3ccc50e4..9b941041 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java @@ -104,25 +104,9 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { tally.setExpectedIndicators(indicatorQuery.getExpectedIndicators()); if (indicatorQuery.getExpectedIndicators() != null && indicatorQuery.getExpectedIndicators().size() > 0) { - HashMap tallies = extractIndicatorTallies(result - , ReportingLibrary.getInstance().getDefaultMultiResultProcessor() - , ReportingLibrary.getInstance().getMultiResultProcessors() - , tally); - - result = new ArrayList<>(); - - List expectedIndicators = indicatorQuery.getExpectedIndicators(); - - for (String expectedIndicator: expectedIndicators) { - Float indicatorValue = 0F; - if (tallies.get(expectedIndicator) != null) { - indicatorValue = tallies.get(expectedIndicator); - } - - result.add(new Object[]{expectedIndicator, indicatorValue}); - } - - tally.setValueSet(new Gson().toJson(result)); + ArrayList finalMultiResult = extractExpectedIndicatorsFromMultiResult(indicatorQuery, tally, result); + finalMultiResult.add(0, result.get(0)); + tally.setValueSet(new Gson().toJson(finalMultiResult)); } } @@ -153,6 +137,29 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { } } + @VisibleForTesting + public ArrayList extractExpectedIndicatorsFromMultiResult(@NonNull IndicatorQuery indicatorQuery, @NonNull CompositeIndicatorTally tally, @NonNull ArrayList tallyDecomposedValueSet) { + HashMap tallies = extractIndicatorTallies(tallyDecomposedValueSet + , ReportingLibrary.getInstance().getDefaultMultiResultProcessor() + , ReportingLibrary.getInstance().getMultiResultProcessors() + , tally); + + ArrayList finalMultiResult = new ArrayList<>(); + + List expectedIndicators = indicatorQuery.getExpectedIndicators(); + + for (String expectedIndicator: expectedIndicators) { + Float indicatorValue = 0F; + if (tallies.get(expectedIndicator) != null) { + indicatorValue = tallies.get(expectedIndicator); + } + + finalMultiResult.add(new Object[]{expectedIndicator, indicatorValue}); + } + + return finalMultiResult; + } + private LinkedHashMap getReportEventDates(String lastProcessedDate, SQLiteDatabase database) { ArrayList> values; @@ -216,6 +223,7 @@ private int executeQueryAndReturnCount(String queryString, String date, SQLiteDa return count; } + @NonNull private ArrayList executeQueryAndReturnMultiResult(@NonNull String queryString, @Nullable String date, @NonNull SQLiteDatabase database) { // Use date in querying if specified String query = ""; @@ -272,6 +280,7 @@ private ArrayList executeQueryAndReturnMultiResult(@NonNull String query return rows; } + @Nullable private Date formatDate(String date, String format) { try { return new SimpleDateFormat(format, Locale.getDefault()).parse(date); @@ -294,15 +303,16 @@ public void setIndicatorRepository(IndicatorRepository indicatorRepository) { this.indicatorRepository = indicatorRepository; } + @NonNull @VisibleForTesting - public HashMap extractIndicatorTallies(ArrayList compositeTallies, @NonNull MultiResultProcessor defaultMultiResultProcessor + public HashMap extractIndicatorTallies(ArrayList compositeTallyDecomposedValueSet, @NonNull MultiResultProcessor defaultMultiResultProcessor , @NonNull ArrayList multiResultProcessors, @NonNull CompositeIndicatorTally compositeIndicatorTally) { HashMap tallies = new HashMap<>(); List uncondensedTallies = null; - if (compositeTallies.size() > 1) { - Object[] objectFieldNames = (Object[]) compositeTallies.get(0); + if (compositeTallyDecomposedValueSet.size() > 1) { + Object[] objectFieldNames = (Object[]) compositeTallyDecomposedValueSet.get(0); String[] fieldNames = new String[objectFieldNames.length]; for (int i = 0; i < objectFieldNames.length; i++) { diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java index d511205c..95af3666 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java @@ -1,13 +1,22 @@ package org.smartregister.reporting.dao; +import com.google.gson.Gson; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.junit.MockitoJUnitRunner; +import org.robolectric.util.ReflectionHelpers; +import org.smartregister.Context; +import org.smartregister.commonregistry.CommonFtsObject; +import org.smartregister.reporting.ReportingLibrary; import org.smartregister.reporting.domain.CompositeIndicatorTally; +import org.smartregister.reporting.domain.IndicatorQuery; import org.smartregister.reporting.processor.DefaultMultiResultProcessor; +import org.smartregister.repository.Repository; import java.util.ArrayList; import java.util.Date; @@ -20,19 +29,70 @@ @RunWith(MockitoJUnitRunner.class) public class ReportIndicatorDaoImplTest { + private ReportIndicatorDaoImpl reportIndicatorDao; + @Before public void setUp() { MockitoAnnotations.initMocks(this); + reportIndicatorDao = new ReportIndicatorDaoImpl(); + + // STRONG NOTE ! This init instance works every time + ReflectionHelpers.setStaticField(ReportingLibrary.class, "instance", null); + ReportingLibrary.init(Mockito.mock(Context.class), Mockito.mock(Repository.class), Mockito.mock(CommonFtsObject.class), 9, 1); } @Test - public void extractIndicatorTalliesShouldReturnEmptyMapIfCompositeTalliesContainsOnlyColumnName() { - ReportIndicatorDaoImpl reportIndicatorDao = new ReportIndicatorDaoImpl(); - + public void extractIndicatorTalliesShouldReturnEmptyMapWhenGivenMultiResultWithOnlyColumnName() { ArrayList tallies = new ArrayList<>(); tallies.add(new Object[]{"gender", "counter"}); HashMap actualResult = reportIndicatorDao.extractIndicatorTallies(tallies, new DefaultMultiResultProcessor(), null, new CompositeIndicatorTally(9L, 5, "ISN", new Date())); Assert.assertEquals(0, actualResult.size()); } + + @Test + public void extractIndicatorTalliesShouldReturnMapWithIndicatorTalliesInResultSetWhenMultiResultContainsData() { + + ArrayList tallies = new ArrayList<>(); + tallies.add(new Object[]{"gender", "counter"}); + tallies.add(new Object[]{"male", 9F}); + tallies.add(new Object[]{"Female", 30F}); + + CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(9L, 5, "ISN", new Date()); + compositeIndicatorTally.setValueSet(new Gson().toJson(tallies)); + HashMap actualResult = reportIndicatorDao.extractIndicatorTallies(tallies, new DefaultMultiResultProcessor(), null, compositeIndicatorTally); + Assert.assertEquals(2, actualResult.size()); + Assert.assertTrue(actualResult.containsKey("ISN_male")); + Assert.assertTrue(actualResult.containsKey("ISN_Female")); + + Assert.assertEquals(9F, actualResult.get("ISN_male"), 0); + Assert.assertEquals(30F, actualResult.get("ISN_Female"), 0); + } + + @Test + public void extractExpectedIndicatorsFromMultiResultShouldReturnZeroValuesForUnlocatedIndicatorCodes() { + ArrayList expectedIndicators = new ArrayList<>(); + expectedIndicators.add("ISN_IVN"); + expectedIndicators.add("ISN_OPV"); + expectedIndicators.add("ISN_PCV"); + IndicatorQuery indicatorQuery = new IndicatorQuery(8L, "ISN", "SOme query", 1, true, expectedIndicators); + + CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(9L, 5, "ISN", new Date()); + ArrayList tallies = new ArrayList<>(); + tallies.add(new Object[]{"gender", "counter"}); + tallies.add(new Object[]{"BCG", 9F}); + tallies.add(new Object[]{"Polio", 30F}); + tallies.add(new Object[]{"IVN", 30F}); + compositeIndicatorTally.setValueSet(new Gson().toJson(tallies)); + + ArrayList finalMultiResult = reportIndicatorDao.extractExpectedIndicatorsFromMultiResult(indicatorQuery, compositeIndicatorTally, tallies); + + Assert.assertEquals(3, finalMultiResult.size()); + Assert.assertEquals(30D, (Float) ((Object[]) finalMultiResult.get(0))[1], 0F); + Assert.assertEquals(0D, (Float) ((Object[]) finalMultiResult.get(1))[1], 0F); + Assert.assertEquals(0D, (Float) ((Object[]) finalMultiResult.get(2))[1], 0F); + + Assert.assertEquals("ISN_OPV", (String) ((Object[]) finalMultiResult.get(1))[0]); + Assert.assertEquals("ISN_PCV", (String) ((Object[]) finalMultiResult.get(2))[0]); + } } diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/IndicatorQueryRepositoryTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/IndicatorQueryRepositoryTest.java index 6590915b..446e47e3 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/IndicatorQueryRepositoryTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/IndicatorQueryRepositoryTest.java @@ -61,7 +61,7 @@ public void findQueryByIndicatorCodeInvokeDbWithCorrectParams() { Mockito.verify(sqLiteDatabase, Mockito.times(1)).query( ArgumentMatchers.eq(Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) - , MockitoHamcrest.argThat(IsArrayWithSize.arrayWithSize(5)) + , MockitoHamcrest.argThat(IsArrayWithSize.arrayWithSize(6)) , ArgumentMatchers.eq(Constants.IndicatorQueryRepository.QUERY + " = ?") , MockitoHamcrest.argThat(IsArrayWithSize.arrayWithSize(1)) , ArgumentMatchers.isNull(String.class) From bf55d2761e5c3461cba9ad84b796a679b5ec3877 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Wed, 14 Aug 2019 14:44:00 +0300 Subject: [PATCH 03/12] Move expectedIndicator condition to work when reading counts from multi-result queries - Break the previous functionality where multi-result data is edited before saving - Add tests for the new functionality The current functionality checks the retrieved multi-result data and compares it with the registered expectedIndicators Fixes #32 --- .../reporting/dao/ReportIndicatorDaoImpl.java | 75 ----------- .../DailyIndicatorCountRepository.java | 126 ++++++++++++++---- .../dao/ReportIndicatorDaoImplTest.java | 98 -------------- .../DailyIndicatorCountRepositoryTest.java | 78 +++++++++-- 4 files changed, 171 insertions(+), 206 deletions(-) delete mode 100644 opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java index 9b941041..f5cf90a9 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java @@ -2,7 +2,6 @@ import android.support.annotation.NonNull; import android.support.annotation.Nullable; -import android.support.annotation.VisibleForTesting; import com.google.gson.Gson; @@ -14,7 +13,6 @@ import org.smartregister.reporting.domain.IndicatorQuery; import org.smartregister.reporting.domain.IndicatorTally; import org.smartregister.reporting.domain.ReportIndicator; -import org.smartregister.reporting.processor.MultiResultProcessor; import org.smartregister.reporting.repository.DailyIndicatorCountRepository; import org.smartregister.reporting.repository.IndicatorQueryRepository; import org.smartregister.reporting.repository.IndicatorRepository; @@ -102,13 +100,6 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { tally.setValueSetFlag(true); tally.setValueSet(new Gson().toJson(result)); tally.setExpectedIndicators(indicatorQuery.getExpectedIndicators()); - - if (indicatorQuery.getExpectedIndicators() != null && indicatorQuery.getExpectedIndicators().size() > 0) { - ArrayList finalMultiResult = extractExpectedIndicatorsFromMultiResult(indicatorQuery, tally, result); - finalMultiResult.add(0, result.get(0)); - tally.setValueSet(new Gson().toJson(finalMultiResult)); - } - } } else { count = executeQueryAndReturnCount(indicatorQuery.getQuery(), dates.getKey(), database); @@ -137,29 +128,6 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { } } - @VisibleForTesting - public ArrayList extractExpectedIndicatorsFromMultiResult(@NonNull IndicatorQuery indicatorQuery, @NonNull CompositeIndicatorTally tally, @NonNull ArrayList tallyDecomposedValueSet) { - HashMap tallies = extractIndicatorTallies(tallyDecomposedValueSet - , ReportingLibrary.getInstance().getDefaultMultiResultProcessor() - , ReportingLibrary.getInstance().getMultiResultProcessors() - , tally); - - ArrayList finalMultiResult = new ArrayList<>(); - - List expectedIndicators = indicatorQuery.getExpectedIndicators(); - - for (String expectedIndicator: expectedIndicators) { - Float indicatorValue = 0F; - if (tallies.get(expectedIndicator) != null) { - indicatorValue = tallies.get(expectedIndicator); - } - - finalMultiResult.add(new Object[]{expectedIndicator, indicatorValue}); - } - - return finalMultiResult; - } - private LinkedHashMap getReportEventDates(String lastProcessedDate, SQLiteDatabase database) { ArrayList> values; @@ -302,47 +270,4 @@ public void setDailyIndicatorCountRepository(DailyIndicatorCountRepository daily public void setIndicatorRepository(IndicatorRepository indicatorRepository) { this.indicatorRepository = indicatorRepository; } - - @NonNull - @VisibleForTesting - public HashMap extractIndicatorTallies(ArrayList compositeTallyDecomposedValueSet, @NonNull MultiResultProcessor defaultMultiResultProcessor - , @NonNull ArrayList multiResultProcessors, @NonNull CompositeIndicatorTally compositeIndicatorTally) { - - HashMap tallies = new HashMap<>(); - List uncondensedTallies = null; - - if (compositeTallyDecomposedValueSet.size() > 1) { - Object[] objectFieldNames = (Object[]) compositeTallyDecomposedValueSet.get(0); - String[] fieldNames = new String[objectFieldNames.length]; - - for (int i = 0; i < objectFieldNames.length; i++) { - fieldNames[i] = (String) objectFieldNames[i]; - } - - if (defaultMultiResultProcessor.canProcess(fieldNames.length,fieldNames)) { - uncondensedTallies = DailyIndicatorCountRepository.processMultipleTallies(defaultMultiResultProcessor, compositeIndicatorTally); - } else { - for (MultiResultProcessor multiResultProcessor: multiResultProcessors) { - if (multiResultProcessor.canProcess(fieldNames.length, fieldNames)) { - uncondensedTallies = DailyIndicatorCountRepository.processMultipleTallies(multiResultProcessor, compositeIndicatorTally); - } - } - } - } - - if (uncondensedTallies != null) { - for (IndicatorTally indicatorTally: uncondensedTallies) { - Float value = tallies.get(indicatorTally.getIndicatorCode()); - if (value != null) { - value = value + indicatorTally.getFloatCount(); - } else { - value = indicatorTally.getFloatCount(); - } - - tallies.put(indicatorTally.getIndicatorCode(), value); - } - } - - return tallies; - } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index 9c253e47..3db64736 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -3,6 +3,7 @@ import android.content.ContentValues; import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.annotation.VisibleForTesting; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -42,8 +43,9 @@ public class DailyIndicatorCountRepository extends BaseRepository { public static String CREATE_DAILY_TALLY_TABLE = "CREATE TABLE " + Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE + "(" + Constants.DailyIndicatorCountRepository.ID + " INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, " - + Constants.DailyIndicatorCountRepository.INDICATOR_CODE + " TEXT NOT NULL, " + Constants.DailyIndicatorCountRepository.INDICATOR_VALUE - + " INTEGER, " + Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET + " TEXT, " + + Constants.DailyIndicatorCountRepository.INDICATOR_CODE + " TEXT NOT NULL, " + + Constants.DailyIndicatorCountRepository.INDICATOR_VALUE + " INTEGER, " + + Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET + " TEXT, " + Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG + " BOOLEAN NOT NULL default 0, " + Constants.DailyIndicatorCountRepository.DAY + " DATETIME NOT NULL DEFAULT (DATETIME('now')))"; @@ -87,17 +89,24 @@ public List> getAllDailyTallies() { Map tallyMap; SQLiteDatabase database = getReadableDatabase(); - String[] columns = {Constants.DailyIndicatorCountRepository.ID - , Constants.DailyIndicatorCountRepository.INDICATOR_CODE - , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE - , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET - , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG - , Constants.DailyIndicatorCountRepository.DAY}; + String[] queryArgs = { + Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.ID + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.DAY + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_CODE + }; Cursor cursor = null; try { - cursor = database.query(Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE - , columns, null, null, null, null, null, null); + cursor = database.rawQuery(String.format("SELECT %s.%s, %s.%s, %s.%s, %s.%s, %s.%s, %s.%s, %s.%s FROM %s INNER JOIN %s ON %s.%s = %s.%s", queryArgs) + , null); if (cursor != null && cursor.getCount() > 0 && cursor.moveToFirst()) { MultiResultProcessor defaultMultiResultProcessor = ReportingLibrary.getInstance().getDefaultMultiResultProcessor(); ArrayList multiResultProcessors = ReportingLibrary.getInstance().getMultiResultProcessors(); @@ -161,6 +170,11 @@ private void extractIndicatorTalliesFromMultiResult(@NonNull Map 0) { + uncondensedTallies = extractOnlyRequiredIndicatorTalliesAndProvideDefault(compositeIndicatorTally, uncondensedTallies); + } + if (uncondensedTallies != null) { for (IndicatorTally indicatorTally: uncondensedTallies) { tallyMap.put(indicatorTally.getIndicatorCode(), indicatorTally); @@ -168,11 +182,60 @@ private void extractIndicatorTalliesFromMultiResult(@NonNull Map extractOnlyRequiredIndicatorTalliesAndProvideDefault( + @NonNull CompositeIndicatorTally compositeIndicatorTally, + @Nullable List uncondensedTallies) { + + List expectedIndicators = compositeIndicatorTally.getExpectedIndicators(); + + if (uncondensedTallies == null) { + List tallies = new ArrayList<>(); + + for (String expectedIndicatorCode: expectedIndicators) { + IndicatorTally indicatorTally = new IndicatorTally(); + indicatorTally.setCount(0F); + indicatorTally.setIndicatorCode(expectedIndicatorCode); + indicatorTally.setId(compositeIndicatorTally.getId()); + indicatorTally.setCreatedAt(compositeIndicatorTally.getCreatedAt()); + + tallies.add(indicatorTally); + } + + return tallies; + } else { + + List tallies = new ArrayList<>(); + HashMap indicatorTallyHashMap = new HashMap<>(); + + for (IndicatorTally indicatorTally: uncondensedTallies) { + indicatorTallyHashMap.put(indicatorTally.getIndicatorCode(), indicatorTally); + } + + for (String expectedIndicatorCode: expectedIndicators) { + if (indicatorTallyHashMap.containsKey(expectedIndicatorCode)) { + tallies.add(indicatorTallyHashMap.get(expectedIndicatorCode)); + } else { + IndicatorTally indicatorTally = new IndicatorTally(); + indicatorTally.setCount(0F); + indicatorTally.setIndicatorCode(expectedIndicatorCode); + indicatorTally.setId(compositeIndicatorTally.getId()); + indicatorTally.setCreatedAt(compositeIndicatorTally.getCreatedAt()); + + tallies.add(indicatorTally); + } + } + + return tallies; + } + } + @Nullable - public static List processMultipleTallies(@NonNull MultiResultProcessor defaultMultiResultProcessor + public static List processMultipleTallies(@NonNull MultiResultProcessor multiResultProcessor , @NonNull CompositeIndicatorTally compositeIndicatorTally) { try { - return defaultMultiResultProcessor.processMultiResultTally(compositeIndicatorTally); + return multiResultProcessor.processMultiResultTally(compositeIndicatorTally); } catch (MultiResultProcessorException ex) { Timber.e(ex); return null; @@ -183,12 +246,22 @@ public Map getDailyTallies(@NonNull Date date) { Map tallyMap = new HashMap<>(); SQLiteDatabase database = getReadableDatabase(); - String[] columns = {Constants.DailyIndicatorCountRepository.ID - , Constants.DailyIndicatorCountRepository.INDICATOR_CODE - , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE - , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET - , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG - , Constants.DailyIndicatorCountRepository.DAY}; + + String[] queryArgs = { + Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.ID + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.DAY + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_CODE + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.DAY + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.DAY + }; Cursor cursor = null; @@ -205,11 +278,10 @@ public Map getDailyTallies(@NonNull Date date) { long dayEndMillis = dayStart.getTimeInMillis(); try { - cursor = database.query(Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE - , columns - , Constants.DailyIndicatorCountRepository.DAY + " >= ? AND " + Constants.DailyIndicatorCountRepository.DAY + " < ?", - new String[]{String.valueOf(dayStartMillis), String.valueOf(dayEndMillis)} - , null,null, null, null); + cursor = database.rawQuery(String.format( + "SELECT %s.%s, %s.%s, %s.%s, %s.%s, %s.%s, %s.%s, %s.%s FROM %s INNER JOIN %s ON %s.%s = %s.%s WHERE %s.%s >= ? %s.%s AND < ?" + , queryArgs), + new String[]{String.valueOf(dayStartMillis), String.valueOf(dayEndMillis)}); if (cursor != null && cursor.getCount() > 0 && cursor.moveToFirst()) { while (!cursor.isAfterLast()) { CompositeIndicatorTally compositeIndicatorTally = processCursorRow(cursor); @@ -239,6 +311,14 @@ private CompositeIndicatorTally processCursorRow(@NonNull Cursor cursor) { compositeIndicatorTally.setCount(cursor.getInt(cursor.getColumnIndex(Constants.DailyIndicatorCountRepository.INDICATOR_VALUE))); } + if (cursor.getColumnIndex(Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS) != -1) { + compositeIndicatorTally.setExpectedIndicators( + (List) new Gson().fromJson(cursor.getString(cursor.getColumnIndex( + Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS)) + , new TypeToken>() {}.getType()) + ); + } + compositeIndicatorTally.setCreatedAt(new Date(cursor.getLong(cursor.getColumnIndex(Constants.DailyIndicatorCountRepository.DAY)))); return compositeIndicatorTally; } diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java deleted file mode 100644 index 95af3666..00000000 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.smartregister.reporting.dao; - -import com.google.gson.Gson; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.junit.MockitoJUnitRunner; -import org.robolectric.util.ReflectionHelpers; -import org.smartregister.Context; -import org.smartregister.commonregistry.CommonFtsObject; -import org.smartregister.reporting.ReportingLibrary; -import org.smartregister.reporting.domain.CompositeIndicatorTally; -import org.smartregister.reporting.domain.IndicatorQuery; -import org.smartregister.reporting.processor.DefaultMultiResultProcessor; -import org.smartregister.repository.Repository; - -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; - -/** - * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-08-13 - */ - -@RunWith(MockitoJUnitRunner.class) -public class ReportIndicatorDaoImplTest { - - private ReportIndicatorDaoImpl reportIndicatorDao; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - reportIndicatorDao = new ReportIndicatorDaoImpl(); - - // STRONG NOTE ! This init instance works every time - ReflectionHelpers.setStaticField(ReportingLibrary.class, "instance", null); - ReportingLibrary.init(Mockito.mock(Context.class), Mockito.mock(Repository.class), Mockito.mock(CommonFtsObject.class), 9, 1); - } - - @Test - public void extractIndicatorTalliesShouldReturnEmptyMapWhenGivenMultiResultWithOnlyColumnName() { - ArrayList tallies = new ArrayList<>(); - tallies.add(new Object[]{"gender", "counter"}); - - HashMap actualResult = reportIndicatorDao.extractIndicatorTallies(tallies, new DefaultMultiResultProcessor(), null, new CompositeIndicatorTally(9L, 5, "ISN", new Date())); - Assert.assertEquals(0, actualResult.size()); - } - - @Test - public void extractIndicatorTalliesShouldReturnMapWithIndicatorTalliesInResultSetWhenMultiResultContainsData() { - - ArrayList tallies = new ArrayList<>(); - tallies.add(new Object[]{"gender", "counter"}); - tallies.add(new Object[]{"male", 9F}); - tallies.add(new Object[]{"Female", 30F}); - - CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(9L, 5, "ISN", new Date()); - compositeIndicatorTally.setValueSet(new Gson().toJson(tallies)); - HashMap actualResult = reportIndicatorDao.extractIndicatorTallies(tallies, new DefaultMultiResultProcessor(), null, compositeIndicatorTally); - Assert.assertEquals(2, actualResult.size()); - Assert.assertTrue(actualResult.containsKey("ISN_male")); - Assert.assertTrue(actualResult.containsKey("ISN_Female")); - - Assert.assertEquals(9F, actualResult.get("ISN_male"), 0); - Assert.assertEquals(30F, actualResult.get("ISN_Female"), 0); - } - - @Test - public void extractExpectedIndicatorsFromMultiResultShouldReturnZeroValuesForUnlocatedIndicatorCodes() { - ArrayList expectedIndicators = new ArrayList<>(); - expectedIndicators.add("ISN_IVN"); - expectedIndicators.add("ISN_OPV"); - expectedIndicators.add("ISN_PCV"); - IndicatorQuery indicatorQuery = new IndicatorQuery(8L, "ISN", "SOme query", 1, true, expectedIndicators); - - CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(9L, 5, "ISN", new Date()); - ArrayList tallies = new ArrayList<>(); - tallies.add(new Object[]{"gender", "counter"}); - tallies.add(new Object[]{"BCG", 9F}); - tallies.add(new Object[]{"Polio", 30F}); - tallies.add(new Object[]{"IVN", 30F}); - compositeIndicatorTally.setValueSet(new Gson().toJson(tallies)); - - ArrayList finalMultiResult = reportIndicatorDao.extractExpectedIndicatorsFromMultiResult(indicatorQuery, compositeIndicatorTally, tallies); - - Assert.assertEquals(3, finalMultiResult.size()); - Assert.assertEquals(30D, (Float) ((Object[]) finalMultiResult.get(0))[1], 0F); - Assert.assertEquals(0D, (Float) ((Object[]) finalMultiResult.get(1))[1], 0F); - Assert.assertEquals(0D, (Float) ((Object[]) finalMultiResult.get(2))[1], 0F); - - Assert.assertEquals("ISN_OPV", (String) ((Object[]) finalMultiResult.get(1))[0]); - Assert.assertEquals("ISN_PCV", (String) ((Object[]) finalMultiResult.get(2))[0]); - } -} diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java index 52ce1d3f..aa9b079d 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java @@ -28,6 +28,9 @@ import java.sql.Date; import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -70,8 +73,7 @@ public void addIndicatorTallyInvokesWritableDBInsert() throws Exception { public void getAllDailyTalliesInvokesReadableDBQuery() { Mockito.when(dailyIndicatorCountRepositorySpy.getReadableDatabase()).thenReturn(sqLiteDatabase); dailyIndicatorCountRepositorySpy.getAllDailyTallies(); - Mockito.verify(sqLiteDatabase, Mockito.times(1)).query(ArgumentMatchers.anyString(), ArgumentMatchers.any(String[].class), - ArgumentMatchers.isNull(String.class), ArgumentMatchers.isNull(String[].class), ArgumentMatchers.isNull(String.class), ArgumentMatchers.isNull(String.class), ArgumentMatchers.isNull(String.class), ArgumentMatchers.isNull(String.class)); + Mockito.verify(sqLiteDatabase, Mockito.times(1)).rawQuery(ArgumentMatchers.anyString(), ArgumentMatchers.isNull(String[].class)); } @Test @@ -103,14 +105,8 @@ public void getDailyTalliesShouldInvokeQueryAndReturnEmptyHashMap() { Map tallyMap = dailyIndicatorCountRepositorySpy.getDailyTallies(new Date(System.currentTimeMillis())); Mockito.verify(sqLiteDatabase, Mockito.times(1)) - .query(ArgumentMatchers.eq(Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE) - , MockitoHamcrest.argThat(IsArrayWithSize.arrayWithSize(6)) - , ArgumentMatchers.eq(Constants.DailyIndicatorCountRepository.DAY + " >= ? AND " + Constants.DailyIndicatorCountRepository.DAY + " < ?") - , MockitoHamcrest.argThat(IsArrayWithSize.arrayWithSize(2)) - , ArgumentMatchers.isNull(String.class) - , ArgumentMatchers.isNull(String.class) - , ArgumentMatchers.isNull(String.class) - , ArgumentMatchers.isNull(String.class)); + .rawQuery(ArgumentMatchers.anyString() + , MockitoHamcrest.argThat(IsArrayWithSize.arrayWithSize(2))); Assert.assertEquals(0, tallyMap.size()); } @@ -166,4 +162,66 @@ public void processCursorRowShouldReturnValueSetWhenCursorIsValueSetRow() { Assert.assertEquals(indicatorCode, indicatorTally.getIndicatorCode()); Assert.assertEquals(id, indicatorTally.getId().longValue()); } + + @Test + public void extractOnlyRequiredIndicatorTalliesAndProvideDefaultShouldReturnZeroValuesForUnlocatedIndicatorsWhenGivenExpectedIndicatorsAndUnwantedIndicators() { + java.util.Date timeNow = Calendar.getInstance().getTime(); + + List indicatorTally = new ArrayList<>(); + indicatorTally.add(new IndicatorTally(0L, 54, "ISN_OPV", timeNow)); + indicatorTally.add(new IndicatorTally(0L, 6, "ISN_HEPB", timeNow)); + indicatorTally.add(new IndicatorTally(0L, 9, "ISN_PCV", timeNow)); + indicatorTally.add(new IndicatorTally(0L, 5, "ISN_ROTA", timeNow)); + + ArrayList expectedIndicators = new ArrayList<>(); + expectedIndicators.add("ISN_BCG"); + expectedIndicators.add("ISN_OPV"); + expectedIndicators.add("ISN_PENTA"); + + CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(); + compositeIndicatorTally.setValueSetFlag(true); + compositeIndicatorTally.setExpectedIndicators(expectedIndicators); + compositeIndicatorTally.setIndicatorCode("ISN"); + compositeIndicatorTally.setCreatedAt(timeNow); + + List finalTallies = dailyIndicatorCountRepositorySpy.extractOnlyRequiredIndicatorTalliesAndProvideDefault(compositeIndicatorTally, indicatorTally); + + Assert.assertEquals(3, finalTallies.size()); + Assert.assertEquals("ISN_BCG", finalTallies.get(0).getIndicatorCode()); + Assert.assertEquals("ISN_OPV", finalTallies.get(1).getIndicatorCode()); + Assert.assertEquals("ISN_PENTA", finalTallies.get(2).getIndicatorCode()); + + Assert.assertEquals(0, finalTallies.get(0).getCount()); + Assert.assertEquals(54, finalTallies.get(1).getCount()); + Assert.assertEquals(0, finalTallies.get(2).getCount()); + } + + @Test + public void extractOnlyRequiredIndicatorTalliesAndProvideDefaultShouldReturnZeroValuesForAllUnlocatedIndicatorsWhenGivenNull() { + java.util.Date timeNow = Calendar.getInstance().getTime(); + + ArrayList expectedIndicators = new ArrayList<>(); + expectedIndicators.add("ISN_BCG"); + expectedIndicators.add("ISN_OPV"); + expectedIndicators.add("ISN_PENTA"); + + CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(); + compositeIndicatorTally.setValueSetFlag(true); + compositeIndicatorTally.setExpectedIndicators(expectedIndicators); + compositeIndicatorTally.setIndicatorCode("ISN"); + compositeIndicatorTally.setCreatedAt(timeNow); + + List finalTallies = dailyIndicatorCountRepositorySpy.extractOnlyRequiredIndicatorTalliesAndProvideDefault(compositeIndicatorTally, null); + + Assert.assertEquals(3, finalTallies.size()); + Assert.assertEquals("ISN_BCG", finalTallies.get(0).getIndicatorCode()); + Assert.assertEquals("ISN_OPV", finalTallies.get(1).getIndicatorCode()); + Assert.assertEquals("ISN_PENTA", finalTallies.get(2).getIndicatorCode()); + + Assert.assertEquals(0, finalTallies.get(0).getCount()); + Assert.assertEquals(0, finalTallies.get(1).getCount()); + Assert.assertEquals(0, finalTallies.get(2).getCount()); + + + } } From 00c80786e3813b28edb1e2ddbd5f3e3c8cf0b594 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Wed, 14 Aug 2019 18:21:36 +0300 Subject: [PATCH 04/12] Add migration scripts to clean up incremental values 1. Check if there are multiple indicator tallies for a single day 2. Rewrite all values into the same table when the multiple indicator tallies are summed up --- .../DailyIndicatorCountRepository.java | 44 ++++++++++++++++++- .../smartregister/reporting/util/Utils.java | 35 +++++++++++++++ .../DailyIndicatorCountRepositoryTest.java | 2 - 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index 3db64736..01fb762f 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -63,7 +63,17 @@ public static void performMigrations(@NonNull SQLiteDatabase database) { && !Utils.isColumnExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET)) { addValueSetColumns(database); - aggregateDailyTallies(database); + } + + if (Utils.isTableExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE)) { + // If there are multiple indicator tallies for a single day the we need to aggregate + ArrayList results = Utils.performQuery(database, + "SELECT count(*) AS total_count FROM indicator_daily_tally GROUP BY indicator_code" + + ", strftime('%Y-%m-%d', day), indicator_is_value_set ORDER BY total_count DESC LIMIT 1"); + + if (results.size() > 0 & ((int) results.get(0)[0]) > 1) { + aggregateDailyTallies(database); + } } } @@ -371,6 +381,38 @@ public static void addValueSetColumns(@NonNull SQLiteDatabase database) { public static void aggregateDailyTallies(@NonNull SQLiteDatabase database) { // Code to migrate the code over from incremental tallies should be written here + database.execSQL("PRAGMA foreign_keys=off"); + database.beginTransaction(); + + // Read all the indicator counts & clear the table + + database.execSQL("CREATE TABLE indicator_daily_tally_old(_id INTEGER NOT NULL PRIMARY KEY, indicator_code TEXT NOT NULL, indicator_value INTEGER, indicator_value_set TEXT, indicator_is_value_set BOOLEAN NOT NULL default 0, day DATETIME NOT NULL DEFAULT (DATETIME('now')))"); + database.execSQL("INSERT INTO indicator_daily_tally_old SELECT * FROM indicator_daily_tally"); + + // Delete the old values + database.execSQL("DELETE FROM indicator_daily_tally"); + database.execSQL(String.format("INSERT INTO %s(%s, %s, %s, %s) SELECT %s, sum(%s), %s, strftime('%Y-%m-%d', %s) FROM %s GROUP BY %s, strftime('%Y-%m-%d', %s), %s" + , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE + , Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE + , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG + , Constants.DailyIndicatorCountRepository.DAY + // SELECT args START HERE + , Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE + , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG + , Constants.DailyIndicatorCountRepository.DAY + , "indicator_daily_tally_old" + // GROUP BY args START HERE + , Constants.DailyIndicatorCountRepository.INDICATOR_CODE + , Constants.DailyIndicatorCountRepository.DAY + , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET_FLAG)); + + database.execSQL("DROP TABLE indicator_daily_tally_old"); + + database.setTransactionSuccessful(); + database.endTransaction(); + database.execSQL("PRAGMA foreign_keys=on;"); } } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java index f03c1da6..5296c080 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java @@ -5,6 +5,8 @@ import net.sqlcipher.Cursor; import net.sqlcipher.database.SQLiteDatabase; +import java.util.ArrayList; + /** * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-07-09 */ @@ -74,4 +76,37 @@ public static boolean isTableExists(@NonNull SQLiteDatabase sqliteDatabase, @Non return false; } + + public static ArrayList performQuery(@NonNull SQLiteDatabase sqliteDatabase, @NonNull String query) { + ArrayList rows = new ArrayList<>(); + Cursor cursor = sqliteDatabase.rawQuery(query,null); + if (null != cursor) { + int cols = cursor.getColumnCount(); + rows.add(cursor.getColumnNames()); + while (cursor.moveToNext()) { + Object[] col = new Object[cols]; + + for (int i = 0; i < cols; i++) { + int type = cursor.getType(i); + Object cellValue = null; + + if (type == Cursor.FIELD_TYPE_FLOAT) { + cellValue = (Float) cursor.getFloat(i); + } else if (type == Cursor.FIELD_TYPE_INTEGER) { + cellValue = (Integer) cursor.getInt(i); + } else if (type == Cursor.FIELD_TYPE_STRING) { + cellValue = (String) cursor.getString(i); + } + + // Types BLOB and NULL are ignored + // Blob is not supposed to a reporting result & NULL is already defined in the cellValue at the top + col[i] = cellValue; + } + + rows.add(col); + } + } + + return rows; + } } diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java index aa9b079d..cf54a21c 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java @@ -221,7 +221,5 @@ public void extractOnlyRequiredIndicatorTalliesAndProvideDefaultShouldReturnZero Assert.assertEquals(0, finalTallies.get(0).getCount()); Assert.assertEquals(0, finalTallies.get(1).getCount()); Assert.assertEquals(0, finalTallies.get(2).getCount()); - - } } From 1b845eb70f964ec4ce330d20e646c4cd6bbdf49f Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Wed, 14 Aug 2019 19:20:58 +0300 Subject: [PATCH 05/12] Perform queries using the Y-m-d - Change date format for queries to Y-m-d from Y-m-d h:m:s - Perform queries every time the job runs and no only when a new event has been registered since the last event recorded - Save IndicatorTally date as Y-m-d for easy overwriting and queries Fixes #15 --- .../reporting/dao/ReportIndicatorDaoImpl.java | 58 +++++++++++------- .../DailyIndicatorCountRepository.java | 14 ++++- .../dao/ReportIndicatorDaoImplTest.java | 61 +++++++++++++++++++ 3 files changed, 109 insertions(+), 24 deletions(-) create mode 100644 opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java index f5cf90a9..9a350ae5 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java @@ -2,6 +2,8 @@ import android.support.annotation.NonNull; import android.support.annotation.Nullable; +import android.support.annotation.VisibleForTesting; +import android.text.TextUtils; import com.google.gson.Gson; @@ -16,12 +18,12 @@ import org.smartregister.reporting.repository.DailyIndicatorCountRepository; import org.smartregister.reporting.repository.IndicatorQueryRepository; import org.smartregister.reporting.repository.IndicatorRepository; -import org.smartregister.reporting.util.AppProperties; import org.smartregister.repository.EventClientRepository; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; @@ -39,8 +41,9 @@ */ public class ReportIndicatorDaoImpl implements ReportIndicatorDao { - public static final String REPORT_LAST_PROCESSED_DATE = "REPORT_LAST_PROCESSED_DATE"; + public static String DAILY_TALLY_DATE_FORMAT = "yyyy-MM-dd"; + public static String PREVIOUS_REPORT_DATES_QUERY = "select distinct eventDate, " + EventClientRepository.event_column.updatedAt + " from " + EventClientRepository.Table.event.name(); @@ -80,13 +83,19 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { int count; SQLiteDatabase database = ReportingLibrary.getInstance().getRepository().getWritableDatabase(); - LinkedHashMap reportEventDates = getReportEventDates(lastProcessedDate, database); + Date timeNow = Calendar.getInstance().getTime(); + LinkedHashMap reportEventDates = getReportEventDates(timeNow, lastProcessedDate, database); + Map indicatorQueries = indicatorQueryRepository.getAllIndicatorQueries(); if (!reportEventDates.isEmpty() && !indicatorQueries.isEmpty()) { - String lastUpdatedDate = ""; + String lastUpdatedDate = null; + for (Map.Entry dates : reportEventDates.entrySet()) { - lastUpdatedDate = new SimpleDateFormat(eventDateFormat, Locale.getDefault()).format(dates.getValue()); + if (dates.getValue().getTime() != timeNow.getTime()) { + lastUpdatedDate = new SimpleDateFormat(eventDateFormat, Locale.getDefault()).format(dates.getValue()); + } + for (Map.Entry entry : indicatorQueries.entrySet()) { IndicatorQuery indicatorQuery = entry.getValue(); CompositeIndicatorTally tally = null; @@ -123,12 +132,17 @@ public void generateDailyIndicatorTallies(String lastProcessedDate) { } } - ReportingLibrary.getInstance().getContext().allSharedPreferences().savePreference(REPORT_LAST_PROCESSED_DATE, lastUpdatedDate); + if (!TextUtils.isEmpty(lastUpdatedDate)) { + ReportingLibrary.getInstance().getContext().allSharedPreferences().savePreference(REPORT_LAST_PROCESSED_DATE, lastUpdatedDate); + } + Timber.i("generateDailyIndicatorTallies: Generate daily tallies complete"); } } - private LinkedHashMap getReportEventDates(String lastProcessedDate, SQLiteDatabase database) { + @VisibleForTesting + @NonNull + protected LinkedHashMap getReportEventDates(@NonNull Date timeNow, @Nullable String lastProcessedDate, @NonNull SQLiteDatabase database) { ArrayList> values; if (lastProcessedDate == null || lastProcessedDate.isEmpty()) { @@ -137,7 +151,7 @@ private LinkedHashMap getReportEventDates(String lastProcessedDate values = dailyIndicatorCountRepository.rawQuery(database, PREVIOUS_REPORT_DATES_QUERY.concat(" where " + EventClientRepository.event_column.updatedAt + " > '" + lastProcessedDate + "'" + " order by eventDate asc")); } - LinkedHashMap result = new LinkedHashMap<>(); + LinkedHashMap reportEventDates = new LinkedHashMap<>(); Date eventDate; Date updateDate; @@ -145,29 +159,31 @@ private LinkedHashMap getReportEventDates(String lastProcessedDate eventDate = formatDate(val.get(EventClientRepository.event_column.eventDate.name()), eventDateFormat); updateDate = formatDate(val.get(EventClientRepository.event_column.updatedAt.name()), eventDateFormat); - String keyDate = new SimpleDateFormat(eventDateFormat, Locale.getDefault()).format(eventDate); + String keyDate = new SimpleDateFormat(DAILY_TALLY_DATE_FORMAT, Locale.getDefault()).format(eventDate); - if (result.get(keyDate) != null && updateDate != null) { - if (result.get(keyDate).getTime() < updateDate.getTime()) { - result.put(keyDate, updateDate); + if (reportEventDates.get(keyDate) != null && updateDate != null) { + if (reportEventDates.get(keyDate).getTime() < updateDate.getTime()) { + reportEventDates.put(keyDate, updateDate); } } else { - result.put(keyDate, updateDate); + reportEventDates.put(keyDate, updateDate); } } - return result; + + String dateToday = new SimpleDateFormat(DAILY_TALLY_DATE_FORMAT, Locale.getDefault()).format(timeNow); + + if (reportEventDates.get(dateToday) == null) { + reportEventDates.put(dateToday, timeNow); + } + + return reportEventDates; } private int executeQueryAndReturnCount(String queryString, String date, SQLiteDatabase database) { // Use date in querying if specified String query = ""; if (date != null) { - if(!ReportingLibrary.getInstance().getAppProperties().hasProperty(AppProperties.KEY.COUNT_INCREMENTAL) - || ReportingLibrary.getInstance().getAppProperties().getPropertyBoolean(AppProperties.KEY.COUNT_INCREMENTAL)) { - query = queryString.contains("%s") ? queryString.replaceAll("%s", date) : queryString; - } else { - query = queryString.contains("%s") ? queryString.replaceAll("%s", date.split(" ")[0]) : queryString; - } + query = queryString.contains("%s") ? queryString.replaceAll("%s", date) : queryString; Timber.i("QUERY : %s", query); } Cursor cursor = null; @@ -197,7 +213,7 @@ private ArrayList executeQueryAndReturnMultiResult(@NonNull String query String query = ""; if (date != null) { Timber.i("QUERY : %s", queryString); - query = queryString.contains("'%s'") ? String.format(queryString, date) : queryString; + query = queryString.contains("%s") ? queryString.replaceAll("%s", date) : queryString; } Cursor cursor = null; ArrayList rows = new ArrayList<>(); diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index 01fb762f..845d54b1 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -12,6 +12,7 @@ import net.sqlcipher.database.SQLiteDatabase; import org.smartregister.reporting.ReportingLibrary; +import org.smartregister.reporting.dao.ReportIndicatorDaoImpl; import org.smartregister.reporting.domain.CompositeIndicatorTally; import org.smartregister.reporting.domain.IndicatorTally; import org.smartregister.reporting.exception.MultiResultProcessorException; @@ -89,8 +90,15 @@ public void add(@Nullable CompositeIndicatorTally indicatorTally) { SQLiteDatabase database = getWritableDatabase(); database.delete(Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE - , Constants.DailyIndicatorCountRepository.INDICATOR_CODE + " = ? AND " + Constants.DailyIndicatorCountRepository.DAY + " = ? " - , new String[]{indicatorTally.getIndicatorCode(), new SimpleDateFormat(ReportingLibrary.getInstance().getDateFormat(), Locale.getDefault()).format(indicatorTally.getCreatedAt())}); + , Constants.DailyIndicatorCountRepository.INDICATOR_CODE + " = ? AND " + + Constants.DailyIndicatorCountRepository.DAY + " = ? " + , new String[]{ + indicatorTally.getIndicatorCode(), + new SimpleDateFormat( + ReportIndicatorDaoImpl.DAILY_TALLY_DATE_FORMAT, + Locale.getDefault()).format(indicatorTally.getCreatedAt() + ) + }); database.insert(Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, null, createContentValues(indicatorTally)); } @@ -335,7 +343,7 @@ private CompositeIndicatorTally processCursorRow(@NonNull Cursor cursor) { public ContentValues createContentValues(@NonNull CompositeIndicatorTally compositeIndicatorTally) { ContentValues values = new ContentValues(); - SimpleDateFormat dateFormat = new SimpleDateFormat(ReportingLibrary.getInstance().getDateFormat(), Locale.getDefault()); + SimpleDateFormat dateFormat = new SimpleDateFormat(ReportIndicatorDaoImpl.DAILY_TALLY_DATE_FORMAT, Locale.getDefault()); values.put(Constants.DailyIndicatorCountRepository.INDICATOR_CODE, compositeIndicatorTally.getIndicatorCode()); if (compositeIndicatorTally.isValueSet()) { diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java new file mode 100644 index 00000000..9fe23b41 --- /dev/null +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java @@ -0,0 +1,61 @@ +package org.smartregister.reporting.dao; + +import net.sqlcipher.database.SQLiteDatabase; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.MockitoJUnitRunner; +import org.smartregister.reporting.repository.DailyIndicatorCountRepository; +import org.smartregister.reporting.repository.IndicatorQueryRepository; +import org.smartregister.reporting.repository.IndicatorRepository; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedHashMap; + +import static org.junit.Assert.*; + +/** + * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-08-14 + */ + +@RunWith(MockitoJUnitRunner.class) +public class ReportIndicatorDaoImplTest { + + private ReportIndicatorDaoImpl reportIndicatorDao; + + @Mock + private IndicatorQueryRepository indicatorQueryRepository; + + @Mock + private DailyIndicatorCountRepository dailyIndicatorCountRepository; + + @Mock + private IndicatorRepository indicatorRepository; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + reportIndicatorDao = new ReportIndicatorDaoImpl(indicatorQueryRepository, dailyIndicatorCountRepository, indicatorRepository); + } + + @Test + public void getReportEventDatesShouldReturnCurrentDateWhenNoDatesRetrievedFromEventTable() { + Date timeNow = Calendar.getInstance().getTime(); + SQLiteDatabase database = Mockito.mock(SQLiteDatabase.class); + + Mockito.doReturn(new ArrayList>()) + .when(dailyIndicatorCountRepository) + .rawQuery(Mockito.any(SQLiteDatabase.class), Mockito.anyString()); + + LinkedHashMap reportEventDates = reportIndicatorDao.getReportEventDates(timeNow, null, database); + + assertEquals(1, reportEventDates.size()); + } +} \ No newline at end of file From 1810d6f42396ac198cc2e9f296abf283d7614de9 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Thu, 15 Aug 2019 11:19:06 +0300 Subject: [PATCH 06/12] Fix tests --- .../repository/DailyIndicatorCountRepository.java | 4 ++-- .../java/org/smartregister/reporting/dao/DaoTest.java | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index 845d54b1..6b7e43a5 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -107,7 +107,7 @@ public List> getAllDailyTallies() { Map tallyMap; SQLiteDatabase database = getReadableDatabase(); - String[] queryArgs = { + Object[] queryArgs = { Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.ID , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_CODE , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE @@ -265,7 +265,7 @@ public Map getDailyTallies(@NonNull Date date) { SQLiteDatabase database = getReadableDatabase(); - String[] queryArgs = { + Object[] queryArgs = { Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.ID , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_CODE , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE, Constants.DailyIndicatorCountRepository.INDICATOR_VALUE diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java index 43def7ef..cb4013a4 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/DaoTest.java @@ -21,7 +21,6 @@ import org.smartregister.reporting.repository.DailyIndicatorCountRepository; import org.smartregister.reporting.repository.IndicatorQueryRepository; import org.smartregister.reporting.repository.IndicatorRepository; -import org.smartregister.reporting.util.AppProperties; import org.smartregister.repository.AllSharedPreferences; import org.smartregister.repository.Repository; @@ -62,9 +61,6 @@ public class DaoTest { private ReportIndicatorDaoImpl daoSpy; - @Mock - private AppProperties appProperties; - @Before public void setUp() { MockitoAnnotations.initMocks(this); @@ -111,12 +107,9 @@ public void generateDailyIndicatorTalliesSetsLastProcessedDatePreference() throw Mockito.when(reportingLibrary.getRepository()).thenReturn(repository); Mockito.when(reportingLibrary.getContext()).thenReturn(context); Mockito.when(repository.getWritableDatabase()).thenReturn(sqLiteDatabase); - PowerMockito.doReturn(reportEventDates).when(daoSpy, "getReportEventDates", ArgumentMatchers.anyString(), ArgumentMatchers.any(SQLiteDatabase.class)); + PowerMockito.doReturn(reportEventDates).when(daoSpy, "getReportEventDates", ArgumentMatchers.any(Date.class), ArgumentMatchers.anyString(), ArgumentMatchers.any(SQLiteDatabase.class)); Mockito.when(indicatorQueryRepository.getAllIndicatorQueries()).thenReturn(indicatorQueries); Mockito.when(context.allSharedPreferences()).thenReturn(sharedPreferences); - PowerMockito.when(reportingLibrary.getAppProperties()).thenReturn(appProperties); - PowerMockito.when(appProperties.hasProperty("reporting.incremental")).thenReturn(true); - PowerMockito.when(appProperties.getPropertyBoolean("reporting.incremental")).thenReturn(true); daoSpy.generateDailyIndicatorTallies(lastProcessedDate); Mockito.verify(sharedPreferences, Mockito.times(1)).savePreference(ArgumentMatchers.anyString(), ArgumentMatchers.anyString()); From 4835ed3baaac183a30d8f7517580cea0e1380567 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Thu, 15 Aug 2019 11:28:07 +0300 Subject: [PATCH 07/12] Disable instrumented tests on Travis --- .travis.yml | 19 -------------- .../reporting/ExampleInstrumentedTest.java | 26 ------------------- 2 files changed, 45 deletions(-) delete mode 100644 opensrp-reporting/src/androidTest/java/org/smartregister/reporting/ExampleInstrumentedTest.java diff --git a/.travis.yml b/.travis.yml index 1ebe4d21..05733cb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,14 +14,6 @@ cache: - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ -env: - matrix: - - ANDROID_TARGET=android-22 ANDROID_ABI=armeabi-v7a - global: - # wait up to 10 minutes for adb to connect to emulator - - MALLOC_ARENA_MAX=2 - - ADB_INSTALL_TIMEOUT=20 #increment timeout to 20 mins - android: components: # tools required @@ -35,17 +27,6 @@ android: - extra-google-m2repository - extra-android-m2repository - # Specify at least one system image, - # if you need to run emulator(s) during your tests - - sys-img-armeabi-v7a-android-22 - -before_script: - # Emulator Management: Create, Start and Wait - - echo no | android create avd --force -n test -t $ANDROID_TARGET --abi $ANDROID_ABI - - emulator -avd test -no-skin -no-audio -no-window & - - android-wait-for-emulator - - adb shell input keyevent 82 & - script: - echo "Travis branch is $TRAVIS_BRANCH" - echo "Travis branch is in pull request $TRAVIS_PULL+REQUEST" diff --git a/opensrp-reporting/src/androidTest/java/org/smartregister/reporting/ExampleInstrumentedTest.java b/opensrp-reporting/src/androidTest/java/org/smartregister/reporting/ExampleInstrumentedTest.java deleted file mode 100644 index 17c995de..00000000 --- a/opensrp-reporting/src/androidTest/java/org/smartregister/reporting/ExampleInstrumentedTest.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.smartregister.reporting; - -import android.content.Context; -import android.support.test.InstrumentationRegistry; -import android.support.test.runner.AndroidJUnit4; - -import org.junit.Test; -import org.junit.runner.RunWith; - -import static org.junit.Assert.assertEquals; - -/** - * Instrumented test, which will execute on an Android device. - * - * @see Testing documentation - */ -@RunWith(AndroidJUnit4.class) -public class ExampleInstrumentedTest { - @Test - public void useAppContext() { - // Context of the app under test. - Context appContext = InstrumentationRegistry.getTargetContext(); - - assertEquals("org.smartregister.reporting", appContext.getPackageName()); - } -} From 01152f7da7b794f197e1e3febb14180ca87fe01b Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Thu, 15 Aug 2019 12:58:46 +0300 Subject: [PATCH 08/12] Add tests --- .../DailyIndicatorCountRepositoryTest.java | 28 +++++++++++ .../reporting/util/UtilsTest.java | 48 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java index cf54a21c..4f38c167 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/repository/DailyIndicatorCountRepositoryTest.java @@ -23,6 +23,8 @@ import org.smartregister.reporting.ReportingLibrary; import org.smartregister.reporting.domain.CompositeIndicatorTally; import org.smartregister.reporting.domain.IndicatorTally; +import org.smartregister.reporting.processor.DefaultMultiResultProcessor; +import org.smartregister.reporting.processor.MultiResultProcessor; import org.smartregister.reporting.util.Constants; import org.smartregister.repository.Repository; @@ -30,6 +32,7 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; +import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -222,4 +225,29 @@ public void extractOnlyRequiredIndicatorTalliesAndProvideDefaultShouldReturnZero Assert.assertEquals(0, finalTallies.get(1).getCount()); Assert.assertEquals(0, finalTallies.get(2).getCount()); } + + @Test + public void extractIndicatorTalliesFromMultiResultShouldAddSingleIndicatorTalliesToMapWhenGivenCompositeIndicatorTally() { + + DefaultMultiResultProcessor defaultMultiResultProcessor = new DefaultMultiResultProcessor(); + ArrayList multiResultProcessors = new ArrayList<>(); + CompositeIndicatorTally compositeIndicatorTally = new CompositeIndicatorTally(); + compositeIndicatorTally.setIndicatorCode("ISN"); + compositeIndicatorTally.setCreatedAt(Calendar.getInstance().getTime()); + compositeIndicatorTally.setValueSetFlag(true); + compositeIndicatorTally.setValueSet("[['gender', 'count'], ['Male', 98.56], ['Female', 89]]"); + + + HashMap tallyMap = new HashMap<>(); + + ReflectionHelpers.callInstanceMethod(dailyIndicatorCountRepositorySpy, "extractIndicatorTalliesFromMultiResult" + , ReflectionHelpers.ClassParameter.from(Map.class, tallyMap) + , ReflectionHelpers.ClassParameter.from(MultiResultProcessor.class, defaultMultiResultProcessor) + , ReflectionHelpers.ClassParameter.from(ArrayList.class, multiResultProcessors) + , ReflectionHelpers.ClassParameter.from(CompositeIndicatorTally.class, compositeIndicatorTally)); + + Assert.assertEquals(2, tallyMap.size()); + Assert.assertTrue(tallyMap.containsKey("ISN_Female")); + Assert.assertTrue(tallyMap.containsKey("ISN_Male")); + } } diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java new file mode 100644 index 00000000..75b83566 --- /dev/null +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java @@ -0,0 +1,48 @@ +package org.smartregister.reporting.util; + +import net.sqlcipher.MatrixCursor; +import net.sqlcipher.database.SQLiteDatabase; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; + + +/** + * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-08-15 + */ + +@RunWith(MockitoJUnitRunner.class) +public class UtilsTest { + + @Mock + private SQLiteDatabase database; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void performQueryShouldReturnArrayListWithResultsWhenGivenValidQuery() { + MatrixCursor matrixCursor = new MatrixCursor(new String[]{"id", "first_name", "last_name", "id_number", "credit_score"}); + matrixCursor.addRow(new Object[]{1, "Adam", "Doe", "909092323", 89.56F}); + matrixCursor.addRow(new Object[]{1, "Jane", "Doe", "929300000", 3.2F}); + + Mockito.doReturn(matrixCursor) + .when(database) + .rawQuery(Mockito.anyString(), Mockito.isNull(String[].class)); + + ArrayList actualResults = Utils.performQuery(database, "SELECT * FROM persons"); + + Assert.assertEquals(3, actualResults.size()); + Assert.assertEquals(5, actualResults.get(1).length); + Assert.assertEquals(3.2F, (Float) actualResults.get(2)[4], 0); + } +} \ No newline at end of file From e8530120e21a53df20811f300ea97e36f7a34043 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Mon, 28 Oct 2019 11:11:10 +0300 Subject: [PATCH 09/12] Close cursor & fix migrations --- .../repository/DailyIndicatorCountRepository.java | 4 ++-- .../main/java/org/smartregister/reporting/util/Utils.java | 8 ++++++++ sample/build.gradle | 4 ++-- .../smartregister/sample/repository/SampleRepository.java | 4 ++-- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index f103a247..82ca31b7 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -72,7 +72,7 @@ public static void performMigrations(@NonNull SQLiteDatabase database) { "SELECT count(*) AS total_count FROM indicator_daily_tally GROUP BY indicator_code" + ", strftime('%Y-%m-%d', day), indicator_is_value_set ORDER BY total_count DESC LIMIT 1"); - if (results.size() > 0 & ((int) results.get(0)[0]) > 1) { + if (results.size() > 0 & ((int) results.get(1)[0]) > 1) { aggregateDailyTallies(database); } } @@ -399,7 +399,7 @@ public static void aggregateDailyTallies(@NonNull SQLiteDatabase database) { // Delete the old values database.execSQL("DELETE FROM indicator_daily_tally"); - database.execSQL(String.format("INSERT INTO %s(%s, %s, %s, %s) SELECT %s, sum(%s), %s, strftime('%Y-%m-%d', %s) FROM %s GROUP BY %s, strftime('%Y-%m-%d', %s), %s" + database.execSQL(String.format("INSERT INTO %s(%s, %s, %s, %s) SELECT %s, sum(%s), %s, strftime('%%Y-%%m-%%d', %s) FROM %s GROUP BY %s, strftime('%%Y-%%m-%%d', %s), %s" , Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE , Constants.DailyIndicatorCountRepository.INDICATOR_CODE , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java index 5296c080..b2926750 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java @@ -41,10 +41,13 @@ public static boolean isColumnExists(@NonNull SQLiteDatabase sqliteDatabase, String name = cursor.getString(nameColumnIndex); if (name.equals(columnToFind)) { + cursor.close(); return true; } } + cursor.close(); + return false; } @@ -70,10 +73,13 @@ public static boolean isTableExists(@NonNull SQLiteDatabase sqliteDatabase, @Non String name = cursor.getString(nameColumnIndex); if (name.equals(tableName)) { + cursor.close(); return true; } } + cursor.close(); + return false; } @@ -105,6 +111,8 @@ public static ArrayList performQuery(@NonNull SQLiteDatabase sqliteDat rows.add(col); } + + cursor.close(); } return rows; diff --git a/sample/build.gradle b/sample/build.gradle index 5b5e844b..1571086f 100644 --- a/sample/build.gradle +++ b/sample/build.gradle @@ -43,13 +43,13 @@ android { release { minifyEnabled false proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' - buildConfigField "int", "DATABASE_VERSION", '1' + buildConfigField "int", "DATABASE_VERSION", '3' buildConfigField "int", "INDICATOR_TALLY_GENERATOR_MINUTES", '2' } debug { - buildConfigField "int", "DATABASE_VERSION", '1' + buildConfigField "int", "DATABASE_VERSION", '3' buildConfigField "int", "INDICATOR_TALLY_GENERATOR_MINUTES", '2' } } diff --git a/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java b/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java index 401b739c..c3295837 100644 --- a/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java +++ b/sample/src/main/java/org/smartregister/sample/repository/SampleRepository.java @@ -17,6 +17,7 @@ import org.smartregister.reporting.repository.IndicatorRepository; import org.smartregister.repository.EventClientRepository; import org.smartregister.repository.Repository; +import org.smartregister.sample.BuildConfig; import org.smartregister.sample.utils.ChartUtil; import java.text.ParseException; @@ -33,10 +34,9 @@ public class SampleRepository extends Repository { private String databasePassword = "db_pass"; private static final String TAG = SampleRepository.class.getCanonicalName(); private Context context; - private static final int DB_VERSION = 2; public SampleRepository(Context context, org.smartregister.Context openSRPContext) { - super(context, AllConstants.DATABASE_NAME, DB_VERSION, openSRPContext.session(), null, openSRPContext.sharedRepositoriesArray()); + super(context, AllConstants.DATABASE_NAME, BuildConfig.DATABASE_VERSION, openSRPContext.session(), null, openSRPContext.sharedRepositoriesArray()); this.context = context; } From b8c43e15ed798e25d2b1facbef2e25c400b24de6 Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Mon, 28 Oct 2019 11:38:25 +0300 Subject: [PATCH 10/12] Remove unused method --- .../reporting/repository/IndicatorQueryRepository.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java index 0e185e5a..01c8cbd1 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java @@ -45,7 +45,6 @@ public static void performMigrations(@NonNull SQLiteDatabase database) { if (Utils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) && !Utils.isColumnExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT)) { addMultiResultFlagField(database); - aggregateDailyTallies(database); } if (Utils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) @@ -168,10 +167,6 @@ public static void addMultiResultFlagField(@NonNull SQLiteDatabase database) { database.execSQL("PRAGMA foreign_keys=on;"); } - public static void aggregateDailyTallies(@NonNull SQLiteDatabase database) { - // Code to migrate the code over from incremental tallies should be written here - } - public static void addExpectedIndicatorColumn(@NonNull SQLiteDatabase database) { database.execSQL("PRAGMA foreign_keys=off"); database.beginTransaction(); From 85197f261f4baf187468735a632c71ab527d76ea Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Mon, 28 Oct 2019 15:16:51 +0300 Subject: [PATCH 11/12] Code cleanup --- .../reporting/dao/ReportIndicatorDaoImpl.java | 1 - .../repository/DailyIndicatorCountRepository.java | 10 +++++----- .../reporting/repository/IndicatorQueryRepository.java | 10 +++++----- .../reporting/util/{Utils.java => ReportingUtils.java} | 4 +--- .../reporting/dao/ReportIndicatorDaoImplTest.java | 4 +--- .../org/smartregister/reporting/util/UtilsTest.java | 2 +- 6 files changed, 13 insertions(+), 18 deletions(-) rename opensrp-reporting/src/main/java/org/smartregister/reporting/util/{Utils.java => ReportingUtils.java} (98%) diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java index 84fee077..7229bdb6 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/dao/ReportIndicatorDaoImpl.java @@ -47,7 +47,6 @@ public class ReportIndicatorDaoImpl implements ReportIndicatorDao { public static String PREVIOUS_REPORT_DATES_QUERY = "select distinct eventDate, " + EventClientRepository.event_column.updatedAt + " from " + EventClientRepository.Table.event.name(); - private static String TAG = ReportIndicatorDaoImpl.class.getCanonicalName(); private static String eventDateFormat = "yyyy-MM-dd HH:mm:ss"; private IndicatorQueryRepository indicatorQueryRepository; private DailyIndicatorCountRepository dailyIndicatorCountRepository; diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java index 82ca31b7..4b35b670 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/DailyIndicatorCountRepository.java @@ -18,7 +18,7 @@ import org.smartregister.reporting.exception.MultiResultProcessorException; import org.smartregister.reporting.processor.MultiResultProcessor; import org.smartregister.reporting.util.Constants; -import org.smartregister.reporting.util.Utils; +import org.smartregister.reporting.util.ReportingUtils; import org.smartregister.repository.BaseRepository; import org.smartregister.repository.Repository; @@ -60,15 +60,15 @@ public DailyIndicatorCountRepository(Repository repository) { public static void performMigrations(@NonNull SQLiteDatabase database) { // Perform migrations - if (Utils.isTableExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE) - && !Utils.isColumnExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE + if (ReportingUtils.isTableExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE) + && !ReportingUtils.isColumnExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE , Constants.DailyIndicatorCountRepository.INDICATOR_VALUE_SET)) { addValueSetColumns(database); } - if (Utils.isTableExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE)) { + if (ReportingUtils.isTableExists(database, Constants.DailyIndicatorCountRepository.INDICATOR_DAILY_TALLY_TABLE)) { // If there are multiple indicator tallies for a single day the we need to aggregate - ArrayList results = Utils.performQuery(database, + ArrayList results = ReportingUtils.performQuery(database, "SELECT count(*) AS total_count FROM indicator_daily_tally GROUP BY indicator_code" + ", strftime('%Y-%m-%d', day), indicator_is_value_set ORDER BY total_count DESC LIMIT 1"); diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java index 01c8cbd1..b1523f65 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/repository/IndicatorQueryRepository.java @@ -13,7 +13,7 @@ import org.smartregister.reporting.domain.IndicatorQuery; import org.smartregister.reporting.util.Constants; -import org.smartregister.reporting.util.Utils; +import org.smartregister.reporting.util.ReportingUtils; import org.smartregister.repository.BaseRepository; import org.smartregister.repository.Repository; @@ -42,13 +42,13 @@ public IndicatorQueryRepository(Repository repository) { public static void performMigrations(@NonNull SQLiteDatabase database) { // Perform migrations - if (Utils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) - && !Utils.isColumnExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT)) { + if (ReportingUtils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) + && !ReportingUtils.isColumnExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE, Constants.IndicatorQueryRepository.INDICATOR_QUERY_IS_MULTI_RESULT)) { addMultiResultFlagField(database); } - if (Utils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) - && !Utils.isColumnExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE + if (ReportingUtils.isTableExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE) + && !ReportingUtils.isColumnExists(database, Constants.IndicatorQueryRepository.INDICATOR_QUERY_TABLE , Constants.IndicatorQueryRepository.INDICATOR_QUERY_EXPECTED_INDICATORS)) { addExpectedIndicatorColumn(database); } diff --git a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/ReportingUtils.java similarity index 98% rename from opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java rename to opensrp-reporting/src/main/java/org/smartregister/reporting/util/ReportingUtils.java index b2926750..9d313de1 100644 --- a/opensrp-reporting/src/main/java/org/smartregister/reporting/util/Utils.java +++ b/opensrp-reporting/src/main/java/org/smartregister/reporting/util/ReportingUtils.java @@ -11,9 +11,7 @@ * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-07-09 */ -public class Utils { - - public static final String TAG = Utils.class.getName(); +public class ReportingUtils { /** * Checks if a column exists on the table. An {@link Exception} is expected to be thrown by the sqlite diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java index c767c3b7..b386e3fc 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/dao/ReportIndicatorDaoImplTest.java @@ -26,8 +26,6 @@ import java.util.HashMap; import java.util.LinkedHashMap; -import static org.junit.Assert.*; - /** * Created by Ephraim Kigamba - ekigamba@ona.io on 2019-08-14 */ @@ -84,6 +82,6 @@ public void getReportEventDatesShouldReturnCurrentDateWhenNoDatesRetrievedFromEv LinkedHashMap reportEventDates = reportIndicatorDao.getReportEventDates(timeNow, null, database); - assertEquals(1, reportEventDates.size()); + Assert.assertEquals(1, reportEventDates.size()); } } \ No newline at end of file diff --git a/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java b/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java index 75b83566..135899f0 100644 --- a/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java +++ b/opensrp-reporting/src/test/java/org/smartregister/reporting/util/UtilsTest.java @@ -39,7 +39,7 @@ public void performQueryShouldReturnArrayListWithResultsWhenGivenValidQuery() { .when(database) .rawQuery(Mockito.anyString(), Mockito.isNull(String[].class)); - ArrayList actualResults = Utils.performQuery(database, "SELECT * FROM persons"); + ArrayList actualResults = ReportingUtils.performQuery(database, "SELECT * FROM persons"); Assert.assertEquals(3, actualResults.size()); Assert.assertEquals(5, actualResults.get(1).length); From cb2ddf938bbb9734725c14a3ddea86f1874c85ab Mon Sep 17 00:00:00 2001 From: Ephraim Kigamba Date: Mon, 28 Oct 2019 15:59:44 +0300 Subject: [PATCH 12/12] Update reporting library version to 0.0.9-SNAPSHOT --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index b1e5f078..114fb136 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=0.0.5-SNAPSHOT +VERSION_NAME=0.0.9-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Reporting Library