Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add syphilis history field to bulk uploader #7809

Merged
merged 3 commits into from
Jun 17, 2024
Merged

Add syphilis history field to bulk uploader #7809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1dbdb7
Add syphilis history field to bulk uploader
mpbrown Jun 14, 2024
bb3626f
Use string literals
mpbrown Jun 14, 2024
5d39ae3
Refactor using list of fields
mpbrown Jun 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions backend/src/main/java/gov/cdc/usds/simplereport/api/model/filerow/TestResultRow.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateEthnicity;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateGendersOfSexualPartners;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePhoneNumber;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePositiveHIVRequiredAOEFields;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateRace;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateRequiredFieldsForPositiveResult;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateResidence;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateSpecimenType;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateState;
Expand All @@ -30,6 +30,7 @@
import gov.cdc.usds.simplereport.config.FeatureFlagsConfig;
import gov.cdc.usds.simplereport.db.model.auxiliary.ResultUploadErrorSource;
import gov.cdc.usds.simplereport.db.model.auxiliary.ResultUploadErrorType;
import gov.cdc.usds.simplereport.service.DiseaseService;
import gov.cdc.usds.simplereport.service.ResultsUploaderCachingService;
import gov.cdc.usds.simplereport.service.ResultsUploaderDeviceService;
import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage;
Expand Down Expand Up @@ -105,6 +106,7 @@ public class TestResultRow implements FileRow {
final ValueOrError testResultStatus;
final ValueOrError testOrderedCode;
final ValueOrError gendersOfSexualPartners;
final ValueOrError syphilisHistory;

static final String PATIENT_LAST_NAME = "patient_last_name";
static final String PATIENT_FIRST_NAME = "patient_first_name";
Expand Down Expand Up @@ -149,6 +151,7 @@ public class TestResultRow implements FileRow {
public static final String ORDERING_FACILITY_ZIP_CODE = "ordering_facility_zip_code";
public static final String ORDERING_FACILITY_PHONE_NUMBER = "ordering_facility_phone_number";
public static final String GENDERS_OF_SEXUAL_PARTNERS = "genders_of_sexual_partners";
public static final String SYPHILIS_HISTORY = "syphilis_history";

public static final ImmutableMap<String, String> diseaseSpecificLoincMap =
new ImmutableMap.Builder<String, String>()
Expand Down Expand Up @@ -452,6 +455,7 @@ public TestResultRow(Map<String, String> rawRow) {
testOrderedCode = getValue(rawRow, "test_ordered_code", isRequired("test_ordered_code"));
gendersOfSexualPartners =
getValue(rawRow, GENDERS_OF_SEXUAL_PARTNERS, isRequired(GENDERS_OF_SEXUAL_PARTNERS));
syphilisHistory = getValue(rawRow, SYPHILIS_HISTORY, isRequired(SYPHILIS_HISTORY));
}

private List<FeedbackMessage> validateDeviceModelAndTestPerformedCode(
Expand Down Expand Up @@ -487,6 +491,17 @@ private boolean isHivResult() {
equipmentModelName.getValue(), testPerformedCode.getValue()));
}

private boolean isSyphilisResult() {
if (equipmentModelName.getValue() == null || testPerformedCode.getValue() == null) {
return false;
}
return resultsUploaderCachingService
.getSyphilisEquipmentModelAndTestPerformedCodeSet()
.contains(
ResultsUploaderCachingService.getKey(
equipmentModelName.getValue(), testPerformedCode.getValue()));
}

private List<FeedbackMessage> generateInvalidDataErrorMessages() {
String errorMessage =
"Invalid " + EQUIPMENT_MODEL_NAME + " and " + TEST_PERFORMED_CODE + " combination";
Expand Down Expand Up @@ -599,7 +614,16 @@ public List<FeedbackMessage> validateIndividualValues() {

if (isHivResult()) {
errors.addAll(
validatePositiveHIVRequiredAOEFields(testResult, gendersOfSexualPartners, pregnant));
validateRequiredFieldsForPositiveResult(
testResult, DiseaseService.HIV_NAME, List.of(gendersOfSexualPartners, pregnant)));
}

if (isSyphilisResult()) {
errors.addAll(
validateRequiredFieldsForPositiveResult(
testResult,
DiseaseService.SYPHILIS_NAME,
List.of(gendersOfSexualPartners, pregnant, syphilisHistory, symptomaticForDisease)));
}

return errors;
Expand Down
3 changes: 3 additions & 0 deletions backend/src/main/java/gov/cdc/usds/simplereport/config/CachingConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ public class CachingConfig {
"covidEquipmentModelAndTestPerformedCodeSet";
public static final String HIV_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET =
"hivEquipmentModelAndTestPerformedCodeSet";
public static final String SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET =
"syphilisEquipmentModelAndTestPerformedCodeSet";
public static final String DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP =
"deviceModelAndTestPerformedCodeMap";
public static final String SPECIMEN_NAME_TO_SNOMED_MAP = "specimenTypeNameSNOMEDMap";
Expand All @@ -26,6 +28,7 @@ public CacheManager cacheManager() {
return new ConcurrentMapCacheManager(
COVID_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET,
HIV_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET,
SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET,
DEVICE_MODEL_AND_TEST_PERFORMED_CODE_MAP,
SPECIMEN_NAME_TO_SNOMED_MAP,
SNOMED_TO_SPECIMEN_NAME_MAP,
Expand Down
1 change: 1 addition & 0 deletions backend/src/main/java/gov/cdc/usds/simplereport/service/DiseaseService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public class DiseaseService {
public static final String FLU_RNA_NAME = "Flu RNA";
public static final String RSV_NAME = "RSV";
public static final String HIV_NAME = "HIV";
public static final String SYPHILIS_NAME = "Syphilis";

private final DiseaseCacheService diseaseCacheService;

Expand Down
7 changes: 7 additions & 0 deletions backend/src/main/java/gov/cdc/usds/simplereport/service/ResultsUploaderCachingService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import static gov.cdc.usds.simplereport.config.CachingConfig.HIV_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET;
import static gov.cdc.usds.simplereport.config.CachingConfig.SNOMED_TO_SPECIMEN_NAME_MAP;
import static gov.cdc.usds.simplereport.config.CachingConfig.SPECIMEN_NAME_TO_SNOMED_MAP;
import static gov.cdc.usds.simplereport.config.CachingConfig.SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET;

import gov.cdc.usds.simplereport.db.model.DeviceType;
import gov.cdc.usds.simplereport.db.model.SpecimenType;
Expand Down Expand Up @@ -150,6 +151,12 @@ public Set<String> getHivEquipmentModelAndTestPerformedCodeSet() {
return getDiseaseSpecificEquipmentModelAndTestPerformedCodeSet(DiseaseService.HIV_NAME);
}

@Cacheable(SYPHILIS_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET)
public Set<String> getSyphilisEquipmentModelAndTestPerformedCodeSet() {
log.info("generating syphilisEquipmentModelAndTestPerformedCodeSet cache");
return getDiseaseSpecificEquipmentModelAndTestPerformedCodeSet(DiseaseService.SYPHILIS_NAME);
}

@Cacheable(COVID_EQUIPMENT_MODEL_AND_TEST_PERFORMED_CODE_SET)
public Set<String> getCovidEquipmentModelAndTestPerformedCodeSet() {
log.info("generating covidEquipmentModelAndTestPerformedCodeSet cache");
Expand Down
2 changes: 2 additions & 0 deletions backend/src/main/java/gov/cdc/usds/simplereport/service/TestResultUploadService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.ORDERING_FACILITY_STREET;
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.ORDERING_FACILITY_STREET2;
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.ORDERING_FACILITY_ZIP_CODE;
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.SYPHILIS_HISTORY;
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.TESTING_LAB_CITY;
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.TESTING_LAB_NAME;
import static gov.cdc.usds.simplereport.api.model.filerow.TestResultRow.TESTING_LAB_PHONE_NUMBER;
Expand Down Expand Up @@ -305,6 +306,7 @@ private Map<String, String> transformCsvRow(Map<String, String> row) {
row.put(DATE_RESULT_RELEASED_COLUMN_NAME, dateResultReleased.toOffsetDateTime().toString());

row.remove(GENDERS_OF_SEXUAL_PARTNERS);
row.remove(SYPHILIS_HISTORY);

return row;
}
Expand Down
66 changes: 36 additions & 30 deletions backend/src/main/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;

public class CsvValidatorUtils {

Expand Down Expand Up @@ -138,6 +139,12 @@ public class CsvValidatorUtils {
private static final String NOT_HISPANIC_LITERAL = "not hispanic or latino";
private static final String NOT_HISPANIC_CODE = "2186-5";
private static final String NOT_HISPANIC_DB_VALUE = "not_hispanic";
private static final String POSITIVE_LITERAL = "positive";
private static final String POSITIVE_CODE = "10828004";
private static final String DETECTED_LITERAL = "detected";
private static final String DETECTED_CODE = "260373001";
private static final Set<String> POSITIVE_TEST_RESULT_VALUES =
Set.of(POSITIVE_LITERAL, DETECTED_LITERAL, POSITIVE_CODE, DETECTED_CODE);
private static final Set<String> GENDER_VALUES =
Set.of(
"m", MALE_LITERAL,
Expand Down Expand Up @@ -176,7 +183,7 @@ public class CsvValidatorUtils {
"n", "no",
"u", UNKNOWN_CODE);
private static final Set<String> TEST_RESULT_VALUES =
Set.of("positive", "negative", "not detected", "detected", "invalid result");
Set.of(POSITIVE_LITERAL, "negative", "not detected", DETECTED_LITERAL, "invalid result");
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unrelated to this change but I realized we need to update the view for the device code lookup because we put "Undetermined" as the test result in the device code lookup tool but we don't support that in the bulk upload and accept "Not detected" instead.

See:

prime-simplereport/frontend/src/app/uploads/DeviceLookup/DeviceDetails.tsx

Line 18 in db562d2

[TEST_RESULTS.UNDETERMINED]: "419984006",

Device code lookup
Screenshot 2024-06-14 at 16 08 20

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice find on this! Yeah I think this is definitely worth creating a new ticket to update the device code lookup, especially given the recent issues with unmapped SNOMED code descriptions in the bulk uploader. Plus noticing that what we have listed as "Positive" is actually the SNOMED code for "Detected" whereas "Positive" SNOMED is 10828004. Seems like we're moving in the direction where it's important that certain tests distinguish the particular test result codes they use, so might be something else that needs updating too

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok! Will make a ticket for this!

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ticket created here: #7826 Feel free to add/edit anything as you see fit!


private static final Set<String> RESIDENCE_VALUES =
Set.of(
Expand Down Expand Up @@ -254,10 +261,13 @@ private static String getRequiredValueErrorMessage(String columnName) {
return "File is missing data in the " + columnName + " column.";
}

private static String getRequiredHivAoeValuesErrorMessage(String columnName) {
private static String getPositiveResultRequiredValueErrorMessage(
String columnName, String diseaseName) {
return "File is missing data in the "
+ columnName
+ " column. This is required because the row contains a positive HIV test result.";
+ " column. This is required because the row contains a positive "
+ diseaseName
+ " test result.";
}

public static List<FeedbackMessage> validateTestResult(ValueOrError input) {
Expand Down Expand Up @@ -637,39 +647,35 @@ public static List<FeedbackMessage> validateGendersOfSexualPartners(ValueOrError
return errors;
}

public static List<FeedbackMessage> validatePositiveHIVRequiredAOEFields(
ValueOrError testResult, ValueOrError gendersOfSexualPartners, ValueOrError pregnant) {
public static List<FeedbackMessage> validateRequiredFieldsForPositiveResult(
ValueOrError testResult, String diseaseName, List<ValueOrError> fields) {
List<FeedbackMessage> errors = new ArrayList<>();
// includes SNOMED values for positive and detected
Set<String> positiveTestResultValues = Set.of("positive", "detected", "260373001", "10828004");
if (!positiveTestResultValues.contains(testResult.getValue().toLowerCase())) {

if (testResult.getValue() == null) {
// if test result is null, then it should already give an error when validating required
// fields
return errors;
}

if (gendersOfSexualPartners.getValue() == null
|| gendersOfSexualPartners.getValue().isBlank()) {
errors.add(
FeedbackMessage.builder()
.scope(ITEM_SCOPE)
.fieldHeader(gendersOfSexualPartners.getHeader())
.source(ResultUploadErrorSource.SIMPLE_REPORT)
.message(getRequiredHivAoeValuesErrorMessage(gendersOfSexualPartners.getHeader()))
.errorType(ResultUploadErrorType.MISSING_DATA)
.fieldRequired(gendersOfSexualPartners.isRequired())
.build());
if (!POSITIVE_TEST_RESULT_VALUES.contains(testResult.getValue().toLowerCase())) {
return errors;
}

if (pregnant.getValue() == null || pregnant.getValue().isBlank()) {
errors.add(
FeedbackMessage.builder()
.scope(ITEM_SCOPE)
.fieldHeader(pregnant.getHeader())
.source(ResultUploadErrorSource.SIMPLE_REPORT)
.message(getRequiredHivAoeValuesErrorMessage(pregnant.getHeader()))
.errorType(ResultUploadErrorType.MISSING_DATA)
.fieldRequired(pregnant.isRequired())
.build());
}
fields.forEach(
field -> {
if (StringUtils.isBlank(field.getValue())) {
errors.add(
FeedbackMessage.builder()
.scope(ITEM_SCOPE)
.fieldHeader(field.getHeader())
.source(ResultUploadErrorSource.SIMPLE_REPORT)
.message(
getPositiveResultRequiredValueErrorMessage(field.getHeader(), diseaseName))
.errorType(ResultUploadErrorType.MISSING_DATA)
.fieldRequired(field.isRequired())
.build());
}
});
return errors;
}

Expand Down
18 changes: 14 additions & 4 deletions backend/src/test/java/gov/cdc/usds/simplereport/validators/CsvValidatorUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateGendersOfSexualPartners;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePartialUnkAddress;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePhoneNumber;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validatePositiveHIVRequiredAOEFields;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateRequiredFieldsForPositiveResult;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateSpecimenType;
import static gov.cdc.usds.simplereport.validators.CsvValidatorUtils.validateZipCode;
import static org.assertj.core.api.Assertions.assertThat;
Expand All @@ -24,6 +24,7 @@
import com.fasterxml.jackson.databind.MappingIterator;
import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
import gov.cdc.usds.simplereport.api.model.errors.CsvProcessingException;
import gov.cdc.usds.simplereport.service.DiseaseService;
import gov.cdc.usds.simplereport.service.model.reportstream.FeedbackMessage;
import gov.cdc.usds.simplereport.utils.UnknownAddressUtils;
import java.io.IOException;
Expand Down Expand Up @@ -293,22 +294,31 @@ void validNegativeHIVNoRequiredAOEFields() {
ValueOrError testResult = new ValueOrError("negative", "test_result");
ValueOrError genders = new ValueOrError("", "genders_of_sexual_partners");
ValueOrError pregnant = new ValueOrError("", "pregnant");
assertThat(validatePositiveHIVRequiredAOEFields(testResult, genders, pregnant)).isEmpty();
assertThat(
validateRequiredFieldsForPositiveResult(
testResult, DiseaseService.HIV_NAME, List.of(genders, pregnant)))
.isEmpty();
}

@Test
void validPositiveHIVRequiredAOEFields() {
ValueOrError testResult = new ValueOrError("positive", "test_result");
ValueOrError genders = new ValueOrError("m, f, tm, tw", "genders_of_sexual_partners");
ValueOrError pregnant = new ValueOrError("n", "pregnant");
assertThat(validatePositiveHIVRequiredAOEFields(testResult, genders, pregnant)).isEmpty();
assertThat(
validateRequiredFieldsForPositiveResult(
testResult, DiseaseService.HIV_NAME, List.of(genders, pregnant)))
.isEmpty();
}

@Test
void invalidPositiveHIVRequiredAOEFields() {
ValueOrError testResult = new ValueOrError("positive", "test_result");
ValueOrError genders = new ValueOrError("", "genders_of_sexual_partners");
ValueOrError pregnant = new ValueOrError("", "pregnant");
assertThat(validatePositiveHIVRequiredAOEFields(testResult, genders, pregnant)).hasSize(2);
assertThat(
validateRequiredFieldsForPositiveResult(
testResult, DiseaseService.HIV_NAME, List.of(genders, pregnant)))
.hasSize(2);
}
}
34 changes: 34 additions & 0 deletions backend/src/test/java/gov/cdc/usds/simplereport/validators/TestResultRowTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,40 @@ void validatePositiveHivRequiredAoeFields() {
.contains("This is required because the row contains a positive HIV test result."));
}

@Test
void validatePositiveSyphilisRequiredAoeFields() {
var missingSyphilisRequiredAoeFields = validRowMap;
missingSyphilisRequiredAoeFields.put("equipment_model_name", "Syphilis model");
missingSyphilisRequiredAoeFields.put("test_performed_code", "80387-4");
missingSyphilisRequiredAoeFields.put("specimen_type", "123456789");
missingSyphilisRequiredAoeFields.put("test_result", "Detected");
missingSyphilisRequiredAoeFields.put("pregnant", "");
missingSyphilisRequiredAoeFields.put("genders_of_sexual_partners", "");
missingSyphilisRequiredAoeFields.put("previous_syphilis_diagnosis", "");
missingSyphilisRequiredAoeFields.put("symptomatic_for_disease", "");

var resultsUploaderCachingService = mock(ResultsUploaderCachingService.class);
when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap())
.thenReturn(Map.of("syphilis model|80387-4", TestDataBuilder.createDeviceType()));
when(resultsUploaderCachingService.getSyphilisEquipmentModelAndTestPerformedCodeSet())
.thenReturn(Set.of("syphilis model|80387-4"));

var testResultRow =
new TestResultRow(
missingSyphilisRequiredAoeFields,
resultsUploaderCachingService,
mock(FeatureFlagsConfig.class));

var actual = testResultRow.validateIndividualValues();

assertThat(actual).hasSize(4);
actual.forEach(
message ->
assertThat(message.getMessage())
.contains(
"This is required because the row contains a positive Syphilis test result."));
}

private ResultsUploaderCachingService mockResultsUploaderCachingService() {
var resultsUploaderCachingService = mock(ResultsUploaderCachingService.class);
when(resultsUploaderCachingService.getModelAndTestPerformedCodeToDeviceMap())
Expand Down
Loading