From 199ab9155adc04dfca5209ca6cfb7947ef3f35d4 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Mon, 4 Nov 2019 15:14:32 +0100 Subject: [PATCH 01/27] Use IMSI MCC MNC in charging --- ocs-grpc-api/src/main/proto/ocs.proto | 1 + .../org/ostelco/prime/ocs/core/KtsServices.kt | 5 +- .../ostelco/prime/ocs/core/OnlineCharging.kt | 5 +- .../resources/ConsumptionPolicyService.kts | 29 +++-- .../ProtobufToDiameterConverter.java | 101 ++++++++++-------- 5 files changed, 84 insertions(+), 57 deletions(-) diff --git a/ocs-grpc-api/src/main/proto/ocs.proto b/ocs-grpc-api/src/main/proto/ocs.proto index 52b084946..b7156a54b 100644 --- a/ocs-grpc-api/src/main/proto/ocs.proto +++ b/ocs-grpc-api/src/main/proto/ocs.proto @@ -100,6 +100,7 @@ message MultipleServiceCreditControl { message PsInformation { string calledStationId = 1; string sgsnMccMnc = 2; + string imsiMccMnc = 3; } message ServiceInfo { diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/KtsServices.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/KtsServices.kt index 7f05eaa74..d2752d1dd 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/KtsServices.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/KtsServices.kt @@ -20,7 +20,8 @@ interface ConsumptionPolicy { fun checkConsumption( msisdn: String, multipleServiceCreditControl: MultipleServiceCreditControl, - mccMnc: String, - apn: String + sgsnMccMnc: String, + apn: String, + imsiMccMnc: String ): Either } diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index 8baceca2a..6375e7d7b 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -132,8 +132,9 @@ object OnlineCharging : OcsAsyncRequestConsumer { consumptionPolicy.checkConsumption( msisdn = msisdn, multipleServiceCreditControl = mscc, - mccMnc = request.serviceInformation.psInformation.sgsnMccMnc, - apn = request.serviceInformation.psInformation.calledStationId) + sgsnMccMnc = request.serviceInformation.psInformation.sgsnMccMnc, + apn = request.serviceInformation.psInformation.calledStationId, + imsiMccMnc = request.serviceInformation.psInformation.imsiMccMnc) .bimap( { consumptionResult -> consumptionResultHandler(consumptionResult) }, { consumptionRequest -> consumeRequestHandler(consumptionRequest) } diff --git a/ocs-ktc/src/main/resources/ConsumptionPolicyService.kts b/ocs-ktc/src/main/resources/ConsumptionPolicyService.kts index b0501d464..e4bbe29a5 100644 --- a/ocs-ktc/src/main/resources/ConsumptionPolicyService.kts +++ b/ocs-ktc/src/main/resources/ConsumptionPolicyService.kts @@ -16,12 +16,17 @@ object : ConsumptionPolicy { override fun checkConsumption( msisdn: String, multipleServiceCreditControl: MultipleServiceCreditControl, - mccMnc: String, - apn: String): Either { + sgsnMccMnc: String, + apn: String, + imsiMccMnc: String): Either { val requested = multipleServiceCreditControl.requested?.totalOctets ?: 0 val used = multipleServiceCreditControl.used?.totalOctets ?: 0 + if (!isMccMncAllowed(sgsnMccMnc, imsiMccMnc)) { + return blockConsumption(msisdn) + } + return when (ServiceIdRatingGroup( serviceId = multipleServiceCreditControl.serviceIdentifier, ratingGroup = multipleServiceCreditControl.ratingGroup)) { @@ -39,13 +44,19 @@ object : ConsumptionPolicy { } // BLOCKED - else -> { - ConsumptionResult( - msisdnAnalyticsId = msisdn, - granted = 0L, - balance = 0L - ).left() - } + else -> blockConsumption(msisdn) } } + + fun blockConsumption(msisdn: String) : Either { + return ConsumptionResult( + msisdnAnalyticsId = msisdn, + granted = 0L, + balance = 0L + ).left() + } + + fun isMccMncAllowed(sgsnMccMnc: String, imsiMccMnc: String) : Boolean { + return true + } } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/converter/ProtobufToDiameterConverter.java b/ocsgw/src/main/java/org/ostelco/ocsgw/converter/ProtobufToDiameterConverter.java index 8618092a3..969a77e5f 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/converter/ProtobufToDiameterConverter.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/converter/ProtobufToDiameterConverter.java @@ -4,6 +4,7 @@ import org.ostelco.diameter.model.*; import org.ostelco.ocs.api.CreditControlRequestInfo; import org.ostelco.ocs.api.CreditControlRequestType; +import org.ostelco.ocs.api.PsInformation; import org.ostelco.ocs.api.ServiceInfo; import org.ostelco.ocsgw.datasource.protobuf.GrpcDataSource; import org.slf4j.Logger; @@ -87,66 +88,78 @@ public static CreditControlRequestInfo convertRequestToProtobuf(final CreditCont builder.setRequestNumber(context.getCreditControlRequest().getCcRequestNumber().getInteger32()); - for (MultipleServiceCreditControl mscc : context.getCreditControlRequest().getMultipleServiceCreditControls()) { + addMultipleServiceCreditControl(context, builder); - org.ostelco.ocs.api.MultipleServiceCreditControl.Builder protoMscc = org.ostelco.ocs.api.MultipleServiceCreditControl.newBuilder(); + builder.setRequestId(context.getSessionId()) + .setMsisdn(context.getCreditControlRequest().getMsisdn()) + .setImsi(context.getCreditControlRequest().getImsi()); - if (!mscc.getRequested().isEmpty()) { + addPsInformation(context, builder); - ServiceUnit requested = mscc.getRequested().get(0); + return builder.build(); - protoMscc.setRequested(org.ostelco.ocs.api.ServiceUnit.newBuilder() - .setTotalOctets(requested.getTotal()) // fails at 55904 - .setInputOctets(0L) - .setOutputOctets(0L)); - } + } catch (Exception e) { + LOG.error("Failed to create CreditControlRequestInfo [{}] [{}]", context.getCreditControlRequest().getMsisdn(), context.getSessionId(), e); + } + return null; + } - for (ServiceUnit used : mscc.getUsed()) { + private static void addMultipleServiceCreditControl(final CreditControlContext context, CreditControlRequestInfo.Builder builder) { + for (MultipleServiceCreditControl mscc : context.getCreditControlRequest().getMultipleServiceCreditControls()) { - // We do not track CC-Service-Specific-Units or CC-Time - if (used.getTotal() > 0) { - protoMscc.setUsed(org.ostelco.ocs.api.ServiceUnit.newBuilder() - .setInputOctets(used.getInput()) - .setOutputOctets(used.getOutput()) - .setTotalOctets(used.getTotal())); - } - } + org.ostelco.ocs.api.MultipleServiceCreditControl.Builder protoMscc = org.ostelco.ocs.api.MultipleServiceCreditControl.newBuilder(); + + if (!mscc.getRequested().isEmpty()) { + + ServiceUnit requested = mscc.getRequested().get(0); + + protoMscc.setRequested(org.ostelco.ocs.api.ServiceUnit.newBuilder() + .setTotalOctets(requested.getTotal()) // fails at 55904 + .setInputOctets(0L) + .setOutputOctets(0L)); + } - protoMscc.setRatingGroup(mscc.getRatingGroup()); - protoMscc.setServiceIdentifier(mscc.getServiceIdentifier()); + for (ServiceUnit used : mscc.getUsed()) { - if (mscc.getReportingReason() != null) { - protoMscc.setReportingReasonValue(mscc.getReportingReason().ordinal()); - } else { - protoMscc.setReportingReasonValue(org.ostelco.ocs.api.ReportingReason.UNRECOGNIZED.ordinal()); + // We do not track CC-Service-Specific-Units or CC-Time + if (used.getTotal() > 0) { + protoMscc.setUsed(org.ostelco.ocs.api.ServiceUnit.newBuilder() + .setInputOctets(used.getInput()) + .setOutputOctets(used.getOutput()) + .setTotalOctets(used.getTotal())); } - builder.addMscc(protoMscc); } - builder.setRequestId(context.getSessionId()) - .setMsisdn(context.getCreditControlRequest().getMsisdn()) - .setImsi(context.getCreditControlRequest().getImsi()); + protoMscc.setRatingGroup(mscc.getRatingGroup()); + protoMscc.setServiceIdentifier(mscc.getServiceIdentifier()); - if (!context.getCreditControlRequest().getServiceInformation().isEmpty()) { - final PsInformation psInformation - = context.getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0); + if (mscc.getReportingReason() != null) { + protoMscc.setReportingReasonValue(mscc.getReportingReason().ordinal()); + } else { + protoMscc.setReportingReasonValue(org.ostelco.ocs.api.ReportingReason.UNRECOGNIZED.ordinal()); + } + builder.addMscc(protoMscc); + } + } - if (psInformation != null - && psInformation.getCalledStationId() != null - && psInformation.getSgsnMccMnc() != null) { + private static void addPsInformation(final CreditControlContext context, CreditControlRequestInfo.Builder builder) { + if (!context.getCreditControlRequest().getServiceInformation().isEmpty()) { + final org.ostelco.diameter.model.PsInformation psInformation + = context.getCreditControlRequest().getServiceInformation().get(0).getPsInformation().get(0); - builder.setServiceInformation( - ServiceInfo.newBuilder() - .setPsInformation(org.ostelco.ocs.api.PsInformation.newBuilder() - .setCalledStationId(psInformation.getCalledStationId()) - .setSgsnMccMnc(psInformation.getSgsnMccMnc()))); + if (psInformation != null) { + PsInformation.Builder psInformationBuilder = org.ostelco.ocs.api.PsInformation.newBuilder(); + if (psInformation.getCalledStationId() != null) { + psInformationBuilder.setCalledStationId(psInformation.getCalledStationId()); } + if (psInformation.getSgsnMccMnc() != null) { + psInformationBuilder.setSgsnMccMnc(psInformation.getSgsnMccMnc()); + } + if (psInformation.getImsiMccMnc() != null) { + psInformationBuilder.setImsiMccMnc(psInformation.getImsiMccMnc()); + } + builder.setServiceInformation(ServiceInfo.newBuilder().setPsInformation(psInformationBuilder)); } - return builder.build(); - - } catch (Exception e) { - LOG.error("Failed to create CreditControlRequestInfo [{}] [{}]", context.getCreditControlRequest().getMsisdn(), context.getSessionId(), e); } - return null; } } From 1c069a1d9ec5f8f8d246955acab13ed0d7ed9d62 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 5 Nov 2019 14:35:11 +0100 Subject: [PATCH 02/27] Use direct executor instead of own executor. --- .../analytics/publishers/DelegatePubSubPublisher.kt | 10 ++-------- .../prime/analytics/publishers/PubSubPublisher.kt | 2 -- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt index ca304fc2b..78405a18b 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt @@ -7,14 +7,13 @@ import com.google.api.gax.grpc.GrpcTransportChannel import com.google.api.gax.rpc.ApiException import com.google.api.gax.rpc.FixedTransportChannelProvider import com.google.cloud.pubsub.v1.Publisher +import com.google.common.util.concurrent.MoreExecutors import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage import io.grpc.ManagedChannelBuilder import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.analytics.events.Event import org.ostelco.prime.getLogger -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit class DelegatePubSubPublisher( @@ -24,12 +23,8 @@ class DelegatePubSubPublisher( private lateinit var publisher: Publisher private val logger by getLogger() - override lateinit var singleThreadScheduledExecutor: ScheduledExecutorService - override fun start() { - singleThreadScheduledExecutor = Executors.newSingleThreadScheduledExecutor() - val topicName = ProjectTopicName.of(projectId, topicId) val strSocketAddress = System.getenv("PUBSUB_EMULATOR_HOST") publisher = if (!strSocketAddress.isNullOrEmpty()) { @@ -49,7 +44,6 @@ class DelegatePubSubPublisher( // When finished with the publisher, shutdown to free up resources. publisher.shutdown() publisher.awaitTermination(1, TimeUnit.MINUTES) - singleThreadScheduledExecutor.shutdown() } override fun publishPubSubMessage(pubsubMessage: PubsubMessage) { @@ -74,7 +68,7 @@ class DelegatePubSubPublisher( // Once published, returns server-assigned message ids (unique within the topic) logger.debug("Published message $messageId to topic $topicId") } - }, DataConsumptionInfoPublisher.singleThreadScheduledExecutor) + }, MoreExecutors.directExecutor()) } override fun publishEvent(event: Event) { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt index f422af657..1b470d7a3 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt @@ -3,10 +3,8 @@ package org.ostelco.prime.analytics.publishers import com.google.pubsub.v1.PubsubMessage import io.dropwizard.lifecycle.Managed import org.ostelco.prime.analytics.events.Event -import java.util.concurrent.ScheduledExecutorService interface PubSubPublisher : Managed { - var singleThreadScheduledExecutor: ScheduledExecutorService fun publishPubSubMessage(pubsubMessage: PubsubMessage) fun publishEvent(event: Event) } From 13d267ac661e658d31d0c9d644cd3f0d2558f7eb Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 4 Nov 2019 15:12:59 +0100 Subject: [PATCH 03/27] Formatting --- .../prime/admin/resources/KYCResource.kt | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt b/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt index f37f1a6dd..cdf9f3928 100644 --- a/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt +++ b/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt @@ -59,10 +59,10 @@ class KYCResource { return map } - private fun toScanStatus(status: String): ScanStatus { + private fun toScanStatus(status: String): ScanStatus { return when (status) { - "SUCCESS" -> { ScanStatus.APPROVED } - else -> { ScanStatus.REJECTED } + "SUCCESS" -> ScanStatus.APPROVED + else -> ScanStatus.REJECTED } } @@ -109,24 +109,26 @@ class KYCResource { } } } - val countryCode = getCountryCodeForScan(scanId) - if (countryCode != null) { - return ScanInformation(scanId, countryCode, status, ScanResult( - vendorScanReference = vendorScanReference, - verificationStatus = verificationStatus, - time = time, - type = type, - country = country, - firstName = firstName, - lastName = lastName, - dob = dob, - rejectReason = rejectReason - )) - } else { - return null - } - } - catch (e: NullPointerException) { + return getCountryCodeForScan(scanId) + ?.let { countryCode -> + ScanInformation( + scanId, + countryCode, + status, + ScanResult( + vendorScanReference = vendorScanReference, + verificationStatus = verificationStatus, + time = time, + type = type, + country = country, + firstName = firstName, + lastName = lastName, + dob = dob, + rejectReason = rejectReason + ) + ) + } + } catch (e: NullPointerException) { logger.error("Missing mandatory fields in scan result $dataMap", e) return null } @@ -148,8 +150,8 @@ class KYCResource { } logger.info("Updating scan information ${scanInformation.scanId} jumioIdScanReference ${scanInformation.scanResult?.vendorScanReference}") return updateScanInformation(scanInformation, formData).fold( - { apiError -> Response.status(apiError.status).entity(asJson(apiError)) }, - { Response.status(Response.Status.OK).entity(asJson(scanInformation)) }).build() + { apiError -> Response.status(apiError.status).entity(asJson(apiError)) }, + { Response.status(Response.Status.OK).entity(asJson(scanInformation)) }).build() } private fun getCountryCodeForScan(scanId: String): String? { @@ -177,6 +179,7 @@ class KYCResource { Either.left(InternalServerError("Failed to update scan information", ApiErrorCode.FAILED_TO_UPDATE_SCAN_RESULTS)) } } + //TODO: Prasanth, remove this method after testing private fun dumpRequestInfo(request: HttpServletRequest, httpHeaders: HttpHeaders, formData: MultivaluedMap): String { var result = "" From c6f429d3c6e30ced94071560b75ca8cba510841a Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 4 Nov 2019 15:13:33 +0100 Subject: [PATCH 04/27] Added kyc expiry date --- .../prime/admin/resources/KYCResource.kt | 2 + .../org/ostelco/prime/model/Entities.kt | 3 ++ .../ostelco/prime/storage/graph/Neo4jStore.kt | 39 ++++++++++++++----- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt b/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt index cdf9f3928..f4babc616 100644 --- a/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt +++ b/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt @@ -88,6 +88,7 @@ class KYCResource { val firstName: String? = dataMap[JumioScanData.ID_FIRSTNAME.s] val lastName: String? = dataMap[JumioScanData.ID_LASTNAME.s] val dob: String? = dataMap[JumioScanData.ID_DOB.s] + val expiry: String? = dataMap[JumioScanData.ID_EXPIRY.s] val scanId: String = dataMap[JumioScanData.SCAN_ID.s]!! val identityVerificationData: String? = dataMap[JumioScanData.IDENTITY_VERIFICATION.s] var rejectReason: IdentityVerification? = null @@ -124,6 +125,7 @@ class KYCResource { firstName = firstName, lastName = lastName, dob = dob, + expiry = expiry, rejectReason = rejectReason ) ) diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index a1360748a..39853d77b 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -57,6 +57,7 @@ data class RegionDetails( val region: Region, val status: CustomerRegionStatus, val kycStatusMap: Map = emptyMap(), + val kycExpiryDate: String? = null, val simProfiles: Collection = emptyList()) enum class CustomerRegionStatus { @@ -132,6 +133,7 @@ data class ScanResult( val firstName: String?, val lastName: String?, val dob: String?, + val expiry: String?, val rejectReason: IdentityVerification?) data class ScanInformation( @@ -177,6 +179,7 @@ enum class JumioScanData(val s: String) { ID_FIRSTNAME("idFirstName"), ID_LASTNAME("idLastName"), ID_DOB("idDob"), + ID_EXPIRY("idExpiry"), SCAN_IMAGE("idScanImage"), SCAN_IMAGE_FACE("idScanImageFace"), SCAN_IMAGE_BACKSIDE("idScanImageBackside"), diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 33855fbb5..afa915d55 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1723,14 +1723,15 @@ object Neo4jStoreSingleton : GraphStore { override fun updateScanInformation(scanInformation: ScanInformation, vendorData: MultivaluedMap): Either = writeTransaction { logger.info("updateScanInformation : ${scanInformation.scanId} status: ${scanInformation.status}") - getCustomerUsingScanId(scanInformation.scanId).flatMap { customer -> - update { scanInformation }.flatMap { - logger.info("updating scan Information for : ${customer.contactEmail} id: ${scanInformation.scanId} status: ${scanInformation.status}") - val extendedStatus = scanInformationDatastore.getExtendedStatusInformation(scanInformation) - if (scanInformation.status == ScanStatus.APPROVED) { - - logger.info("Inserting scan Information to cloud storage : id: ${scanInformation.scanId} countryCode: ${scanInformation.countryCode}") - scanInformationDatastore.upsertVendorScanInformation(customer.id, scanInformation.countryCode, vendorData) + val updatedScanInformation = scanInformation.copy(status = verifyAndUpdateScanStatus(scanInformation)) + getCustomerUsingScanId(updatedScanInformation.scanId).flatMap { customer -> + update { updatedScanInformation }.flatMap { + logger.info("updating scan Information for : ${customer.contactEmail} id: ${updatedScanInformation.scanId} status: ${updatedScanInformation.status}") + val extendedStatus = scanInformationDatastore.getExtendedStatusInformation(updatedScanInformation) + if (updatedScanInformation.status == ScanStatus.APPROVED) { + + logger.info("Inserting scan Information to cloud storage : id: ${updatedScanInformation.scanId} countryCode: ${updatedScanInformation.countryCode}") + scanInformationDatastore.upsertVendorScanInformation(customer.id, updatedScanInformation.countryCode, vendorData) .flatMap { appNotifier.notify( notificationType = NotificationType.JUMIO_VERIFICATION_SUCCEEDED, @@ -1740,8 +1741,9 @@ object Neo4jStoreSingleton : GraphStore { logger.info(NOTIFY_OPS_MARKER, "Jumio verification succeeded for ${customer.contactEmail} Info: $extendedStatus") setKycStatus( customer = customer, - regionCode = scanInformation.countryCode.toLowerCase(), + regionCode = updatedScanInformation.countryCode.toLowerCase(), kycType = JUMIO, + kycExpiryDate = updatedScanInformation?.scanResult?.expiry, transaction = transaction) } } else { @@ -1754,7 +1756,7 @@ object Neo4jStoreSingleton : GraphStore { logger.info(NOTIFY_OPS_MARKER, "Jumio verification failed for ${customer.contactEmail} Info: $extendedStatus") setKycStatus( customer = customer, - regionCode = scanInformation.countryCode.toLowerCase(), + regionCode = updatedScanInformation.countryCode.toLowerCase(), kycType = JUMIO, kycStatus = REJECTED, transaction = transaction) @@ -1763,6 +1765,22 @@ object Neo4jStoreSingleton : GraphStore { }.ifFailedThenRollback(transaction) } + private fun verifyAndUpdateScanStatus(scanInformation: ScanInformation): ScanStatus { + + val is18yrsOfAge = scanInformation.scanResult + ?.dob + ?.let(LocalDate::parse) + ?.plusYears(18) + ?.isBefore(LocalDate.now()) + ?: true + + return if (is18yrsOfAge) { + scanInformation.status + } else { + ScanStatus.REJECTED + } + } + // // eKYC - MyInfo // @@ -1945,6 +1963,7 @@ object Neo4jStoreSingleton : GraphStore { regionCode: String, kycType: KycType, kycStatus: KycStatus = KycStatus.APPROVED, + kycExpiryDate: String? = null, transaction: Transaction): Either { return IO { From 3dcca56a598674059ecde4cc2ac7e5640da854f5 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Mon, 4 Nov 2019 15:13:58 +0100 Subject: [PATCH 05/27] Updated unit tests --- .../kotlin/org/ostelco/prime/admin/KYCResourceTest.kt | 3 ++- .../org/ostelco/prime/storage/graph/Neo4jStoreTest.kt | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/admin-endpoint/src/test/kotlin/org/ostelco/prime/admin/KYCResourceTest.kt b/admin-endpoint/src/test/kotlin/org/ostelco/prime/admin/KYCResourceTest.kt index 6347b0f0c..c846915b4 100644 --- a/admin-endpoint/src/test/kotlin/org/ostelco/prime/admin/KYCResourceTest.kt +++ b/admin-endpoint/src/test/kotlin/org/ostelco/prime/admin/KYCResourceTest.kt @@ -125,6 +125,7 @@ class KYCResourceTest { firstName = "Ole", lastName = "Nordmann", dob = "1988-01-23", + expiry = null, rejectReason = id) val scanInformation = ScanInformation( scanId = "123456", @@ -134,7 +135,7 @@ class KYCResourceTest { if (id != null) { val result = """{"scanId":"123456","countryCode":"sg","status":"REJECTED","""+ """"scanResult":{"vendorScanReference":"7890123","verificationStatus":"APPROVED_VERIFIED","time":123456,"type":"PASSPORT","country":"NORWAY","firstName":"Ole","""+ - """"lastName":"Nordmann","dob":"1988-01-23","rejectReason":{"similarity":"NO_MATCH","validity":true,"reason":null,"handwrittenNoteMatches":null}}}""" + """"lastName":"Nordmann","dob":"1988-01-23","expiry":null,"rejectReason":{"similarity":"NO_MATCH","validity":true,"reason":null,"handwrittenNoteMatches":null}}}""" Assertions.assertThat(objectMapper.writeValueAsString(scanInformation)).isEqualTo(result) } Assertions.assertThat(id).isNotNull diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt index 1861b2342..5e8238e1d 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt @@ -199,7 +199,7 @@ class Neo4jStoreTest { customer = CUSTOMER, referredBy = "blah") .fold({ assertEquals( - expected = "Failed to create REFERRED - blah -> ${CUSTOMER.id}", + expected = "Customer - blah not found.", actual = it.message) }, { fail("Created customer in spite of invalid 'referred by'") }) @@ -558,7 +558,8 @@ class Neo4jStoreTest { country = "NOR", firstName = "Test User", lastName = "Family", - dob = "1980/10/10", + dob = "1980-10-10", + expiry = null, rejectReason = null ) ) @@ -608,7 +609,8 @@ class Neo4jStoreTest { country = "NOR", firstName = "Test User", lastName = "Family", - dob = "1980/10/10", + dob = "1980-10-10", + expiry = null, rejectReason = null ) ) From b17e5ca8eeeec81330a8edad11da9e0334645006 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 5 Nov 2019 11:43:14 +0100 Subject: [PATCH 06/27] Removed deprecated MyInfo API v2 --- .circleci/prime-canary-values.yaml | 3 +- .circleci/prime-dev-values.yaml | 2 - .circleci/prime-prod-values.yaml | 2 - .../kotlin/org/ostelco/at/common/TestUser.kt | 2 +- .../kotlin/org/ostelco/at/jersey/Tests.kt | 80 ------ .../kotlin/org/ostelco/at/okhttp/Tests.kt | 72 ------ .../endpoint/resources/KycResources.kt | 31 --- .../endpoint/resources/KycResourcesTest.kt | 19 +- .../org.ostelco.prime.ekyc.MyInfoKycService | 1 + ...co.prime.paymentprocessor.PaymentProcessor | 2 +- .../org/ostelco/prime/ekyc/KycModule.kt | 15 -- .../prime/ekyc/myinfo/v2/MyInfoClient.kt | 239 ------------------ .../org.ostelco.prime.ekyc.MyInfoKycService | 1 - .../prime/ekyc/myinfo/v2/MyInfoClientTest.kt | 213 ---------------- .../prime/ekyc/myinfo/v3/MyInfoClientTest.kt | 9 +- .../ostelco/ext/myinfo/MyInfoEmulatorApp.kt | 3 - .../org/ostelco/ext/myinfo/v2/Resources.kt | 186 -------------- .../org/ostelco/prime/model/Entities.kt | 1 - .../ostelco/prime/storage/graph/Neo4jStore.kt | 2 - prime/config/config.yaml | 8 - prime/config/test.yaml | 8 - prime/infra/dev/prime-customer-api.yaml | 36 --- prime/infra/prime-direct-values.yaml | 2 - 23 files changed, 21 insertions(+), 916 deletions(-) create mode 100644 customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService delete mode 100644 ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClient.kt delete mode 100644 ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClientTest.kt delete mode 100644 ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/v2/Resources.kt diff --git a/.circleci/prime-canary-values.yaml b/.circleci/prime-canary-values.yaml index 5fc606a9c..f6ad0a7d6 100644 --- a/.circleci/prime-canary-values.yaml +++ b/.circleci/prime-canary-values.yaml @@ -40,8 +40,7 @@ prime: ACTIVATE_TOPIC_ID: ocs-activate CCR_SUBSCRIPTION_ID: ocs-ccr-sub GOOGLE_APPLICATION_CREDENTIALS: /secret/prime-service-account.json - MY_INFO_API_URI: https://myinfosg.api.gov.sg/v2 - MY_INFO_API_REALM: prod + MY_INFO_V3_API_URI: https://api.myinfo.gov.sg/com/v3 MY_INFO_REDIRECT_URI: https://dl.oya.world/links/myinfo secretVolumes: diff --git a/.circleci/prime-dev-values.yaml b/.circleci/prime-dev-values.yaml index a8f090b2c..5a595e8b8 100644 --- a/.circleci/prime-dev-values.yaml +++ b/.circleci/prime-dev-values.yaml @@ -48,9 +48,7 @@ prime: ACTIVATE_TOPIC_ID: ocs-activate CCR_SUBSCRIPTION_ID: ocs-ccr-sub GOOGLE_APPLICATION_CREDENTIALS: /secret/prime-service-account.json - MY_INFO_V2_API_URI: https://myinfosgstg.api.gov.sg/test/v2 MY_INFO_V3_API_URI: https://test.api.myinfo.gov.sg/com/v3 - MY_INFO_API_REALM: dev MY_INFO_REDIRECT_URI: https://dl-dev.oya.world/links/myinfo secretVolumes: diff --git a/.circleci/prime-prod-values.yaml b/.circleci/prime-prod-values.yaml index ac6da9416..e5d018b90 100644 --- a/.circleci/prime-prod-values.yaml +++ b/.circleci/prime-prod-values.yaml @@ -48,9 +48,7 @@ prime: ACTIVATE_TOPIC_ID: ocs-activate CCR_SUBSCRIPTION_ID: ocs-ccr-sub GOOGLE_APPLICATION_CREDENTIALS: /secret/prime-service-account.json - MY_INFO_V2_API_URI: https://myinfosg.api.gov.sg/v2 MY_INFO_V3_API_URI: https://api.myinfo.gov.sg/com/v3 - MY_INFO_API_REALM: prod MY_INFO_REDIRECT_URI: https://dl.oya.world/links/myinfo secretVolumes: diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt index 5f554678b..31b361cdd 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/TestUser.kt @@ -41,7 +41,7 @@ fun enableRegion(email: String, region: String = "no") { when (region) { "sg" -> { get { - path = "/regions/sg/kyc/myInfo/activation-code" + path = "/regions/sg/kyc/myInfo/v3/personData/activation-code" this.email = email } put(expectedResultCode = 204) { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 72cdaad53..39930fc73 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -1433,32 +1433,6 @@ class JumioKycTest { class SingaporeKycTest { - @Test - fun `jersey test - GET myinfoConfig v2`() { - - val email = "myinfo-v2-${randomInt()}@test.com" - var customerId = "" - try { - - customerId = createCustomer(name = "Test MyInfoConfig v2 Customer", email = email).id - - val myInfoConfig = get { - path = "/regions/sg/kyc/myInfoConfig" - this.email = email - } - - assertEquals( - "http://ext-myinfo-emulator:8080/v2/authorise" + - "?client_id=STG2-MYINFO-SELF-TEST" + - "&attributes=name,sex,dob,residentialstatus,nationality,mobileno,email,mailadd" + - "&redirect_uri=http://localhost:3001/callback", - myInfoConfig.url) - - } finally { - StripePayment.deleteCustomer(customerId = customerId) - } - } - @Test fun `jersey test - GET myinfoConfig v3`() { @@ -1485,60 +1459,6 @@ class SingaporeKycTest { } } - @Test - fun `jersey test - GET myinfo v2`() { - - val email = "myinfo-v2-${randomInt()}@test.com" - var customerId = "" - try { - - customerId = createCustomer(name = "Test MyInfo v2 Customer", email = email).id - - run { - val regionDetailsList = get { - path = "/regions" - this.email = email - } - regionDetailsList.forEach { - assertTrue(it.status == AVAILABLE, "All regions should be in available state") - assertTrue(it.simProfiles.isEmpty(), "All regions should have empty Sim profile list") - } - } - - val personData: String = get { - path = "/regions/sg/kyc/myInfo/authCode" - this.email = email - } - - val expectedPersonData = """{"name":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"TAN XIAO HUI"},"sex":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"F"},"nationality":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"SG"},"dob":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"1970-05-17"},"email":{"lastupdated":"2018-08-23","source":"4","classification":"C","value":"myinfotesting@gmail.com"},"mobileno":{"lastupdated":"2018-08-23","code":"65","source":"4","classification":"C","prefix":"+","nbr":"97399245"},"mailadd":{"country":"SG","unit":"128","street":"BEDOK NORTH AVENUE 4","lastupdated":"2018-03-20","block":"102","postal":"460102","source":"1","classification":"C","floor":"09","building":"PEARL GARDEN"},"uinfin":"S9812381D"}""" - assertEquals(expectedPersonData, personData, "MyInfo PersonData do not match") - - run { - val regionDetailsList = get { - path = "/regions" - this.email = email - } - - val sgRegionDetails = regionDetailsList.singleOrNull { it.region.id == "sg" } - assertTrue(sgRegionDetails != null, "regionDetailsList should contain sg region") - - val regionDetails = RegionDetails() - .region(Region().id("sg").name("Singapore")) - .status(PENDING) - .kycStatusMap(mutableMapOf( - KycType.JUMIO.name to KycStatus.PENDING, - KycType.MY_INFO.name to KycStatus.APPROVED, - KycType.ADDRESS.name to KycStatus.PENDING, - KycType.NRIC_FIN.name to KycStatus.PENDING)) - .simProfiles(SimProfileList()) - - assertEquals(regionDetails, sgRegionDetails, "RegionDetails do not match") - } - } finally { - StripePayment.deleteCustomer(customerId = customerId) - } - } - @Test fun `jersey test - GET myinfo v3`() { diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index d79a4e7d4..ed85cfbb6 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -605,31 +605,6 @@ class PurchaseTest { class SingaporeKycTest { - @Test - fun `okhttp test - GET myinfoConfig v2`() { - - val email = "myinfo-${randomInt()}@test.com" - var customerId = "" - try { - - customerId = createCustomer(name = "Test MyInfoConfig v2 Customer", email = email).id - - val client = clientForSubject(subject = email) - - val myInfoConfig = client.myInfoV2Config - - assertEquals( - "http://ext-myinfo-emulator:8080/v2/authorise" + - "?client_id=STG2-MYINFO-SELF-TEST" + - "&attributes=name,sex,dob,residentialstatus,nationality,mobileno,email,mailadd" + - "&redirect_uri=http://localhost:3001/callback", - myInfoConfig.url) - - } finally { - StripePayment.deleteCustomer(customerId = customerId) - } - } - @Test fun `okhttp test - GET myinfoConfig v3`() { @@ -655,53 +630,6 @@ class SingaporeKycTest { } } - @Test - fun `okhttp test - GET myinfo v2`() { - - val email = "myinfo-v2-${randomInt()}@test.com" - var customerId = "" - try { - - customerId = createCustomer(name = "Test MyInfo v2 Customer", email = email).id - - val client = clientForSubject(subject = email) - - run { - val regionDetailsList = client.allRegions - - regionDetailsList.forEach { - assertTrue(it.status == AVAILABLE, "All regions should be in available state") - } - } - - val personData: String = jacksonObjectMapper().writeValueAsString(client.getCustomerMyInfoV2Data("authCode")) - - val expectedPersonData = """{"name":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"TAN XIAO HUI"},"sex":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"F"},"nationality":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"SG"},"dob":{"lastupdated":"2018-03-20","source":"1","classification":"C","value":"1970-05-17"},"email":{"lastupdated":"2018-08-23","source":"4","classification":"C","value":"myinfotesting@gmail.com"},"mobileno":{"lastupdated":"2018-08-23","code":"65","source":"4","classification":"C","prefix":"+","nbr":"97399245"},"mailadd":{"country":"SG","unit":"128","street":"BEDOK NORTH AVENUE 4","lastupdated":"2018-03-20","block":"102","postal":"460102","source":"1","classification":"C","floor":"09","building":"PEARL GARDEN"},"uinfin":"S9812381D"}""" - assertEquals(expectedPersonData, personData, "MyInfo PersonData do not match") - - run { - val regionDetailsList = client.allRegions - - val sgRegionIndex = regionDetailsList.indexOfFirst { it.region.id == "sg" } - assertTrue(sgRegionIndex != -1, "regionDetailsList should contain sg region") - - val regionDetails = RegionDetails() - .region(Region().id("sg").name("Singapore")) - .status(PENDING) - .kycStatusMap(mutableMapOf( - KycType.JUMIO.name to KycStatus.PENDING, - KycType.MY_INFO.name to KycStatus.APPROVED, - KycType.ADDRESS.name to KycStatus.PENDING, - KycType.NRIC_FIN.name to KycStatus.PENDING)) - .simProfiles(SimProfileList()) - - assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") - } - } finally { - StripePayment.deleteCustomer(customerId = customerId) - } - } - @Test fun `okhttp test - GET myinfo v3`() { diff --git a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResources.kt b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResources.kt index a97959a39..915990e48 100644 --- a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResources.kt +++ b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResources.kt @@ -7,7 +7,6 @@ import org.ostelco.prime.customer.endpoint.store.SubscriberDAO import org.ostelco.prime.ekyc.MyInfoKycService import org.ostelco.prime.getLogger import org.ostelco.prime.model.MyInfoApiVersion -import org.ostelco.prime.model.MyInfoApiVersion.V2 import org.ostelco.prime.model.MyInfoApiVersion.V3 import org.ostelco.prime.module.getResource import org.ostelco.prime.tracing.EnableTracing @@ -48,36 +47,6 @@ class SingaporeKycResource(private val dao: SubscriberDAO): KycResource(regionCo private val logger by getLogger() - // MyInfo v2 - private val myInfoKycService by lazy { getResource("v2") } - - @GET - @Path("/myInfo/{authorisationCode}") - @Produces(MediaType.APPLICATION_JSON) - fun getCustomerMyInfoData(@Auth token: AccessTokenPrincipal?, - @NotNull - @PathParam("authorisationCode") - authorisationCode: String): Response = - if (token == null) { - Response.status(Response.Status.UNAUTHORIZED) - } else { - dao.getCustomerMyInfoData( - identity = token.identity, - version = V2, - authorisationCode = authorisationCode) - .responseBuilder(jsonEncode = false) - }.build() - - @GET - @Path("/myInfoConfig") - @Produces(MediaType.APPLICATION_JSON) - fun getMyInfoConfig(@Auth token: AccessTokenPrincipal?): Response = - if (token == null) { - Response.status(Response.Status.UNAUTHORIZED) - } else { - Response.status(Response.Status.OK).entity(myInfoKycService.getConfig()) - }.build() - // MyInfo v3 @Path("/myInfo/v3") diff --git a/customer-endpoint/src/test/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResourcesTest.kt b/customer-endpoint/src/test/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResourcesTest.kt index 5a914deb5..d636cde68 100644 --- a/customer-endpoint/src/test/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResourcesTest.kt +++ b/customer-endpoint/src/test/kotlin/org/ostelco/prime/customer/endpoint/resources/KycResourcesTest.kt @@ -20,15 +20,22 @@ import org.ostelco.prime.auth.AccessTokenPrincipal import org.ostelco.prime.auth.OAuthAuthenticator import org.ostelco.prime.customer.endpoint.store.SubscriberDAO import org.ostelco.prime.customer.endpoint.util.AccessToken +import org.ostelco.prime.ekyc.MyInfoKycService import org.ostelco.prime.jsonmapper.objectMapper import org.ostelco.prime.model.Identity -import org.ostelco.prime.model.MyInfoApiVersion.V2 +import org.ostelco.prime.model.MyInfoApiVersion.V3 import org.ostelco.prime.model.ScanInformation import org.ostelco.prime.model.ScanStatus import java.util.* +import javax.inject.Named import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response +private val MOCK_MY_INFO_KYC_SERVICE: MyInfoKycService = Mockito.mock(MyInfoKycService::class.java) + +@Named("v3") +class MockMyInfoKycService : MyInfoKycService by MOCK_MY_INFO_KYC_SERVICE + class KycResourcesTest { private val email = "mw@internet.org" @@ -99,10 +106,10 @@ class KycResourcesTest { val identityCaptor = argumentCaptor() val authorisationCodeCaptor = argumentCaptor() - `when`>(DAO.getCustomerMyInfoData(identityCaptor.capture(), eq(V2), authorisationCodeCaptor.capture())) + `when`>(DAO.getCustomerMyInfoData(identityCaptor.capture(), eq(V3), authorisationCodeCaptor.capture())) .thenReturn("{}".right()) - val resp = RULE.target("regions/sg/kyc/myInfo/code123") + val resp = RULE.target("regions/sg/kyc/myInfo/v3/personData/code123") .request() .header("Authorization", "Bearer ${AccessToken.withEmail(email)}") .get(Response::class.java) @@ -123,10 +130,10 @@ class KycResourcesTest { val identityCaptor = argumentCaptor() val authorisationCodeCaptor = argumentCaptor() - `when`>(DAO.getCustomerMyInfoData(identityCaptor.capture(), eq(V2), authorisationCodeCaptor.capture())) + `when`>(DAO.getCustomerMyInfoData(identityCaptor.capture(), eq(V3), authorisationCodeCaptor.capture())) .thenReturn("{}".right()) - val resp = RULE.target("regions/no/kyc/myInfo/code123") + val resp = RULE.target("regions/no/kyc/myInfo/v3/personData/code123") .request() .header("Authorization", "Bearer ${AccessToken.withEmail(email)}") .get(Response::class.java) @@ -137,7 +144,7 @@ class KycResourcesTest { @Before fun setUp() { - Mockito.`when`(AUTHENTICATOR.authenticate(ArgumentMatchers.anyString())) + `when`(AUTHENTICATOR.authenticate(ArgumentMatchers.anyString())) .thenReturn(Optional.of(AccessTokenPrincipal(Identity(email, "EMAIL","email")))) } diff --git a/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService b/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService new file mode 100644 index 000000000..4d0e4b83d --- /dev/null +++ b/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService @@ -0,0 +1 @@ +org.ostelco.prime.customer.endpoint.resources.MockMyInfoKycService \ No newline at end of file diff --git a/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.paymentprocessor.PaymentProcessor b/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.paymentprocessor.PaymentProcessor index 14b9531c5..491bc3ed7 100644 --- a/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.paymentprocessor.PaymentProcessor +++ b/customer-endpoint/src/test/resources/META-INF/services/org.ostelco.prime.paymentprocessor.PaymentProcessor @@ -1 +1 @@ -org.ostelco.prime.client.api.resources.MockPaymentProcessor +org.ostelco.prime.customer.endpoint.resources.MockPaymentProcessor diff --git a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/KycModule.kt b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/KycModule.kt index 202c76336..c778f6eed 100644 --- a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/KycModule.kt +++ b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/KycModule.kt @@ -14,7 +14,6 @@ class KycModule : PrimeModule { @JsonProperty fun setConfig(config: Config) { - ConfigRegistry.myInfoV2 = config.myInfoV2 ConfigRegistry.myInfoV3 = config.myInfoV3 } @@ -36,22 +35,9 @@ class KycModule : PrimeModule { } data class Config( - val myInfoV2: MyInfoV2Config, val myInfoV3: MyInfoV3Config ) -data class MyInfoV2Config( - val myInfoApiUri: String, - val myInfoApiClientId: String, - val myInfoApiClientSecret: String, - val myInfoApiEnableSecurity: Boolean = true, - val myInfoApiRealm: String, - val myInfoRedirectUri: String, - val myInfoServerPublicKey: String, - val myInfoClientPrivateKey: String, - val myInfoPersonDataAttributes: String = "name,sex,dob,residentialstatus,nationality,mobileno,email,mailadd" -) - data class MyInfoV3Config( val myInfoApiUri: String, val myInfoApiClientId: String, @@ -64,7 +50,6 @@ data class MyInfoV3Config( ) object ConfigRegistry { - lateinit var myInfoV2: MyInfoV2Config lateinit var myInfoV3: MyInfoV3Config } diff --git a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClient.kt b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClient.kt deleted file mode 100644 index b7e815f7e..000000000 --- a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClient.kt +++ /dev/null @@ -1,239 +0,0 @@ -package org.ostelco.prime.ekyc.myinfo.v2 - -import io.jsonwebtoken.Jwts -import org.apache.cxf.rs.security.jose.jwe.JweCompactConsumer -import org.apache.cxf.rs.security.jose.jwe.JweUtils -import org.apache.http.HttpResponse -import org.apache.http.client.methods.HttpGet -import org.apache.http.client.methods.HttpPost -import org.apache.http.entity.StringEntity -import org.ostelco.prime.ekyc.MyInfoData -import org.ostelco.prime.ekyc.MyInfoKycService -import org.ostelco.prime.ekyc.Registry.myInfoClient -import org.ostelco.prime.ekyc.myinfo.ExtendedCompressionCodecResolver -import org.ostelco.prime.ekyc.myinfo.HttpMethod -import org.ostelco.prime.ekyc.myinfo.HttpMethod.GET -import org.ostelco.prime.ekyc.myinfo.HttpMethod.POST -import org.ostelco.prime.ekyc.myinfo.TokenApiResponse -import org.ostelco.prime.getLogger -import org.ostelco.prime.jsonmapper.objectMapper -import org.ostelco.prime.model.MyInfoConfig -import java.net.URLEncoder -import java.nio.charset.StandardCharsets -import java.security.KeyFactory -import java.security.SecureRandom -import java.security.Signature -import java.security.spec.PKCS8EncodedKeySpec -import java.security.spec.X509EncodedKeySpec -import java.time.Instant -import java.util.* -import javax.inject.Named -import javax.ws.rs.core.MediaType -import kotlin.system.measureTimeMillis -import org.ostelco.prime.ekyc.ConfigRegistry.myInfoV2 as config - -@Named("v2") -class MyInfoClient : MyInfoKycService by MyInfoClientSingleton - -object MyInfoClientSingleton : MyInfoKycService { - - private val logger by getLogger() - - override fun getConfig(): MyInfoConfig = MyInfoConfig( - url = "${config.myInfoApiUri}/authorise" + - "?client_id=${config.myInfoApiClientId}" + - "&attributes=${config.myInfoPersonDataAttributes}" + - "&redirect_uri=${config.myInfoRedirectUri}") - - override fun getPersonData(authorisationCode: String): MyInfoData? { - - // Call /token API to get access_token - val tokenApiResponse = getToken(authorisationCode = authorisationCode) - ?.let { content -> - objectMapper.readValue(content, TokenApiResponse::class.java) - } - ?: return null - - // extract uin_fin out of "subject" of claims of access_token - val claims = getClaims(tokenApiResponse.accessToken) - val uinFin = claims.body.subject - - // Using access_token and uin_fin, call /person API to get Person Data - val personData = getPersonData( - uinFin = uinFin, - accessToken = tokenApiResponse.accessToken) - - return MyInfoData(uinFin = uinFin, personData = personData) - } - - private fun getToken(authorisationCode: String): String? = - sendSignedRequest( - httpMethod = POST, - path = "/token", - queryParams = mapOf( - "grant_type" to "authorization_code", - "code" to authorisationCode, - "redirect_uri" to config.myInfoRedirectUri, - "client_id" to config.myInfoApiClientId, - "client_secret" to config.myInfoApiClientSecret)) - - - private fun getClaims(accessToken: String) = Jwts.parser() - .setCompressionCodecResolver(ExtendedCompressionCodecResolver) - .setSigningKey(KeyFactory - .getInstance("RSA") - .generatePublic(X509EncodedKeySpec(Base64 - .getDecoder() - .decode(config.myInfoServerPublicKey)))) - .parseClaimsJws(accessToken) - - - private fun getPersonData(uinFin: String, accessToken: String): String? = - sendSignedRequest( - httpMethod = GET, - path = "/person/$uinFin", - queryParams = mapOf( - "client_id" to config.myInfoApiClientId, - "attributes" to config.myInfoPersonDataAttributes), - accessToken = accessToken) - - /** - * Ref: https://www.ndi-api.gov.sg/library/trusted-data/myinfo/tutorial3 - */ - private fun sendSignedRequest( - httpMethod: HttpMethod, - path: String, - queryParams: Map, - accessToken: String? = null): String? { - - val queryParamsString = queryParams.entries.joinToString("&") { """${it.key}=${URLEncoder.encode(it.value, StandardCharsets.US_ASCII)}""" } - - val requestUrl = "${config.myInfoApiUri}$path" - - // Create HTTP request - val request = when (httpMethod) { - GET -> HttpGet("$requestUrl?$queryParamsString") - POST -> HttpPost(requestUrl).also { - it.entity = StringEntity(queryParamsString) - } - } - - if (config.myInfoApiEnableSecurity) { - - val nonce = SecureRandom.getInstance("SHA1PRNG").nextLong() - val timestamp = Instant.now().toEpochMilli() - - // A) Construct the Authorisation Token Parameter - val defaultAuthHeaders = mapOf( - "apex_l2_eg_timestamp" to "$timestamp", - "apex_l2_eg_nonce" to "$nonce", - "apex_l2_eg_app_id" to config.myInfoApiClientId, - "apex_l2_eg_signature_method" to "SHA256withRSA", - "apex_l2_eg_version" to "1.0") - - // B) Forming the Base String - // Base String is a representation of the entire request (ensures message integrity) - - val baseStringParams = defaultAuthHeaders + queryParams - - // i) Normalize request parameters - val baseParamString = baseStringParams.entries - .sortedBy { it.key } - .joinToString("&") { "${it.key}=${it.value}" } - - // ii) construct request URL ---> url is passed in to this function - // NOTE: need to include the ".e." in order for the security authorisation header to work - //myinfosgstg.api.gov.sg -> myinfosgstg.e.api.gov.sg - - val url = "${config.myInfoApiUri.toLowerCase().replace(".api.gov.sg", ".e.api.gov.sg")}$path" - - // iii) concatenate request elements (HTTP method + url + base string parameters) - val baseString = "$httpMethod&$url&$baseParamString" - - // C) Signing Base String to get Digital Signature - // Load pem file containing the x509 cert & private key & sign the base string with it to produce the Digital Signature - val signature = Signature.getInstance("SHA256withRSA") - .also { sign -> - sign.initSign(KeyFactory - .getInstance("RSA") - .generatePrivate(PKCS8EncodedKeySpec( - Base64.getDecoder().decode(config.myInfoClientPrivateKey)))) - } - .also { sign -> sign.update(baseString.toByteArray()) } - .let(Signature::sign) - .let(Base64.getEncoder()::encodeToString) - - // D) Assembling the Authorization Header - - val authHeaders = mapOf("realm" to config.myInfoApiRealm) + - defaultAuthHeaders + - mapOf("apex_l2_eg_signature" to signature) - - var authHeaderString = "apex_l2_eg " + - authHeaders.entries - .joinToString(",") { """${it.key}="${it.value}"""" } - - if (accessToken != null) { - authHeaderString = "$authHeaderString,Bearer $accessToken" - } - - request.addHeader("Authorization", authHeaderString) - - } else if (accessToken != null) { - request.addHeader("Authorization", "Bearer $accessToken") - } - - request.addHeader("Cache-Control", "no-cache") - request.addHeader("Accept", MediaType.APPLICATION_JSON) - - if (httpMethod == POST) { - request.addHeader("Content-Type", MediaType.APPLICATION_FORM_URLENCODED) - } - - var response: HttpResponse? = null - - val latency = measureTimeMillis { - response = myInfoClient.execute(request) - } - - logger.info("Latency is $latency ms for MyInfo $httpMethod") - - val statusCode = response?.statusLine?.statusCode - if (statusCode != 200) { - logger.info("response: $httpMethod status: ${response?.statusLine}") - } - - val content = response - ?.entity - ?.content - ?.readAllBytes() - ?.let { String(it) } - - if (content == null || statusCode != 200) { - logger.info("$httpMethod Response content: $content") - return null - } - - if (config.myInfoApiEnableSecurity && httpMethod == GET) { - return decodeJweCompact(content) - } - - return content - } - - private fun decodeJweCompact(jwePayload: String): String { - - val privateKey = KeyFactory - .getInstance("RSA") - .generatePrivate(PKCS8EncodedKeySpec( - Base64.getDecoder().decode(config.myInfoClientPrivateKey))) - - val jweHeaders = JweCompactConsumer(jwePayload).jweHeaders - - return String(JweUtils.decrypt( - privateKey, - jweHeaders.keyEncryptionAlgorithm, - jweHeaders.contentEncryptionAlgorithm, - jwePayload)) - } -} diff --git a/ekyc/src/main/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService b/ekyc/src/main/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService index 7682c967a..29ce9e4ab 100644 --- a/ekyc/src/main/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService +++ b/ekyc/src/main/resources/META-INF/services/org.ostelco.prime.ekyc.MyInfoKycService @@ -1,2 +1 @@ -org.ostelco.prime.ekyc.myinfo.v2.MyInfoClient org.ostelco.prime.ekyc.myinfo.v3.MyInfoClient \ No newline at end of file diff --git a/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClientTest.kt b/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClientTest.kt deleted file mode 100644 index ce59f8b33..000000000 --- a/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v2/MyInfoClientTest.kt +++ /dev/null @@ -1,213 +0,0 @@ -package org.ostelco.prime.ekyc.myinfo.v2 - -import io.dropwizard.testing.ConfigOverride -import io.dropwizard.testing.DropwizardTestSupport -import io.dropwizard.testing.ResourceHelpers -import org.apache.http.impl.client.HttpClientBuilder -import org.junit.AfterClass -import org.junit.Assert.assertArrayEquals -import org.junit.Assert.assertEquals -import org.junit.Before -import org.junit.BeforeClass -import org.junit.Test -import org.ostelco.ext.myinfo.MyInfoEmulatorApp -import org.ostelco.ext.myinfo.MyInfoEmulatorConfig -import org.ostelco.prime.ekyc.ConfigRegistry -import org.ostelco.prime.ekyc.MyInfoV2Config -import org.ostelco.prime.ekyc.Registry -import org.ostelco.prime.getLogger -import java.io.File -import java.io.FileInputStream -import java.security.KeyFactory -import java.security.KeyPair -import java.security.KeyPairGenerator -import java.security.PrivateKey -import java.security.cert.CertificateFactory -import java.security.spec.PKCS8EncodedKeySpec -import java.security.spec.X509EncodedKeySpec -import java.util.* -import kotlin.system.measureTimeMillis - -/** - * Ref: https://www.ndi-api.gov.sg/assets/lib/trusted-data/myinfo/specs/myinfo-kyc-v2.1.1.yaml.html#section/Environments - * - * https://{ENV_DOMAIN_NAME}/{VERSION}/{RESOURCE} - * - * ENV_DOMAIN_NAME: - * - Sandbox/Dev: https://myinfosgstg.api.gov.sg/dev/ - * - Staging: https://myinfosgstg.api.gov.sg/test/ - * - Production: https://myinfosg.api.gov.sg/ - * - * VERSION: `/v2` - */ -// Using certs from https://github.com/jamesleegovtech/myinfo-demo-app/tree/master/ssl -private val templateTestConfig = String(File("src/test/resources/stg-demoapp-client-privatekey-2018.pem").readBytes()) - .replace("\n","") - .removePrefix("-----BEGIN PRIVATE KEY-----") - .removeSuffix("-----END PRIVATE KEY-----") - .let { base64Encoded -> PKCS8EncodedKeySpec(Base64.getDecoder().decode(base64Encoded)) } - .let { keySpec -> KeyFactory.getInstance("RSA").generatePrivate(keySpec) } - .let { clientPrivateKey: PrivateKey -> - MyInfoV2Config( - myInfoApiUri = "https://myinfosgstg.api.gov.sg/test/v2", - myInfoApiClientId = "STG2-MYINFO-SELF-TEST", - myInfoApiClientSecret = "44d953c796cccebcec9bdc826852857ab412fbe2", - myInfoRedirectUri = "http://localhost:3001/callback", - myInfoApiRealm = "http://localhost:3001", - myInfoPersonDataAttributes = "name,sex,race,nationality,dob,email,mobileno,regadd,housingtype,hdbtype,marital,edulevel,ownerprivate,cpfcontributions,cpfbalances", - myInfoServerPublicKey = "", - myInfoClientPrivateKey = Base64.getEncoder().encodeToString(clientPrivateKey.encoded) - ) - } - -class MyInfoClientTest { - - private val logger by getLogger() - - @Before - fun setupUnitTest() { - ConfigRegistry.myInfoV2 = templateTestConfig.copy( - myInfoApiUri = "http://localhost:8080", - myInfoServerPublicKey = Base64.getEncoder().encodeToString(myInfoServerKeyPair.public.encoded)) - - Registry.myInfoClient = HttpClientBuilder.create().build() - } - - /** - * This test to send request to real staging server of MyInfo API. - * - * Some setup is needed to run this test. - * - * 1. Checkout the forked repo: https://github.com/ostelco/myinfo-demo-app - * 2. npm install - * 3. ./start.sh - * 4. Open web browser with Developer console open. - * 5. Goto http://localhost:3001 - * 6. Click button "RETRIEVE INFO" - * 7. Login to SingPass using username: S9812381D and password: MyInfo2o15 - * 8. Click on "Accept" button. - * 9. Copy authorisationCode from developer console of web browser. - * 10. Use this value in `test myInfo client` test. - * 11. Set @Before annotation on 'setupRealStaging()' instead of 'setupUnitTest()'. - * - */ - // @Before - fun setupRealStaging() { - // Using certs from https://github.com/jamesleegovtech/myinfo-demo-app/tree/master/ssl - // server public key - val certificateFactory = CertificateFactory.getInstance("X.509") - val certificate= certificateFactory.generateCertificate(FileInputStream("src/test/resources/stg-auth-signing-public.pem")) - - ConfigRegistry.myInfoV2 = templateTestConfig.copy( - myInfoPersonDataAttributes = "name,sex,race,nationality,dob,email,mobileno,regadd,housingtype,hdbtype,marital,edulevel,assessableincome,ownerprivate,assessyear,cpfcontributions,cpfbalances", - myInfoServerPublicKey = Base64.getEncoder().encodeToString(certificate.publicKey.encoded)) - - Registry.myInfoClient = HttpClientBuilder.create().build() - } - - @Test - fun `test myInfo client`() { - val durationInMilliSec = measureTimeMillis { - val myInfoData = MyInfoClientSingleton.getPersonData(authorisationCode = "authorisation-code") - logger.info("MyInfo - UIN/FIN: {}", myInfoData?.uinFin) - logger.info("MyInfo - PersonData: {}", myInfoData?.personData) - } - logger.info("Time taken to fetch personData: {} sec", durationInMilliSec / 1000) - } - - companion object { - - val myInfoServerKeyPair: KeyPair = KeyPairGenerator.getInstance("RSA") - .apply { this.initialize(2048) } - .genKeyPair() - - @JvmStatic - val SUPPORT: DropwizardTestSupport = CertificateFactory - .getInstance("X.509") - .generateCertificate(FileInputStream("src/test/resources/stg-demoapp-client-publiccert-2018.pem")) - .let { certificate -> - DropwizardTestSupport( - MyInfoEmulatorApp::class.java, - ResourceHelpers.resourceFilePath("myinfo-emulator-config.yaml"), - ConfigOverride.config("myInfoApiClientId", templateTestConfig.myInfoApiClientId), - ConfigOverride.config("myInfoApiClientSecret", templateTestConfig.myInfoApiClientSecret), - ConfigOverride.config("myInfoRedirectUri", templateTestConfig.myInfoRedirectUri), - ConfigOverride.config("myInfoServerPublicKey", Base64.getEncoder().encodeToString(myInfoServerKeyPair.public.encoded)), - ConfigOverride.config("myInfoServerPrivateKey", Base64.getEncoder().encodeToString(myInfoServerKeyPair.private.encoded)), - ConfigOverride.config("myInfoClientPublicKey", Base64.getEncoder().encodeToString(certificate.publicKey.encoded))) - } - - @JvmStatic - @BeforeClass - fun beforeClass() = SUPPORT.before() - - @JvmStatic - @AfterClass - fun afterClass() = SUPPORT.after() - } -} - -class RSAKeyTest { - - @Test - fun `test encode and decode`() { - val keyPair: KeyPair = KeyPairGenerator.getInstance("RSA") - .apply { this.initialize(2048) } - .genKeyPair() - - val encodedPublicKey = keyPair.public.encoded - - val base64PublicKey = Base64.getEncoder().encodeToString(encodedPublicKey) - val decodedPublicKey = Base64.getDecoder().decode(base64PublicKey) - - assertArrayEquals(encodedPublicKey, decodedPublicKey) - - assertEquals(keyPair.public, KeyFactory - .getInstance("RSA") - .generatePublic(X509EncodedKeySpec(Base64 - .getDecoder() - .decode(base64PublicKey)))) - - val encodedPrivateKey = keyPair.private.encoded - val base64PrivateKey = Base64.getEncoder().encodeToString(encodedPrivateKey) - val decodedPrivateKey = Base64.getDecoder().decode(base64PrivateKey) - - assertArrayEquals(encodedPrivateKey, decodedPrivateKey) - - assertEquals(keyPair.private, KeyFactory - .getInstance("RSA") - .generatePrivate(PKCS8EncodedKeySpec(Base64 - .getDecoder() - .decode(base64PrivateKey)))) - } - - @Test - fun `test loading MyInfo Staging Key`() { - - // Using public cert from https://github.com/jamesleegovtech/myinfo-demo-app/tree/master/ssl - val certificateFactory = CertificateFactory.getInstance("X.509") - val certificate= certificateFactory.generateCertificate(FileInputStream("src/test/resources/stg-auth-signing-public.pem")) - certificate.publicKey - } - - @Test - fun `test loading MyInfo Staging client private key`() { - - // Using cert from https://github.com/jamesleegovtech/myinfo-demo-app/tree/master/ssl - val base64Encoded = String(File("src/test/resources/stg-demoapp-client-privatekey-2018.pem").readBytes()) - .replace("\n","") - .removePrefix("-----BEGIN PRIVATE KEY-----") - .removeSuffix("-----END PRIVATE KEY-----") - val keySpec = PKCS8EncodedKeySpec(Base64.getDecoder().decode(base64Encoded)) - val clientPrivateKey = KeyFactory.getInstance("RSA").generatePrivate(keySpec) - } - - @Test - fun `test MyInfo Staging client public key`() { - - // Using public cert from https://github.com/jamesleegovtech/myinfo-demo-app/tree/master/ssl - val certificateFactory = CertificateFactory.getInstance("X.509") - val certificate= certificateFactory.generateCertificate(FileInputStream("src/test/resources/stg-demoapp-client-publiccert-2018.pem")) - certificate.publicKey - } -} \ No newline at end of file diff --git a/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClientTest.kt b/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClientTest.kt index c79fe1fad..8dd182e2d 100644 --- a/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClientTest.kt +++ b/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClientTest.kt @@ -15,7 +15,6 @@ import org.ostelco.ext.myinfo.MyInfoEmulatorConfig import org.ostelco.prime.ekyc.ConfigRegistry import org.ostelco.prime.ekyc.MyInfoV3Config import org.ostelco.prime.ekyc.Registry -import org.ostelco.prime.ekyc.myinfo.v2.MyInfoClientSingleton import org.ostelco.prime.getLogger import java.io.File import java.io.FileInputStream @@ -30,14 +29,14 @@ import java.util.* import kotlin.system.measureTimeMillis /** - * Ref: https://www.ndi-api.gov.sg/assets/lib/trusted-data/myinfo/specs/myinfo-kyc-v2.1.1.yaml.html#section/Environments + * Ref: https://www.ndi-api.gov.sg/assets/lib/trusted-data/myinfo/specs/myinfo-kyc-v3.0.2.html#section/Environments * * https://{ENV_DOMAIN_NAME}/{VERSION}/{RESOURCE} * * ENV_DOMAIN_NAME: - * - Sandbox/Dev: https://myinfosgstg.api.gov.sg/dev/ - * - Staging: https://myinfosgstg.api.gov.sg/test/ - * - Production: https://myinfosg.api.gov.sg/ + * - Sandbox/Dev: https://sandbox.api.myinfo.gov.sg/com/v3 + * - Staging: https://test.api.myinfo.gov.sg/com/v3 + * - Production: https://api.myinfo.gov.sg/com/v3 * * VERSION: `/v2` */ diff --git a/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/MyInfoEmulatorApp.kt b/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/MyInfoEmulatorApp.kt index f4fe45fa8..28452afa4 100644 --- a/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/MyInfoEmulatorApp.kt +++ b/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/MyInfoEmulatorApp.kt @@ -31,9 +31,6 @@ class MyInfoEmulatorApp : Application() { Logger.getLogger(LoggingFeature.DEFAULT_LOGGER_NAME), PAYLOAD_ANY)) - env.jersey().register(org.ostelco.ext.myinfo.v2.TokenResource(config)) - env.jersey().register(org.ostelco.ext.myinfo.v2.PersonResource(config)) - env.jersey().register(org.ostelco.ext.myinfo.v3.TokenResource(config)) env.jersey().register(org.ostelco.ext.myinfo.v3.PersonResource(config)) } diff --git a/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/v2/Resources.kt b/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/v2/Resources.kt deleted file mode 100644 index 88bf0e34e..000000000 --- a/ext-myinfo-emulator/src/main/kotlin/org/ostelco/ext/myinfo/v2/Resources.kt +++ /dev/null @@ -1,186 +0,0 @@ -package org.ostelco.ext.myinfo.v2 - -import org.ostelco.ext.myinfo.JsonUtils.compactJson -import org.ostelco.ext.myinfo.JweCompactUtils -import org.ostelco.ext.myinfo.JwtUtils.createAccessToken -import org.ostelco.ext.myinfo.JwtUtils.getClaims -import org.ostelco.ext.myinfo.MyInfoEmulatorConfig -import org.ostelco.prime.getLogger -import javax.ws.rs.Consumes -import javax.ws.rs.FormParam -import javax.ws.rs.GET -import javax.ws.rs.HeaderParam -import javax.ws.rs.POST -import javax.ws.rs.Path -import javax.ws.rs.PathParam -import javax.ws.rs.Produces -import javax.ws.rs.QueryParam -import javax.ws.rs.core.Context -import javax.ws.rs.core.HttpHeaders -import javax.ws.rs.core.MediaType -import javax.ws.rs.core.Response - -@Path("/v2/token") -class TokenResource(private val config: MyInfoEmulatorConfig) { - - private val logger by getLogger() - - @POST - @Consumes(MediaType.APPLICATION_FORM_URLENCODED) - @Produces(MediaType.APPLICATION_JSON) - fun getToken( - @FormParam("grant_type") grantType: String?, - @FormParam("code") authorisationCode: String?, - @FormParam("redirect_uri") redirectUri: String?, - @FormParam("client_id") clientId: String?, - @FormParam("client_secret") clientSecret: String?, - @HeaderParam("Authorization") authHeaderString: String?, - @Context headers: HttpHeaders, - body: String): Response { - - logger.debug("Content-Type: ${headers.mediaType}") - logger.debug("Headers >>>\n${headers.requestHeaders.entries.joinToString("\n")}\n<<< End of Headers") - logger.debug("Body >>>\n$body\n<<< End of Body") - - return when { - - headers.mediaType != MediaType.APPLICATION_FORM_URLENCODED_TYPE -> - Response.status(Response.Status.BAD_REQUEST) - .entity("""{reason: "Invalid Content-Type - ${headers.mediaType}"}""") - .build() - - grantType != "authorization_code" -> - Response.status(Response.Status.BAD_REQUEST) - .entity("""{reason: "Invalid grant_type"}""") - .build() - - redirectUri != config.myInfoRedirectUri -> - Response.status(Response.Status.FORBIDDEN) - .entity("""{reason: "Invalid redirect_uri"}""") - .build() - - clientId != config.myInfoApiClientId -> - Response.status(Response.Status.FORBIDDEN) - .entity("""{reason: "Invalid client_id"}""") - .build() - - clientSecret != config.myInfoApiClientSecret -> - Response.status(Response.Status.FORBIDDEN) - .entity("""{reason: "Invalid client_secret"}""") - .build() - - else -> - Response.status(Response.Status.OK).entity(""" - { - "access_token":"${createAccessToken(config.myInfoServerPrivateKey)}", - "scope":"mobileno nationality dob name mailadd email sex residentialstatus", - "token_type":"Bearer", - "expires_in":1799 - }""".trimIndent()) - .build() - } - } -} - -@Path("/v2/person") -class PersonResource(private val config: MyInfoEmulatorConfig) { - - private val logger by getLogger() - - @GET - @Produces(MediaType.APPLICATION_JSON) - @Path("/{uinFin}") - fun getToken( - @PathParam("uinFin") uinFin: String, - @QueryParam("client_id") clientId: String, - @QueryParam("attributes") attributes: String, - @HeaderParam("Authorization") authHeaderString: String, - @Context headers: HttpHeaders, - body: String): Response { - - logger.debug("Content-Type: ${headers.mediaType}") - logger.debug("Headers >>>\n${headers.requestHeaders.entries.joinToString("\n")}\n<<< End of Headers") - logger.debug("Body >>>\n$body\n<<< End of Body") - - if (!authHeaderString.contains("Bearer ")) { - return Response.status(Response.Status.FORBIDDEN) - .entity("""{reason: "Missing JWT Access Token"}""") - .build() - } - - val claims = getClaims(authHeaderString.substringAfter("Bearer "), config.myInfoServerPublicKey) - - if (claims.body.subject != uinFin) { - return Response.status(Response.Status.FORBIDDEN) - .entity("""{reason: "Invalid Subject in Access Token"}""") - .build() - } - - if (authHeaderString.startsWith("Bearer ")) { - return Response - .status(Response.Status.OK) - .entity(getPersonData(uinFin = uinFin)) - .build() - } - - return Response - .status(Response.Status.OK) - .entity(JweCompactUtils.encrypt(config.myInfoClientPublicKey, getPersonData(uinFin = uinFin))) - .build() - } - - private fun getPersonData(uinFin: String): String = compactJson(""" -{ - "name": { - "lastupdated": "2018-03-20", - "source": "1", - "classification": "C", - "value": "TAN XIAO HUI" - }, - "sex": { - "lastupdated": "2018-03-20", - "source": "1", - "classification": "C", - "value": "F" - }, - "nationality": { - "lastupdated": "2018-03-20", - "source": "1", - "classification": "C", - "value": "SG" - }, - "dob": { - "lastupdated": "2018-03-20", - "source": "1", - "classification": "C", - "value": "1970-05-17" - }, - "email": { - "lastupdated": "2018-08-23", - "source": "4", - "classification": "C", - "value": "myinfotesting@gmail.com" - }, - "mobileno": { - "lastupdated": "2018-08-23", - "code": "65", - "source": "4", - "classification": "C", - "prefix": "+", - "nbr": "97399245" - }, - "mailadd": { - "country": "SG", - "unit": "128", - "street": "BEDOK NORTH AVENUE 4", - "lastupdated": "2018-03-20", - "block": "102", - "postal": "460102", - "source": "1", - "classification": "C", - "floor": "09", - "building": "PEARL GARDEN" - }, - "uinfin": "$uinFin" -}""") -} \ No newline at end of file diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index 39853d77b..d343361b4 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -82,7 +82,6 @@ enum class KycStatus { data class MyInfoConfig(val url: String) enum class MyInfoApiVersion { - V2, V3 } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index afa915d55..ca7cb1d5f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -53,7 +53,6 @@ import org.ostelco.prime.model.KycType.JUMIO import org.ostelco.prime.model.KycType.MY_INFO import org.ostelco.prime.model.KycType.NRIC_FIN import org.ostelco.prime.model.MyInfoApiVersion -import org.ostelco.prime.model.MyInfoApiVersion.V2 import org.ostelco.prime.model.MyInfoApiVersion.V3 import org.ostelco.prime.model.PaymentType.SUBSCRIPTION import org.ostelco.prime.model.Plan @@ -1808,7 +1807,6 @@ object Neo4jStoreSingleton : GraphStore { val myInfoData = try { when (version) { - V2 -> myInfoKycV2Service V3 -> myInfoKycV3Service }.getPersonData(authorisationCode) } catch (e: Exception) { diff --git a/prime/config/config.yaml b/prime/config/config.yaml index f61971804..3d3319586 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -41,14 +41,6 @@ modules: connectionRequestTimeout: 1s - type: kyc config: - myInfoV2: - myInfoApiUri: ${MY_INFO_V2_API_URI} - myInfoApiClientId: ${MY_INFO_API_CLIENT_ID} - myInfoApiClientSecret: ${MY_INFO_API_CLIENT_SECRET} - myInfoApiRealm: ${MY_INFO_API_REALM} - myInfoRedirectUri: ${MY_INFO_REDIRECT_URI} - myInfoServerPublicKey: ${MY_INFO_SERVER_PUBLIC_KEY} - myInfoClientPrivateKey: ${MY_INFO_CLIENT_PRIVATE_KEY} myInfoV3: myInfoApiUri: ${MY_INFO_V3_API_URI} myInfoApiClientId: ${MY_INFO_API_CLIENT_ID} diff --git a/prime/config/test.yaml b/prime/config/test.yaml index 3cc7467b6..ab8887d25 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -19,14 +19,6 @@ modules: csvFile: /config-data/imeiDb.csv - type: kyc config: - myInfoV2: - myInfoApiUri: http://ext-myinfo-emulator:8080/v2 - myInfoApiClientId: STG2-MYINFO-SELF-TEST - myInfoApiClientSecret: 44d953c796cccebcec9bdc826852857ab412fbe2 - myInfoApiRealm: http://localhost:3001 - myInfoRedirectUri: http://localhost:3001/callback - myInfoServerPublicKey: MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqWWA2rH1wuBkd1zp0uOh+dnCRRcQWiI89ildk9UGSd3kgzPx1mYEL40cBOBVpSIkbRp65fJDjBm+MhzlHBgWZ1q27S30nczwnzAUJqUfJvLeCW7HLwqwPVSQlqby/n4MV2AKUu0jMacOeXE3Bevm92BEOH9wQhv81Rd7HZXRJGgMecqmVehMT7Mk88xHJvvWD1bYSQL5ADnNz1v0wq/afOVYPWAOl7xYoIgokYJQD3WwnKHVcotZcP8B5mu0AuMnP71JnzjVsRpwuO8N/m28fmzXCY7ARwRpz20Q6oOq09+ZMiJkpdT5TTqEF1u3FxTq5TY8CY60q9L5RqEUNJA9fQIDAQAB - myInfoClientPrivateKey: MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDGBRdsiDqKPGyHgOpzxmSU2EQkm+zYZLvlPlwkwyfFWLndFLZ3saxJS+LIixsFhunrrUT9ZZ0x+bB6MV55o70z4ABOJRFNWx1wbMGqdiC0Fyfpwad3iYpRVjZO+5etHA9JEoaTPoFxv+ktd8kVAL9P5I7/Pi6g1R+B2t2lsaE2bMSwtZqgs55gb7fsCR3Z4nQi7BddYR7MZ2lAMWf7h7Dkm6uRlGhl2RvtmYa6dXFnK3RhIpdQOUT3quyhweMGspowC/tYSG+BNhy1WukbwhIP5vTAvv1WbHTg+WaUUV+pP0TjPQcY73clHxNpI5zrNqDmwD2rogNfePKRUI63yBUfAgMBAAECggEAGy/7xVT25J/jLr+OcRLeIGmJAZW+8P7zpUfoksuQnFHQQwBjBRAJ3Y5jtrESprGdUFRb0oavDHuBtWUt2XmXspWgtRn1xC8sXZExDdxmJRPA0SFbgtgJe51gm3uDmarullPK0lCUqS92Ll3x58ZQfgGdeIHrGP3p84Q/Rk6bGcObcPhDYWSOYKm4i2DPM01bnZG2z4BcrWSseOmeWUxqZcMlGz9GAyepUU/EoqRIHxw/2Y+TGus1JSy5DdhPE0HAEWKZH729ZdoyikOZCMxApQglUkRwkwhtXzVAemm6OSoy3BEWvSEJh/F82tFrmquUoe/xd5JastlBHyD78RAakQKBgQDkHAzo1fowRI19tk7VCPn0zMdF/UTRghtLywc/4xnw1Nd13m+orArOdVzPlQokLVNL81dIVKXnId0Hw/kX8CRyRYz8tkL81spc39DfalZW7QI7Fschfq1Htgkxd/QEjBlIaqjkOjGSbX9xYjYU1Db8PuGoGXWOsYiv9PCsKR056wKBgQDeOzfZSpV5kX8SECJXRA+emyCnO9S29p0W+5BCTQp3OPnmbL7b/mGqBVJ0DC+IiN67Lu8xxzejswqLZqaRvmQuioqH+8mOGpXYZwhShAif2AuixxvL7OK6dvDmMqoKhBI9nZ9+XI60Cd/LjnWgyFO04uq4otnTukmYsSP+fp6wnQKBgEopYH0WjFfDAelcKzcRywouxZ7Yn9Ypoaw7nujDcfydhktY/R5uiLjk6T7H6tsmLU2lGLx4YNPLa6wJp+ODfKX2PMcwjojbYEFftu3cCaQLPE1vs2ANalLFOSnvINOVpOapXq2Mye8cUHHRh1mwQQwzeXQIivLQf2sNjG28lDbvAoGACsh80UJZNmjk7Y9y2yEmUN/eGb9Bdw9IWBEk0tLCKz7MgW3NZQdW3dUcRx1AQTPC+vowCQ5NmNfbLyBv/KpsWgXG6wpAoXCQzMtTEA3wDTGCfweCRcbcyYdz8PeMYK4/5FV9o7gCBKJmBY6IDqEpzqEkGolsYGWtpIcT5Alo0dECgYEA3hzC9NLwumi/1JWm+ASSADTO3rrGo9hicG/WKGzSHD5l1f+IO1SfmUN/6i2JjcnE07eYArNrCfbMgkFavj502ne2fSaYM4p0o147O9Ty8jCyY9vuh/ZGid6qUe3TBI6/okWfmYw6FVbRpNfVEeG7kPfkDW/JdH7qkWTFbh3eH1k= myInfoV3: myInfoApiUri: http://ext-myinfo-emulator:8080/v3 myInfoApiClientId: STG2-MYINFO-SELF-TEST diff --git a/prime/infra/dev/prime-customer-api.yaml b/prime/infra/dev/prime-customer-api.yaml index 6dff36ed3..25d247190 100644 --- a/prime/infra/dev/prime-customer-api.yaml +++ b/prime/infra/dev/prime-customer-api.yaml @@ -257,42 +257,6 @@ paths: description: "Region or Scan not found." security: - firebase: [] - "/regions/sg/kyc/myInfoConfig": - get: - description: "Get Singapore MyInfo v2 service config (deprecated)." - produces: - - application/json - operationId: "getMyInfoV2Config" - responses: - 200: - description: "MyInfo service config." - schema: - $ref: "#/definitions/MyInfoConfig" - 404: - description: "Config not found." - security: - - firebase: [] - "/regions/sg/kyc/myInfo/{authorisationCode}": - get: - description: "Get Customer Data from Singapore MyInfo v2 service (deprecated)." - produces: - - application/json - operationId: "getCustomerMyInfoV2Data" - parameters: - - name: authorisationCode - in: path - description: "Authorisation Code" - required: true - type: string - responses: - 200: - description: "Successfully retrieved Customer Data from MyInfo v2 service." - schema: - type: object - 404: - description: "Person Data not found." - security: - - firebase: [] "/regions/sg/kyc/myInfo/v3/config": get: description: "Get Singapore MyInfo v3 service config." diff --git a/prime/infra/prime-direct-values.yaml b/prime/infra/prime-direct-values.yaml index 731194964..65b8e9210 100644 --- a/prime/infra/prime-direct-values.yaml +++ b/prime/infra/prime-direct-values.yaml @@ -33,9 +33,7 @@ prime: ACTIVATE_TOPIC_ID: ocs-activate CCR_SUBSCRIPTION_ID: ocs-ccr-prime-direct-sub GOOGLE_APPLICATION_CREDENTIALS: /secret/prime-service-account.json - MY_INFO_V2_API_URI: https://myinfosgstg.api.gov.sg/test/v2 MY_INFO_V3_API_URI: https://test.api.myinfo.gov.sg/com/v3 - MY_INFO_API_REALM: direct MY_INFO_REDIRECT_URI: https://dl-dev.oya.world/links/myinfo secretVolumes: From 5178cc8b1de427584a8027c4dba381b4c31ef14c Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 5 Nov 2019 14:38:33 +0100 Subject: [PATCH 07/27] Updated prime version --- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index 069e9d257..69f69b380 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.70.3" +version = "1.71.0" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 16dd743be..b8d3e49c2 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.70.3,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.71.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From b9db3b71654f39696d69293a4c781ff5294f8805 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 5 Nov 2019 20:08:17 +0100 Subject: [PATCH 08/27] Add new module for shared classes in publisher --- publisher-extensions/build.gradle.kts | 14 ++++ .../PublisherExtensions.kt | 83 +++++++++++++++++++ settings.gradle.kts | 2 + 3 files changed, 99 insertions(+) create mode 100644 publisher-extensions/build.gradle.kts create mode 100644 publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt diff --git a/publisher-extensions/build.gradle.kts b/publisher-extensions/build.gradle.kts new file mode 100644 index 000000000..d02e4caca --- /dev/null +++ b/publisher-extensions/build.gradle.kts @@ -0,0 +1,14 @@ +import org.ostelco.prime.gradle.Version + +plugins { + kotlin("jvm") + `java-library` +} + +dependencies { + implementation(project(":prime-modules")) + + implementation("com.google.cloud:google-cloud-pubsub:${Version.googleCloudPubSub}") + implementation("com.google.code.gson:gson:${Version.gson}") + implementation("com.google.guava:guava:${Version.guava}") +} \ No newline at end of file diff --git a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt new file mode 100644 index 000000000..e6e45b998 --- /dev/null +++ b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt @@ -0,0 +1,83 @@ +package org.ostelco.common.publisherex + +import com.google.api.core.ApiFutureCallback +import com.google.api.core.ApiFutures +import com.google.api.gax.core.NoCredentialsProvider +import com.google.api.gax.grpc.GrpcTransportChannel +import com.google.api.gax.rpc.ApiException +import com.google.api.gax.rpc.FixedTransportChannelProvider +import com.google.cloud.pubsub.v1.Publisher +import com.google.common.util.concurrent.MoreExecutor +import com.google.pubsub.v1.ProjectTopicName +import com.google.pubsub.v1.PubsubMessage +import io.grpc.ManagedChannelBuilder +import org.ostelco.prime.analytics.ConfigRegistry +import org.ostelco.prime.analytics.events.Event +import org.ostelco.prime.getLogger +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit + +class DelegatePubSubPublisher( + private val topicId: String, + private val projectId: String = ConfigRegistry.config.projectId) : PubSubPublisher { + + private lateinit var publisher: Publisher + private val logger by getLogger() + + override fun start() { + + val topicName = ProjectTopicName.of(projectId, topicId) + val strSocketAddress = System.getenv("PUBSUB_EMULATOR_HOST") + publisher = if (!strSocketAddress.isNullOrEmpty()) { + val channel = ManagedChannelBuilder.forTarget(strSocketAddress).usePlaintext().build() + // Create a publisher instance with default settings bound to the topic + val channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) + Publisher.newBuilder(topicName) + .setChannelProvider(channelProvider) + .setCredentialsProvider(NoCredentialsProvider()) + .build() + } else { + Publisher.newBuilder(topicName).build() + } + } + + override fun stop() { + // When finished with the publisher, shutdown to free up resources. + publisher.shutdown() + publisher.awaitTermination(1, TimeUnit.MINUTES) + singleThreadScheduledExecutor.shutdown() + } + + override fun publishPubSubMessage(pubsubMessage: PubsubMessage) { + val future = publisher.publish(pubsubMessage) + + // add an asynchronous callback to handle success / failure + ApiFutures.addCallback(future, object : ApiFutureCallback { + + override fun onFailure(throwable: Throwable) { + if (throwable is ApiException) { + // details on the API exception + logger.warn("Error publishing message to Pubsub topic: $topicId\n" + + "Message: ${throwable.message}\n" + + "Status code: ${throwable.statusCode.code}\n" + + "Retrying: ${throwable.isRetryable}") + } else { + logger.warn("Error publishing message to Pubsub topic: $topicId") + } + } + + override fun onSuccess(messageId: String) { + // Once published, returns server-assigned message ids (unique within the topic) + logger.debug("Published message $messageId to topic $topicId") + } + }, MoreExecutors.directExecutor()) + } + + override fun publishEvent(event: Event) { + val message = PubsubMessage.newBuilder() + .setData(event.toJsonByteString()) + .build() + publishPubSubMessage(message) + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index a795c0a98..fcc6319e3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -35,6 +35,7 @@ include(":prime") include(":prime-admin") include(":prime-modules") include(":prime-customer-api") +include(":publisher-extensions") include(":scaninfo-datastore") include(":scaninfo-shredder") include(":secure-archive") @@ -84,6 +85,7 @@ project(":prime").projectDir = File("$rootDir/prime") project(":prime-admin").projectDir = File("$rootDir/tools/prime-admin") project(":prime-modules").projectDir = File("$rootDir/prime-modules") project(":prime-customer-api").projectDir = File("$rootDir/prime-customer-api") +project(":publisher-extensions").projectDir = File("$rootDir/publisher-extensions") project(":scaninfo-datastore").projectDir = File("$rootDir/scaninfo-datastore") project(":scaninfo-shredder").projectDir = File("$rootDir/scaninfo-shredder") project(":secure-archive").projectDir = File("$rootDir/secure-archive") From a6779a18acefd8ebafe2d8d7d6e941a1c0ce3d94 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 5 Nov 2019 20:38:28 +0100 Subject: [PATCH 09/27] More files to the library --- .../CommonPubSubJsonSerializer.kt | 24 +++++++++++++++++++ .../PublisherExtensions.kt | 23 ++++++++++++++---- 2 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/CommonPubSubJsonSerializer.kt diff --git a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/CommonPubSubJsonSerializer.kt b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/CommonPubSubJsonSerializer.kt new file mode 100644 index 000000000..640c8e866 --- /dev/null +++ b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/CommonPubSubJsonSerializer.kt @@ -0,0 +1,24 @@ +package org.ostelco.common.publisherex + +import com.google.gson.Gson +import com.google.gson.JsonPrimitive +import com.google.gson.JsonSerializer +import com.google.protobuf.ByteString +import java.time.Instant +import java.time.format.DateTimeFormatter +import java.util.* + +object CommonPubSubJsonSerializer { + private val gson = Gson() + .newBuilder() + .registerTypeAdapter(Instant::class.java, JsonSerializer { src, _, _ -> + JsonPrimitive(DateTimeFormatter.ISO_INSTANT.format(src)) + }) + .registerTypeAdapter(Currency::class.java, JsonSerializer { src, _, _ -> + JsonPrimitive(src.currencyCode) + }) + .create() + + fun toJson(event: Event): String = gson.toJson(event) + fun toJsonByteString(event: Event): ByteString = ByteString.copyFromUtf8(toJson(event)) +} diff --git a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt index e6e45b998..fc7b00de0 100644 --- a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt +++ b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt @@ -7,20 +7,34 @@ import com.google.api.gax.grpc.GrpcTransportChannel import com.google.api.gax.rpc.ApiException import com.google.api.gax.rpc.FixedTransportChannelProvider import com.google.cloud.pubsub.v1.Publisher -import com.google.common.util.concurrent.MoreExecutor +import com.google.common.util.concurrent.MoreExecutors import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage +import io.dropwizard.lifecycle.Managed import io.grpc.ManagedChannelBuilder -import org.ostelco.prime.analytics.ConfigRegistry -import org.ostelco.prime.analytics.events.Event import org.ostelco.prime.getLogger import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit +import java.time.Instant + +/** + * Abstraction for a point-in-time analytics event. + * + * @property timestamp time at which the event occurs + */ +sealed class Event(val timestamp: Instant = Instant.now()) { + fun toJsonByteString() = CommonPubSubJsonSerializer.toJsonByteString(this) +} + +interface PubSubPublisher : Managed { + fun publishPubSubMessage(pubsubMessage: PubsubMessage) + fun publishEvent(event: Event) +} class DelegatePubSubPublisher( private val topicId: String, - private val projectId: String = ConfigRegistry.config.projectId) : PubSubPublisher { + private val projectId: String) : PubSubPublisher { private lateinit var publisher: Publisher private val logger by getLogger() @@ -46,7 +60,6 @@ class DelegatePubSubPublisher( // When finished with the publisher, shutdown to free up resources. publisher.shutdown() publisher.awaitTermination(1, TimeUnit.MINUTES) - singleThreadScheduledExecutor.shutdown() } override fun publishPubSubMessage(pubsubMessage: PubsubMessage) { From 42ad14ee0cddfef070d99fd147e4b499b87fb5b8 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 5 Nov 2019 21:12:54 +0100 Subject: [PATCH 10/27] WIP, changes to Analytics module --- analytics-module/build.gradle.kts | 1 + .../events/CommonPubSubJsonSerializer.kt | 1 + .../ostelco/prime/analytics/events/Events.kt | 11 +---------- .../publishers/ActiveUsersPublisher.kt | 6 +++++- .../DataConsumptionInfoPublisher.kt | 6 +++++- .../publishers/DelegatePubSubPublisher.kt | 8 ++++---- .../analytics/publishers/PubSubPublisher.kt | 4 ++-- .../analytics/publishers/PurchasePublisher.kt | 6 +++++- .../analytics/publishers/RefundPublisher.kt | 6 +++++- .../publishers/SimProvisioningPublisher.kt | 6 +++++- .../SubscriptionStatusUpdatePublisher.kt | 6 +++++- .../PubSubPublisher.kt | 19 +++++++++++++++++++ .../PublisherExtensions.kt | 16 ---------------- 13 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PubSubPublisher.kt diff --git a/analytics-module/build.gradle.kts b/analytics-module/build.gradle.kts index 88e4e612c..984d695ad 100644 --- a/analytics-module/build.gradle.kts +++ b/analytics-module/build.gradle.kts @@ -7,6 +7,7 @@ plugins { dependencies { implementation(project(":prime-modules")) + implementation(project(":publisher-extensions")) implementation("com.google.cloud:google-cloud-pubsub:${Version.googleCloudPubSub}") implementation("com.google.code.gson:gson:${Version.gson}") diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt index 56701b4ac..65e5a5ddc 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt @@ -7,6 +7,7 @@ import com.google.protobuf.ByteString import java.time.Instant import java.time.format.DateTimeFormatter import java.util.* +import org.ostelco.common.publisherex.Event object CommonPubSubJsonSerializer { private val gson = Gson() diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/Events.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/Events.kt index 31c3f7405..774338e92 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/Events.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/Events.kt @@ -2,16 +2,7 @@ package org.ostelco.prime.analytics.events import org.ostelco.prime.model.SimProfileStatus import java.math.BigDecimal -import java.time.Instant - -/** - * Abstraction for a point-in-time analytics event. - * - * @property timestamp time at which the event occurs - */ -sealed class Event(val timestamp: Instant = Instant.now()) { - fun toJsonByteString() = CommonPubSubJsonSerializer.toJsonByteString(this) -} +import org.ostelco.common.publisherex.Event /** * Represents a new purchase. diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt index c31dceff3..31717b1ba 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt @@ -8,12 +8,16 @@ import org.ostelco.analytics.api.ActiveUsersInfo import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.metrics.api.User import java.time.Instant +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher /** * This class publishes the active users information events to the Google Cloud Pub/Sub. */ object ActiveUsersPublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.activeUsersTopicId) { + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.activeUsersTopicId, + projectId = ConfigRegistry.config.projectId) { private val jsonPrinter = JsonFormat.printer().includingDefaultValueFields() diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt index 7d442d483..0d1d8e360 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DataConsumptionInfoPublisher.kt @@ -2,12 +2,16 @@ package org.ostelco.prime.analytics.publishers import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.analytics.events.DataConsumptionEvent +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher /** * This holds logic for publishing the data consumption information events to Google Cloud Pub/Sub. */ object DataConsumptionInfoPublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.dataTrafficTopicId) { + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.dataTrafficTopicId, + projectId = ConfigRegistry.config.projectId) { /** * Publishes a new data consumption record to Cloud Pubsub diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt index ca304fc2b..7b803a757 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt @@ -11,15 +11,15 @@ import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage import io.grpc.ManagedChannelBuilder import org.ostelco.prime.analytics.ConfigRegistry -import org.ostelco.prime.analytics.events.Event import org.ostelco.prime.getLogger import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit +import org.ostelco.common.publisherex.Event -class DelegatePubSubPublisher( +class DelegatePubSubPublisher1( private val topicId: String, - private val projectId: String = ConfigRegistry.config.projectId) : PubSubPublisher { + private val projectId: String = ConfigRegistry.config.projectId) : PubSubPublisher1 { private lateinit var publisher: Publisher private val logger by getLogger() @@ -74,7 +74,7 @@ class DelegatePubSubPublisher( // Once published, returns server-assigned message ids (unique within the topic) logger.debug("Published message $messageId to topic $topicId") } - }, DataConsumptionInfoPublisher.singleThreadScheduledExecutor) + }, singleThreadScheduledExecutor) } override fun publishEvent(event: Event) { diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt index f422af657..233696d34 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt @@ -2,10 +2,10 @@ package org.ostelco.prime.analytics.publishers import com.google.pubsub.v1.PubsubMessage import io.dropwizard.lifecycle.Managed -import org.ostelco.prime.analytics.events.Event import java.util.concurrent.ScheduledExecutorService +import org.ostelco.common.publisherex.Event -interface PubSubPublisher : Managed { +interface PubSubPublisher1 : Managed { var singleThreadScheduledExecutor: ScheduledExecutorService fun publishPubSubMessage(pubsubMessage: PubsubMessage) fun publishEvent(event: Event) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchasePublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchasePublisher.kt index 0cc1db21c..3f022c663 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchasePublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PurchasePublisher.kt @@ -2,12 +2,16 @@ package org.ostelco.prime.analytics.publishers import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.analytics.events.PurchaseEvent +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher /** * This class publishes the purchase information events to Google Cloud Pub/Sub. */ object PurchasePublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.purchaseInfoTopicId) { + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.purchaseInfoTopicId, + projectId = ConfigRegistry.config.projectId) { fun publish(customerAnalyticsId: String, purchaseId: String, sku: String, priceAmountCents: Int, priceCurrency: String) { publishEvent(PurchaseEvent( diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/RefundPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/RefundPublisher.kt index d13d8b088..00eef22d3 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/RefundPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/RefundPublisher.kt @@ -2,13 +2,17 @@ package org.ostelco.prime.analytics.publishers import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.analytics.events.RefundEvent +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher /** * This class publishes the refund information events to Google Cloud Pub/Sub. */ object RefundPublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.refundsTopicId) { + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.refundsTopicId, + projectId = ConfigRegistry.config.projectId) { fun publish(customerAnalyticsId: String, purchaseId: String, reason: String?) { publishEvent(RefundEvent( diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SimProvisioningPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SimProvisioningPublisher.kt index b3fc3330e..fcd25659f 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SimProvisioningPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SimProvisioningPublisher.kt @@ -2,12 +2,16 @@ package org.ostelco.prime.analytics.publishers import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.analytics.events.SimProvisioningEvent +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher /** * This holds logic for sending SIM provisioning events to Cloud Pub/Sub. */ object SimProvisioningPublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.simProvisioningTopicId) { + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.simProvisioningTopicId, + projectId = ConfigRegistry.config.projectId) { fun publish(subscriptionAnalyticsId: String, customerAnalyticsId: String, regionCode: String) { publishEvent(SimProvisioningEvent( diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SubscriptionStatusUpdatePublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SubscriptionStatusUpdatePublisher.kt index 06d001515..89cc8c667 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SubscriptionStatusUpdatePublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/SubscriptionStatusUpdatePublisher.kt @@ -3,12 +3,16 @@ package org.ostelco.prime.analytics.publishers import org.ostelco.prime.analytics.ConfigRegistry import org.ostelco.prime.analytics.events.SubscriptionStatusUpdateEvent import org.ostelco.prime.model.SimProfileStatus +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher /** * This holds logic for sending SIM profile (=== subscription) status update events to Cloud Pub/Sub. */ object SubscriptionStatusUpdatePublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.subscriptionStatusUpdateTopicId) { + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.subscriptionStatusUpdateTopicId, + projectId = ConfigRegistry.config.projectId) { fun publish(subscriptionAnalyticsId: String, status: SimProfileStatus) { publishEvent(SubscriptionStatusUpdateEvent( diff --git a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PubSubPublisher.kt b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PubSubPublisher.kt new file mode 100644 index 000000000..e112bf3a5 --- /dev/null +++ b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PubSubPublisher.kt @@ -0,0 +1,19 @@ +package org.ostelco.common.publisherex + +import java.time.Instant +import io.dropwizard.lifecycle.Managed +import com.google.pubsub.v1.PubsubMessage + +/** + * Abstraction for a point-in-time analytics event. + * + * @property timestamp time at which the event occurs + */ +open class Event(val timestamp: Instant = Instant.now()) { + fun toJsonByteString() = CommonPubSubJsonSerializer.toJsonByteString(this) +} + +interface PubSubPublisher : Managed { + fun publishPubSubMessage(pubsubMessage: PubsubMessage) + fun publishEvent(event: Event) +} diff --git a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt index fc7b00de0..02ec91891 100644 --- a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt +++ b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt @@ -10,27 +10,11 @@ import com.google.cloud.pubsub.v1.Publisher import com.google.common.util.concurrent.MoreExecutors import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage -import io.dropwizard.lifecycle.Managed import io.grpc.ManagedChannelBuilder import org.ostelco.prime.getLogger import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit -import java.time.Instant - -/** - * Abstraction for a point-in-time analytics event. - * - * @property timestamp time at which the event occurs - */ -sealed class Event(val timestamp: Instant = Instant.now()) { - fun toJsonByteString() = CommonPubSubJsonSerializer.toJsonByteString(this) -} - -interface PubSubPublisher : Managed { - fun publishPubSubMessage(pubsubMessage: PubsubMessage) - fun publishEvent(event: Event) -} class DelegatePubSubPublisher( private val topicId: String, From c73804b85cc0fe4b3ec4dd0f6febcf61fc206e81 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 09:15:59 +0100 Subject: [PATCH 11/27] Use directExecutor for callbacks --- .../paymentprocessor/publishers/StripeEventPublisher.kt | 3 ++- .../org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt | 5 ----- .../main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt index 79bbcfdcd..d0e158f3e 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt @@ -3,6 +3,7 @@ package org.ostelco.prime.paymentprocessor.publishers import com.google.api.core.ApiFutureCallback import com.google.api.core.ApiFutures import com.google.api.gax.rpc.ApiException +import com.google.common.util.concurrent.MoreExecutors import com.google.protobuf.ByteString import com.google.protobuf.Timestamp import com.google.pubsub.v1.PubsubMessage @@ -46,7 +47,7 @@ object StripeEventPublisher : logger.debug("Published Stripe event {} as message {}", event.id, messageId) } - }, singleThreadScheduledExecutor) + }, MoreExecutors.directExecutor()) } /* Monkeypatching uber alles! */ diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt index 6fecc0beb..0d731f882 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt @@ -8,8 +8,6 @@ import com.google.cloud.pubsub.v1.Publisher import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage import io.grpc.ManagedChannelBuilder -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit @@ -19,10 +17,8 @@ class DelegatePubSubPublisher( private lateinit var publisher: Publisher - override lateinit var singleThreadScheduledExecutor: ScheduledExecutorService override fun start() { - singleThreadScheduledExecutor = Executors.newSingleThreadScheduledExecutor() val topicName = ProjectTopicName.of(projectId, topicId) val strSocketAddress = System.getenv("PUBSUB_EMULATOR_HOST") @@ -47,7 +43,6 @@ class DelegatePubSubPublisher( override fun stop() { publisher.shutdown() publisher.awaitTermination(1, TimeUnit.MINUTES) - singleThreadScheduledExecutor.shutdown() } override fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture = diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt index 37cb43840..108902015 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt @@ -7,6 +7,5 @@ import java.util.concurrent.ScheduledExecutorService interface PubSubPublisher : Managed { - var singleThreadScheduledExecutor: ScheduledExecutorService fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture } \ No newline at end of file From ee8f558407812ea825e60ca6cee680397383f402 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 10:56:18 +0100 Subject: [PATCH 12/27] Fix compile error --- .../org/ostelco/prime/analytics/publishers/PubSubPublisher.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt index 233696d34..2126e1edc 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt @@ -2,11 +2,9 @@ package org.ostelco.prime.analytics.publishers import com.google.pubsub.v1.PubsubMessage import io.dropwizard.lifecycle.Managed -import java.util.concurrent.ScheduledExecutorService import org.ostelco.common.publisherex.Event interface PubSubPublisher1 : Managed { - var singleThreadScheduledExecutor: ScheduledExecutorService fun publishPubSubMessage(pubsubMessage: PubsubMessage) fun publishEvent(event: Event) } From a88d44a3d3077b966fe352f7ca223f2a583d21e0 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 11:13:52 +0100 Subject: [PATCH 13/27] Use publisher-extension library in payment-processor --- payment-processor/build.gradle.kts | 1 + .../publishers/StripeEventPublisher.kt | 39 ++++--------------- .../prime/pubsub/DelegatePubSubPublisher.kt | 4 +- .../ostelco/prime/pubsub/PubSubPublisher.kt | 2 +- 4 files changed, 11 insertions(+), 35 deletions(-) diff --git a/payment-processor/build.gradle.kts b/payment-processor/build.gradle.kts index 4c341fb37..011542618 100644 --- a/payment-processor/build.gradle.kts +++ b/payment-processor/build.gradle.kts @@ -9,6 +9,7 @@ plugins { dependencies { implementation(project(":prime-modules")) implementation(project(":data-store")) + implementation(project(":publisher-extensions")) implementation("com.stripe:stripe-java:${Version.stripe}") diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt index d0e158f3e..842cc32a7 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt @@ -1,25 +1,18 @@ package org.ostelco.prime.paymentprocessor.publishers -import com.google.api.core.ApiFutureCallback -import com.google.api.core.ApiFutures -import com.google.api.gax.rpc.ApiException -import com.google.common.util.concurrent.MoreExecutors import com.google.protobuf.ByteString import com.google.protobuf.Timestamp import com.google.pubsub.v1.PubsubMessage import com.stripe.model.Event -import org.ostelco.prime.getLogger import org.ostelco.prime.paymentprocessor.ConfigRegistry -import org.ostelco.prime.pubsub.DelegatePubSubPublisher -import org.ostelco.prime.pubsub.PubSubPublisher import java.time.Instant +import org.ostelco.common.publisherex.DelegatePubSubPublisher +import org.ostelco.common.publisherex.PubSubPublisher -object StripeEventPublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.stripeEventTopicId, - projectId = ConfigRegistry.config.projectId) { - - private val logger by getLogger() +object StripeEventPublisher : PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.stripeEventTopicId, + projectId = ConfigRegistry.config.projectId) { fun publish(event: Event) { @@ -28,28 +21,10 @@ object StripeEventPublisher : .setSeconds(Instant.now().epochSecond)) .setData(event.byteString()) .build() - val future = publishPubSubMessage(message) - - ApiFutures.addCallback(future, object : ApiFutureCallback { - - override fun onFailure(throwable: Throwable) { - if (throwable is ApiException) { - logger.warn("Error publishing Stripe event: {} (status code: {}, retrying: {})", - event.id, throwable.statusCode.code, throwable.isRetryable) - } else { - logger.warn("Error publishing Stripe event: {}", event.id) - } - } - - /* Once published, returns server-assigned message ids (unique - within the topic) */ - override fun onSuccess(messageId: String) { - logger.debug("Published Stripe event {} as message {}", event.id, - messageId) - } - }, MoreExecutors.directExecutor()) + publishPubSubMessage(message) } /* Monkeypatching uber alles! */ private fun Event.byteString(): ByteString = ByteString.copyFromUtf8(this.toJson()) + } \ No newline at end of file diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt index 0d731f882..ed1e5b47d 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt @@ -11,9 +11,9 @@ import io.grpc.ManagedChannelBuilder import java.util.concurrent.TimeUnit -class DelegatePubSubPublisher( +class DelegatePubSubPublisher2( private val topicId: String, - private val projectId: String) : PubSubPublisher { + private val projectId: String) : PubSubPublisher2 { private lateinit var publisher: Publisher diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt index 108902015..400cc2271 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt @@ -6,6 +6,6 @@ import io.dropwizard.lifecycle.Managed import java.util.concurrent.ScheduledExecutorService -interface PubSubPublisher : Managed { +interface PubSubPublisher2 : Managed { fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture } \ No newline at end of file From 524a76d6c203cca468f446518056a4233619f0e8 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 11:27:10 +0100 Subject: [PATCH 14/27] Remove files which were made obsolete --- .../events/CommonPubSubJsonSerializer.kt | 25 ------ .../publishers/DelegatePubSubPublisher.kt | 80 ------------------- .../analytics/publishers/PubSubPublisher.kt | 10 --- .../prime/pubsub/DelegatePubSubPublisher.kt | 50 ------------ .../ostelco/prime/pubsub/PubSubPublisher.kt | 11 --- 5 files changed, 176 deletions(-) delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt delete mode 100644 prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt delete mode 100644 prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt deleted file mode 100644 index 65e5a5ddc..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/events/CommonPubSubJsonSerializer.kt +++ /dev/null @@ -1,25 +0,0 @@ -package org.ostelco.prime.analytics.events - -import com.google.gson.Gson -import com.google.gson.JsonPrimitive -import com.google.gson.JsonSerializer -import com.google.protobuf.ByteString -import java.time.Instant -import java.time.format.DateTimeFormatter -import java.util.* -import org.ostelco.common.publisherex.Event - -object CommonPubSubJsonSerializer { - private val gson = Gson() - .newBuilder() - .registerTypeAdapter(Instant::class.java, JsonSerializer { src, _, _ -> - JsonPrimitive(DateTimeFormatter.ISO_INSTANT.format(src)) - }) - .registerTypeAdapter(Currency::class.java, JsonSerializer { src, _, _ -> - JsonPrimitive(src.currencyCode) - }) - .create() - - fun toJson(event: Event): String = gson.toJson(event) - fun toJsonByteString(event: Event): ByteString = ByteString.copyFromUtf8(toJson(event)) -} diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt deleted file mode 100644 index b4958ee5d..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/DelegatePubSubPublisher.kt +++ /dev/null @@ -1,80 +0,0 @@ -package org.ostelco.prime.analytics.publishers - -import com.google.api.core.ApiFutureCallback -import com.google.api.core.ApiFutures -import com.google.api.gax.core.NoCredentialsProvider -import com.google.api.gax.grpc.GrpcTransportChannel -import com.google.api.gax.rpc.ApiException -import com.google.api.gax.rpc.FixedTransportChannelProvider -import com.google.cloud.pubsub.v1.Publisher -import com.google.common.util.concurrent.MoreExecutors -import com.google.pubsub.v1.ProjectTopicName -import com.google.pubsub.v1.PubsubMessage -import io.grpc.ManagedChannelBuilder -import org.ostelco.prime.analytics.ConfigRegistry -import org.ostelco.prime.getLogger -import java.util.concurrent.TimeUnit -import org.ostelco.common.publisherex.Event - -class DelegatePubSubPublisher1( - private val topicId: String, - private val projectId: String = ConfigRegistry.config.projectId) : PubSubPublisher1 { - - private lateinit var publisher: Publisher - private val logger by getLogger() - - override fun start() { - - val topicName = ProjectTopicName.of(projectId, topicId) - val strSocketAddress = System.getenv("PUBSUB_EMULATOR_HOST") - publisher = if (!strSocketAddress.isNullOrEmpty()) { - val channel = ManagedChannelBuilder.forTarget(strSocketAddress).usePlaintext().build() - // Create a publisher instance with default settings bound to the topic - val channelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) - Publisher.newBuilder(topicName) - .setChannelProvider(channelProvider) - .setCredentialsProvider(NoCredentialsProvider()) - .build() - } else { - Publisher.newBuilder(topicName).build() - } - } - - override fun stop() { - // When finished with the publisher, shutdown to free up resources. - publisher.shutdown() - publisher.awaitTermination(1, TimeUnit.MINUTES) - } - - override fun publishPubSubMessage(pubsubMessage: PubsubMessage) { - val future = publisher.publish(pubsubMessage) - - // add an asynchronous callback to handle success / failure - ApiFutures.addCallback(future, object : ApiFutureCallback { - - override fun onFailure(throwable: Throwable) { - if (throwable is ApiException) { - // details on the API exception - logger.warn("Error publishing message to Pubsub topic: $topicId\n" + - "Message: ${throwable.message}\n" + - "Status code: ${throwable.statusCode.code}\n" + - "Retrying: ${throwable.isRetryable}") - } else { - logger.warn("Error publishing message to Pubsub topic: $topicId") - } - } - - override fun onSuccess(messageId: String) { - // Once published, returns server-assigned message ids (unique within the topic) - logger.debug("Published message $messageId to topic $topicId") - } - }, MoreExecutors.directExecutor()) - } - - override fun publishEvent(event: Event) { - val message = PubsubMessage.newBuilder() - .setData(event.toJsonByteString()) - .build() - publishPubSubMessage(message) - } -} diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt deleted file mode 100644 index 2126e1edc..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/PubSubPublisher.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.ostelco.prime.analytics.publishers - -import com.google.pubsub.v1.PubsubMessage -import io.dropwizard.lifecycle.Managed -import org.ostelco.common.publisherex.Event - -interface PubSubPublisher1 : Managed { - fun publishPubSubMessage(pubsubMessage: PubsubMessage) - fun publishEvent(event: Event) -} diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt deleted file mode 100644 index ed1e5b47d..000000000 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/DelegatePubSubPublisher.kt +++ /dev/null @@ -1,50 +0,0 @@ -package org.ostelco.prime.pubsub - -import com.google.api.core.ApiFuture -import com.google.api.gax.core.NoCredentialsProvider -import com.google.api.gax.grpc.GrpcTransportChannel -import com.google.api.gax.rpc.FixedTransportChannelProvider -import com.google.cloud.pubsub.v1.Publisher -import com.google.pubsub.v1.ProjectTopicName -import com.google.pubsub.v1.PubsubMessage -import io.grpc.ManagedChannelBuilder -import java.util.concurrent.TimeUnit - - -class DelegatePubSubPublisher2( - private val topicId: String, - private val projectId: String) : PubSubPublisher2 { - - private lateinit var publisher: Publisher - - - override fun start() { - - val topicName = ProjectTopicName.of(projectId, topicId) - val strSocketAddress = System.getenv("PUBSUB_EMULATOR_HOST") - - publisher = if (!strSocketAddress.isNullOrEmpty()) { - val channel = ManagedChannelBuilder.forTarget(strSocketAddress) - .usePlaintext() - .build() - /* Create a publishers instance with default settings bound - to the topic. */ - val channelProvider = FixedTransportChannelProvider - .create(GrpcTransportChannel.create(channel)) - Publisher.newBuilder(topicName) - .setChannelProvider(channelProvider) - .setCredentialsProvider(NoCredentialsProvider()) - .build() - } else { - Publisher.newBuilder(topicName).build() - } - } - - override fun stop() { - publisher.shutdown() - publisher.awaitTermination(1, TimeUnit.MINUTES) - } - - override fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture = - publisher.publish(pubsubMessage) -} \ No newline at end of file diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt deleted file mode 100644 index 400cc2271..000000000 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/pubsub/PubSubPublisher.kt +++ /dev/null @@ -1,11 +0,0 @@ -package org.ostelco.prime.pubsub - -import com.google.api.core.ApiFuture -import com.google.pubsub.v1.PubsubMessage -import io.dropwizard.lifecycle.Managed -import java.util.concurrent.ScheduledExecutorService - - -interface PubSubPublisher2 : Managed { - fun publishPubSubMessage(pubsubMessage: PubsubMessage): ApiFuture -} \ No newline at end of file From 6f8ab0846e35e4ad6aaeb2b47a6258b7951e93a2 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 11:50:25 +0100 Subject: [PATCH 15/27] Remove unused import --- .../org.ostelco.common.publisherex/PublisherExtensions.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt index 02ec91891..27141698c 100644 --- a/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt +++ b/publisher-extensions/src/main/kotlin/org.ostelco.common.publisherex/PublisherExtensions.kt @@ -12,8 +12,6 @@ import com.google.pubsub.v1.ProjectTopicName import com.google.pubsub.v1.PubsubMessage import io.grpc.ManagedChannelBuilder import org.ostelco.prime.getLogger -import java.util.concurrent.Executors -import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit class DelegatePubSubPublisher( From 11ca13c4ede6e9006935bc24d56e054bc995576e Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 11:51:25 +0100 Subject: [PATCH 16/27] Whitespace fixes --- .../paymentprocessor/publishers/StripeEventPublisher.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt index 842cc32a7..3fdc12dd7 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/publishers/StripeEventPublisher.kt @@ -10,9 +10,10 @@ import org.ostelco.common.publisherex.DelegatePubSubPublisher import org.ostelco.common.publisherex.PubSubPublisher -object StripeEventPublisher : PubSubPublisher by DelegatePubSubPublisher( - topicId = ConfigRegistry.config.stripeEventTopicId, - projectId = ConfigRegistry.config.projectId) { +object StripeEventPublisher : + PubSubPublisher by DelegatePubSubPublisher( + topicId = ConfigRegistry.config.stripeEventTopicId, + projectId = ConfigRegistry.config.projectId) { fun publish(event: Event) { @@ -27,4 +28,4 @@ object StripeEventPublisher : PubSubPublisher by DelegatePubSubPublisher( /* Monkeypatching uber alles! */ private fun Event.byteString(): ByteString = ByteString.copyFromUtf8(this.toJson()) -} \ No newline at end of file +} From f7465e8306670e63547de86b990652570aacbc24 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 6 Nov 2019 11:54:15 +0100 Subject: [PATCH 17/27] Whitespace fixes --- publisher-extensions/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publisher-extensions/build.gradle.kts b/publisher-extensions/build.gradle.kts index d02e4caca..230c25f24 100644 --- a/publisher-extensions/build.gradle.kts +++ b/publisher-extensions/build.gradle.kts @@ -11,4 +11,4 @@ dependencies { implementation("com.google.cloud:google-cloud-pubsub:${Version.googleCloudPubSub}") implementation("com.google.code.gson:gson:${Version.gson}") implementation("com.google.guava:guava:${Version.guava}") -} \ No newline at end of file +} From 1212fbf74b68dcd532e81ddf0355f18110b98957 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 6 Nov 2019 13:45:05 +0100 Subject: [PATCH 18/27] Do not notify the app when the verification fails because the customer has cancelled and hence did not upload an ID. --- .../org/ostelco/prime/storage/graph/Neo4jStore.kt | 15 +++++++++------ prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index ca7cb1d5f..de3cace76 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1746,12 +1746,15 @@ object Neo4jStoreSingleton : GraphStore { transaction = transaction) } } else { - // TODO: find out what more information can be passed to the client. - appNotifier.notify( - notificationType = NotificationType.JUMIO_VERIFICATION_FAILED, - customerId = customer.id, - data = extendedStatus - ) + // Do not notify the app when the verification fails because the customer has cancelled and hence did not upload an ID. + if (updatedScanInformation.scanResult?.verificationStatus != "NO_ID_UPLOADED") { + // TODO: find out what more information can be passed to the client. + appNotifier.notify( + notificationType = NotificationType.JUMIO_VERIFICATION_FAILED, + customerId = customer.id, + data = extendedStatus + ) + } logger.info(NOTIFY_OPS_MARKER, "Jumio verification failed for ${customer.contactEmail} Info: $extendedStatus") setKycStatus( customer = customer, diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index 69f69b380..a60fea6b2 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.71.0" +version = "1.71.1" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index b8d3e49c2..f3cea2bd4 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.71.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.71.1,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From f9e05b7aba22d434b81800de99a10282ddf151e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 15:48:42 +0100 Subject: [PATCH 19/27] Rectify incoming ICCIDs to not having trailing Fs, since that screws up the database lookups. --- .../upload-sim-batch-lib-test.go | 30 ------------------- .../inventory/SimInventoryCallbackService.kt | 24 +++++++++++---- 2 files changed, 19 insertions(+), 35 deletions(-) delete mode 100644 sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go deleted file mode 100644 index e1d249b45..000000000 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go +++ /dev/null @@ -1,30 +0,0 @@ -package uploadtoprime - -import ( - "fmt" - "gotest.tools/assert" - "testing" -) - -func TestParseInputFileGeneratorCommmandline(t *testing.T) { - - parsedBatch := ParseInputFileGeneratorCommmandline() - assert.Equal(t, "Footel", parsedBatch.customer) - assert.Equal(t, "BAR_FOOTEL_STD", parsedBatch.profileType) - assert.Equal(t, "20191007", parsedBatch.orderDate) - assert.Equal(t, "2019100701", parsedBatch.batchNo) - assert.Equal(t, 10, parsedBatch.quantity) - assert.Equal(t, 894700000000002214, parsedBatch.firstIccid) - assert.Equal(t, 242017100012213, parsedBatch.firstImsi) -} - -// TODO: Make a test that checks that the correct number of things are made, -// and also that the right things are made. Already had one fencepost error -// on this stuff. - -func TestGenerateInputFile(t *testing.T) { - parsedBatch := ParseInputFileGeneratorCommmandline() - var result = GenerateInputFile(parsedBatch) - - fmt.Println(result) -} diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt index 1519e5a55..842d39879 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt @@ -1,6 +1,7 @@ package org.ostelco.simcards.inventory import org.ostelco.prime.getLogger +import org.ostelco.prime.notifications.NOTIFY_OPS_MARKER import org.ostelco.sim.es2plus.ES2NotificationPointStatus import org.ostelco.sim.es2plus.ES2RequestHeader import org.ostelco.sim.es2plus.FunctionExecutionStatusType @@ -17,23 +18,36 @@ class SimInventoryCallbackService(val dao: SimInventoryDAO) : SmDpPlusCallbackSe override fun handleDownloadProgressInfo(header: ES2RequestHeader, eid: String?, - iccid: String, + incomingIccid: String, profileType: String, timestamp: String, notificationPointId: Int, notificationPointStatus: ES2NotificationPointStatus, resultData: String?, imei: String?) { - if (notificationPointStatus.status == FunctionExecutionStatusType.ExecutedSuccess) { - /* XXX To be removed or updated to debug. */ + + // Remove padding in ICCIDs with odd number of digits. + // The database don't recognize those and will only get confused when + // trying to use ICCIDs with trailing Fs as keys. + val iccid = incomingIccid.trimEnd('F') + + // If we can't find the ICCID, then cry foul and log an error message + // that will get the ops team's attention asap! + val profileQueryResult = dao.getSimProfileByIccid(iccid) + profileQueryResult.mapLeft { + logger.error(NOTIFY_OPS_MARKER, + "Could not find ICCID='$iccid' in database while handling downloadProgressinfo callback!!") + return + } + + if (notificationPointStatus.status == FunctionExecutionStatusType.ExecutedSuccess) { logger.info("download-progress-info: Received message with status 'executed-success' for ICCID {}" + "(notificationPointId: {}, profileType: {}, resultData: {})", iccid, notificationPointId, profileType, resultData) /* Update EID. */ if (!eid.isNullOrEmpty()) { - /* XXX To be removed or updated to debug. */ logger.info("download-progress-info: Updating EID to {} for ICCID {}", eid, iccid) dao.setEidOfSimProfileByIccid(iccid, eid) @@ -41,7 +55,7 @@ class SimInventoryCallbackService(val dao: SimInventoryDAO) : SmDpPlusCallbackSe /** * Update SM-DP+ state. - * There is a somewhat more subtle failure mode, namly that the SM-DP+ for some reason + * There is a somewhat more subtle failure mode, namely that the SM-DP+ for some reason * is unable to signal back, in that case the state has actually changed, but that fact will not * be picked up by the state as stored in the database, and if the user interface is dependent * on that state, the user interface may suffer a failure. These issues needs to be gamed out From 009ce7d1e95c331edb7e17fd45ee6bc8e2ed1036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 16:07:10 +0100 Subject: [PATCH 20/27] Fixing import, so that we can run the thing. --- build-all.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-all.go b/build-all.go index cfda04956..684f3dbcb 100755 --- a/build-all.go +++ b/build-all.go @@ -6,7 +6,7 @@ package main import ( - "./github.com/ostelco-core/goscript" + "github.com/ostelco/ostelco-core/github.com/ostelco-core/goscript" "encoding/json" "flag" "fmt" From f6d10abaac9ae9b54c31de442108651a0f736347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 16:08:27 +0100 Subject: [PATCH 21/27] adding a space, for readability. --- github.com/ostelco-core/goscript/goscript.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github.com/ostelco-core/goscript/goscript.go b/github.com/ostelco-core/goscript/goscript.go index 6282ca1e5..c0ec2fc1d 100644 --- a/github.com/ostelco-core/goscript/goscript.go +++ b/github.com/ostelco-core/goscript/goscript.go @@ -174,7 +174,7 @@ func AssertThatEnvironmentVariableaAreSet(variableNames ...string) { log.Printf("Checking if environment variables are set...\n") for key := range variableNames { if len(os.Getenv(variableNames[key])) == 0 { - log.Fatalf("Environment variable not set'%s'", variableNames[key]) + log.Fatalf("Environment variable not set '%s'", variableNames[key]) } } log.Printf(" ... they are\n") From f6e8eaf17e38afe3345325f72f3c18ab3fda09a4 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 5 Nov 2019 22:21:46 +0100 Subject: [PATCH 22/27] Validate age and store pass expiry date for MyInfo --- .../kotlin/org/ostelco/at/jersey/Tests.kt | 9 +- .../kotlin/org/ostelco/at/okhttp/Tests.kt | 9 +- .../org/ostelco/prime/ekyc/myinfo/v3/Model.kt | 20 +++ .../prime/ekyc/myinfo/v3/MyInfoClient.kt | 16 +- .../ostelco/prime/ekyc/myinfo/v3/ModelTest.kt | 142 ++++++++++++++++++ .../org/ostelco/prime/model/Entities.kt | 2 +- .../ostelco/prime/storage/graph/Neo4jStore.kt | 69 +++++---- .../prime/storage/graph/model/Model.kt | 3 +- .../org/ostelco/prime/ekyc/KycServices.kt | 3 + prime/build.gradle.kts | 2 +- prime/infra/dev/prime-customer-api.yaml | 11 ++ prime/infra/prod/prime-customer-api.yaml | 47 ++---- prime/script/start.sh | 2 +- 13 files changed, 265 insertions(+), 70 deletions(-) create mode 100644 ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/Model.kt create mode 100644 ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/ModelTest.kt diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 39930fc73..acf11d708 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -30,8 +30,8 @@ import org.ostelco.prime.customer.model.PurchaseRecordList import org.ostelco.prime.customer.model.Region import org.ostelco.prime.customer.model.RegionDetails import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.APPROVED -import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.PENDING import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.AVAILABLE +import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.PENDING import org.ostelco.prime.customer.model.RegionDetailsList import org.ostelco.prime.customer.model.ScanInformation import org.ostelco.prime.customer.model.SimProfile @@ -228,6 +228,7 @@ class RegionsTest { .region(Region().id("no").name("Norway")) .status(APPROVED) .kycStatusMap(mapOf(KycType.JUMIO.name to KycStatus.APPROVED)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, noRegionDetails, "RegionDetails do not match") @@ -1504,6 +1505,7 @@ class SingaporeKycTest { KycType.MY_INFO.name to KycStatus.APPROVED, KycType.ADDRESS.name to KycStatus.PENDING, KycType.NRIC_FIN.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, newRegionDetailsList, "RegionDetails do not match") @@ -1553,6 +1555,7 @@ class SingaporeKycTest { KycType.NRIC_FIN.name to KycStatus.APPROVED, KycType.JUMIO.name to KycStatus.PENDING, KycType.ADDRESS.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, sgRegionDetails, "RegionDetails do not match") @@ -1604,6 +1607,7 @@ class SingaporeKycTest { KycType.NRIC_FIN.name to KycStatus.APPROVED, KycType.JUMIO.name to KycStatus.APPROVED, KycType.ADDRESS.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, sgRegionDetails, "RegionDetails do not match") @@ -1632,6 +1636,7 @@ class SingaporeKycTest { KycType.MY_INFO.name to KycStatus.PENDING, KycType.ADDRESS.name to KycStatus.APPROVED, KycType.NRIC_FIN.name to KycStatus.APPROVED)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, sgRegionDetails, "RegionDetails do not match") @@ -1706,6 +1711,7 @@ class SingaporeKycTest { KycType.NRIC_FIN.name to KycStatus.PENDING, KycType.JUMIO.name to KycStatus.APPROVED, KycType.ADDRESS.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, sgRegionDetails, "RegionDetails do not match") @@ -1734,6 +1740,7 @@ class SingaporeKycTest { KycType.MY_INFO.name to KycStatus.PENDING, KycType.ADDRESS.name to KycStatus.APPROVED, KycType.NRIC_FIN.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, sgRegionDetails, "RegionDetails do not match") diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt index ed85cfbb6..988e7a505 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/okhttp/Tests.kt @@ -29,8 +29,8 @@ import org.ostelco.prime.customer.model.Product import org.ostelco.prime.customer.model.Region import org.ostelco.prime.customer.model.RegionDetails import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.APPROVED -import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.PENDING import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.AVAILABLE +import org.ostelco.prime.customer.model.RegionDetails.StatusEnum.PENDING import org.ostelco.prime.customer.model.RegionDetailsList import org.ostelco.prime.customer.model.ScanInformation import org.ostelco.prime.customer.model.SimProfile @@ -154,6 +154,7 @@ class RegionsTest { .region(Region().id("no").name("Norway")) .status(APPROVED) .kycStatusMap(mapOf(KycType.JUMIO.name to KycStatus.APPROVED)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[noRegionIndex], "RegionDetails do not match") @@ -668,6 +669,7 @@ class SingaporeKycTest { KycType.MY_INFO.name to KycStatus.APPROVED, KycType.ADDRESS.name to KycStatus.PENDING, KycType.NRIC_FIN.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") @@ -711,6 +713,7 @@ class SingaporeKycTest { KycType.NRIC_FIN.name to KycStatus.APPROVED, KycType.JUMIO.name to KycStatus.PENDING, KycType.ADDRESS.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") @@ -756,6 +759,7 @@ class SingaporeKycTest { KycType.NRIC_FIN.name to KycStatus.APPROVED, KycType.JUMIO.name to KycStatus.APPROVED, KycType.ADDRESS.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") @@ -777,6 +781,7 @@ class SingaporeKycTest { KycType.MY_INFO.name to KycStatus.PENDING, KycType.ADDRESS.name to KycStatus.APPROVED, KycType.NRIC_FIN.name to KycStatus.APPROVED)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") @@ -844,6 +849,7 @@ class SingaporeKycTest { KycType.NRIC_FIN.name to KycStatus.PENDING, KycType.JUMIO.name to KycStatus.APPROVED, KycType.ADDRESS.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") @@ -865,6 +871,7 @@ class SingaporeKycTest { KycType.MY_INFO.name to KycStatus.PENDING, KycType.ADDRESS.name to KycStatus.APPROVED, KycType.NRIC_FIN.name to KycStatus.PENDING)) + .kycExpiryDateMap(emptyMap()) .simProfiles(SimProfileList()) assertEquals(regionDetails, regionDetailsList[sgRegionIndex], "RegionDetails do not match") diff --git a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/Model.kt b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/Model.kt new file mode 100644 index 000000000..e2829e87b --- /dev/null +++ b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/Model.kt @@ -0,0 +1,20 @@ +package org.ostelco.prime.ekyc.myinfo.v3 + +import com.fasterxml.jackson.annotation.JsonProperty + +open class DataItem { + var source: String = "" + var classification: String = "" + @JsonProperty("lastupdated") var lastUpdated: String = "" + var unavailable: Boolean = false +} + +class ValueDataItem( + val value: String +) : DataItem() + +data class PersonData( + val name: ValueDataItem, + @JsonProperty("dob") val dateOfBirth: ValueDataItem?, + @JsonProperty("passexpirydate") val passExpiryDate: ValueDataItem? +) \ No newline at end of file diff --git a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt index 108633e0e..db16dc33a 100644 --- a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt +++ b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt @@ -1,5 +1,7 @@ package org.ostelco.prime.ekyc.myinfo.v3 +import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.jsonwebtoken.Jwts import org.apache.cxf.rs.security.jose.jwe.JweCompactConsumer import org.apache.cxf.rs.security.jose.jwe.JweUtils @@ -27,6 +29,7 @@ import java.security.Signature import java.security.spec.PKCS8EncodedKeySpec import java.security.spec.X509EncodedKeySpec import java.time.Instant +import java.time.LocalDate import java.util.* import javax.inject.Named import javax.ws.rs.core.MediaType @@ -40,6 +43,8 @@ object MyInfoClientSingleton : MyInfoKycService { private val logger by getLogger() + private val relaxedObjectMapper = jacksonObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + override fun getConfig(): MyInfoConfig = MyInfoConfig( url = "${config.myInfoApiUri}/authorise" + "?client_id=${config.myInfoApiClientId}" + @@ -60,11 +65,18 @@ object MyInfoClientSingleton : MyInfoKycService { val uinFin = claims.body.subject // Using access_token and uin_fin, call /person API to get Person Data - val personData = getPersonData( + val personDataString = getPersonData( uinFin = uinFin, accessToken = tokenApiResponse.accessToken) - return MyInfoData(uinFin = uinFin, personData = personData) + val personData = relaxedObjectMapper.readValue(personDataString, PersonData::class.java) + + return MyInfoData( + uinFin = uinFin, + personData = personDataString, + birthDate = personData.dateOfBirth?.value?.let(LocalDate::parse), + passExpiryDate = personData.passExpiryDate?.value?.let(LocalDate::parse) + ) } private fun getToken(authorisationCode: String): String? = diff --git a/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/ModelTest.kt b/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/ModelTest.kt new file mode 100644 index 000000000..0227216e5 --- /dev/null +++ b/ekyc/src/test/kotlin/org/ostelco/prime/ekyc/myinfo/v3/ModelTest.kt @@ -0,0 +1,142 @@ +package org.ostelco.prime.ekyc.myinfo.v3 + +import com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.ostelco.ext.myinfo.JsonUtils + +class ModelTest { + + @Test + fun `test - Model class`() { + val personData = jacksonObjectMapper() + .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .readValue(personDataString, PersonData::class.java) + + assertNotNull(personData) + + assertEquals("1998-06-06", personData.dateOfBirth?.value) + assertEquals("2040-06-06", personData.passExpiryDate?.value) + } +} + +private val personDataString = JsonUtils.compactJson(""" +{ + "name": { + "lastupdated": "2019-04-05", + "source": "1", + "classification": "C", + "value": "TAN XIAO HUI" + }, + "sex": { + "lastupdated": "2019-04-05", + "code": "F", + "source": "1", + "classification": "C", + "desc": "FEMALE" + }, + "nationality": { + "lastupdated": "2019-04-05", + "code": "SG", + "source": "1", + "classification": "C", + "desc": "SINGAPORE CITIZEN" + }, + "dob": { + "lastupdated": "2019-04-05", + "source": "1", + "classification": "C", + "value": "1998-06-06" + }, + "email": { + "lastupdated": "2019-04-05", + "source": "2", + "classification": "C", + "value": "myinfotesting@gmail.com" + }, + "mobileno": { + "lastupdated": "2019-04-05", + "source": "2", + "classification": "C", + "areacode": { + "value": "65" + }, + "prefix": { + "value": "+" + }, + "nbr": { + "value": "97399245" + } + }, + "regadd": { + "country": { + "code": "SG", + "desc": "SINGAPORE" + }, + "unit": { + "value": "128" + }, + "street": { + "value": "BEDOK NORTH AVENUE 4" + }, + "lastupdated": "2019-04-05", + "block": { + "value": "102" + }, + "source": "1", + "postal": { + "value": "460102" + }, + "classification": "C", + "floor": { + "value": "09" + }, + "type": "SG", + "building": { + "value": "PEARL GARDEN" + } + }, + "mailadd": { + "country": { + "code": "SG", + "desc": "SINGAPORE" + }, + "unit": { + "value": "128" + }, + "street": { + "value": "BEDOK NORTH AVENUE 4" + }, + "lastupdated": "2019-04-05", + "block": { + "value": "102" + }, + "source": "1", + "postal": { + "value": "460102" + }, + "classification": "C", + "floor": { + "value": "09" + }, + "type": "SG", + "building": { + "value": "PEARL GARDEN" + } + }, + "passexpirydate": { + "lastupdated": "2019-04-05", + "source": "1", + "classification": "C", + "value": "2040-06-06" + }, + "uinfin": { + "value": "S1111111D", + "classification": "C", + "source": "1", + "lastupdated": "2019-03-26" + } +} + """) \ No newline at end of file diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index d343361b4..22680922c 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -57,7 +57,7 @@ data class RegionDetails( val region: Region, val status: CustomerRegionStatus, val kycStatusMap: Map = emptyMap(), - val kycExpiryDate: String? = null, + val kycExpiryDateMap: Map = emptyMap(), val simProfiles: Collection = emptyList()) enum class CustomerRegionStatus { diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index de3cace76..71c34dbc0 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -577,6 +577,7 @@ object Neo4jStoreSingleton : GraphStore { region = region, status = cr.status, kycStatusMap = cr.kycStatusMap, + kycExpiryDateMap = cr.kycExpiryDateMap, simProfiles = simProfiles) } .requireNoNulls() @@ -1742,7 +1743,7 @@ object Neo4jStoreSingleton : GraphStore { customer = customer, regionCode = updatedScanInformation.countryCode.toLowerCase(), kycType = JUMIO, - kycExpiryDate = updatedScanInformation?.scanResult?.expiry, + kycExpiryDate = updatedScanInformation.scanResult?.expiry, transaction = transaction) } } else { @@ -1769,25 +1770,30 @@ object Neo4jStoreSingleton : GraphStore { private fun verifyAndUpdateScanStatus(scanInformation: ScanInformation): ScanStatus { - val is18yrsOfAge = scanInformation.scanResult + val above18yrsOfAge = scanInformation.scanResult ?.dob ?.let(LocalDate::parse) - ?.plusYears(18) - ?.isBefore(LocalDate.now()) - ?: true + .isLessThan18yrsAgo() + .not() - return if (is18yrsOfAge) { + return if (above18yrsOfAge) { scanInformation.status } else { ScanStatus.REJECTED } } + /** + * Return true if value is null or if it is more than 18 years ago. + */ + private fun LocalDate?.isLessThan18yrsAgo() = this + ?.plusYears(18) + ?.isAfter(LocalDate.now()) + ?: false // // eKYC - MyInfo // - private val myInfoKycV2Service by lazy { getResource("v2") } private val myInfoKycV3Service by lazy { getResource("v3") } private val secureArchiveService by lazy { getResource() } @@ -1820,34 +1826,35 @@ object Neo4jStoreSingleton : GraphStore { id = authorisationCode, message = "Failed to fetched MyInfo $version").left().bind() - // TODO vihang: Should we set status for NRIC_FIN to APPROVED? - - // set NRIC_FIN KYC Status to Approved -// setKycStatus( -// customerId = customerId, -// regionCode = "sg", -// kycType = NRIC_FIN).bind() - val personData = myInfoData.personData ?: SystemError( type = "MyInfo Auth Code", id = authorisationCode, message = "Failed to fetched MyInfo $version").left().bind() - secureArchiveService.archiveEncrypted( - customerId = customer.id, - fileName = "myInfoData", - regionCodes = listOf("sg"), - dataMap = mapOf( - "uinFin" to myInfoData.uinFin.toByteArray(), - "personData" to personData.toByteArray() - ) - ).bind() + val kycStatus = if(myInfoData.birthDate.isLessThan18yrsAgo()) { + AuditLog.warn(customerId = customer.id, message = "Customer age is less than 18yrs.") + REJECTED + } else { + // store data only if approved + secureArchiveService.archiveEncrypted( + customerId = customer.id, + fileName = "myInfoData", + regionCodes = listOf("sg"), + dataMap = mapOf( + "uinFin" to myInfoData.uinFin.toByteArray(), + "personData" to personData.toByteArray() + ) + ).bind() + KycStatus.APPROVED + } // set MY_INFO KYC Status to Approved setKycStatus( customer = customer, regionCode = "sg", - kycType = MY_INFO).bind() + kycType = MY_INFO, + kycStatus = kycStatus, + kycExpiryDate = myInfoData.passExpiryDate?.toString()).bind() personData }.fix() @@ -1946,6 +1953,7 @@ object Neo4jStoreSingleton : GraphStore { customer: Customer, regionCode: String, kycType: KycType, + kycExpiryDate: String? = null, kycStatus: KycStatus = KycStatus.APPROVED) = writeTransaction { setKycStatus( @@ -1953,6 +1961,7 @@ object Neo4jStoreSingleton : GraphStore { regionCode = regionCode, kycType = kycType, kycStatus = kycStatus, + kycExpiryDate = kycExpiryDate, transaction = transaction) .ifFailedThenRollback(transaction) } @@ -2020,10 +2029,18 @@ object Neo4jStoreSingleton : GraphStore { ).bind() } + val newKycExpiryDateMap = kycExpiryDate + ?.let { existingCustomerRegion.kycExpiryDateMap.copy(key = kycType, value = it) } + ?: existingCustomerRegion.kycExpiryDateMap + customerRegionRelationStore .createOrUpdate( fromId = customer.id, - relation = CustomerRegion(status = newStatus, kycStatusMap = newKycStatusMap), + relation = CustomerRegion( + status = newStatus, + kycStatusMap = newKycStatusMap, + kycExpiryDateMap = newKycExpiryDateMap + ), toId = regionCode, transaction = transaction) .bind() diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt index c1c6a1c26..2f5f9cc87 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt @@ -23,7 +23,8 @@ data class PlanSubscription( data class CustomerRegion( val status: CustomerRegionStatus, - val kycStatusMap: Map = emptyMap()) + val kycStatusMap: Map = emptyMap(), + val kycExpiryDateMap: Map = emptyMap()) data class SimProfile( override val id: String, diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/ekyc/KycServices.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/ekyc/KycServices.kt index a0421d1ab..7afa48e58 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/ekyc/KycServices.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/ekyc/KycServices.kt @@ -1,6 +1,7 @@ package org.ostelco.prime.ekyc import org.ostelco.prime.model.MyInfoConfig +import java.time.LocalDate interface MyInfoKycService { fun getConfig() : MyInfoConfig @@ -13,5 +14,7 @@ interface DaveKycService { data class MyInfoData( val uinFin: String, + val birthDate: LocalDate?, + val passExpiryDate: LocalDate?, val personData: String? ) \ No newline at end of file diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index a60fea6b2..786f826c1 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.71.1" +version = "1.72.0" dependencies { // interface module between prime and prime-modules diff --git a/prime/infra/dev/prime-customer-api.yaml b/prime/infra/dev/prime-customer-api.yaml index 25d247190..d4ec8999f 100644 --- a/prime/infra/dev/prime-customer-api.yaml +++ b/prime/infra/dev/prime-customer-api.yaml @@ -836,6 +836,17 @@ definitions: MY_INFO: APPROVED NRIC_FIN: REJECTED ADDRESS: PENDING + kycExpiryDateMap: + description: "Map of expiry date for each KYC" + type: object + properties: + kycType: + $ref: '#/definitions/KycType' + additionalProperties: + type: string + example: + JUMIO: "2040-12-31" + MY_INFO: "2030-12-31" simProfiles: $ref: '#/definitions/SimProfileList' KycType: diff --git a/prime/infra/prod/prime-customer-api.yaml b/prime/infra/prod/prime-customer-api.yaml index 3edb5896d..3fd0f3b0c 100644 --- a/prime/infra/prod/prime-customer-api.yaml +++ b/prime/infra/prod/prime-customer-api.yaml @@ -257,42 +257,6 @@ paths: description: "Region or Scan not found." security: - firebase: [] - "/regions/sg/kyc/myInfoConfig": - get: - description: "Get Singapore MyInfo v2 service config (deprecated)." - produces: - - application/json - operationId: "getMyInfoV2Config" - responses: - 200: - description: "MyInfo service config." - schema: - $ref: "#/definitions/MyInfoConfig" - 404: - description: "Config not found." - security: - - firebase: [] - "/regions/sg/kyc/myInfo/{authorisationCode}": - get: - description: "Get Customer Data from Singapore MyInfo v2 service (deprecated)." - produces: - - application/json - operationId: "getCustomerMyInfoV2Data" - parameters: - - name: authorisationCode - in: path - description: "Authorisation Code" - required: true - type: string - responses: - 200: - description: "Successfully retrieved Customer Data from MyInfo v2 service." - schema: - type: object - 404: - description: "Person Data not found." - security: - - firebase: [] "/regions/sg/kyc/myInfo/v3/config": get: description: "Get Singapore MyInfo v3 service config." @@ -872,6 +836,17 @@ definitions: MY_INFO: APPROVED NRIC_FIN: REJECTED ADDRESS: PENDING + kycExpiryDateMap: + description: "Map of expiry date for each KYC" + type: object + properties: + kycType: + $ref: '#/definitions/KycType' + additionalProperties: + type: string + example: + JUMIO: "2040-12-31" + MY_INFO: "2030-12-31" simProfiles: $ref: '#/definitions/SimProfileList' KycType: diff --git a/prime/script/start.sh b/prime/script/start.sh index f3cea2bd4..32dfc4e34 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.71.1,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.72.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From 6a93c762b2622e57106c989d7637a897738d355a Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 7 Nov 2019 11:24:22 +0100 Subject: [PATCH 23/27] Handling empty string for date values in MyInfo PersonData. --- .../ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt index db16dc33a..ed810bdbd 100644 --- a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt +++ b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt @@ -74,11 +74,21 @@ object MyInfoClientSingleton : MyInfoKycService { return MyInfoData( uinFin = uinFin, personData = personDataString, - birthDate = personData.dateOfBirth?.value?.let(LocalDate::parse), - passExpiryDate = personData.passExpiryDate?.value?.let(LocalDate::parse) + birthDate = personData.dateOfBirth.toLocalDate(), + passExpiryDate = personData.passExpiryDate.toLocalDate() ) } + private fun ValueDataItem?.toLocalDate(): LocalDate? { + val value = this?.value + return if (value.isNullOrBlank()) { + // value can be null or blank + null + } else { + value.let(LocalDate::parse) + } + } + private fun getToken(authorisationCode: String): String? = sendSignedRequest( httpMethod = POST, From cb12dd2b4890c72f6e6e7cd6eeb8848c8789eae0 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 7 Nov 2019 11:24:35 +0100 Subject: [PATCH 24/27] Added logging to debug issue of getting multiple products with same SKU. --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 71c34dbc0..acd86d8a0 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1027,13 +1027,20 @@ object Neo4jStoreSingleton : GraphStore { -[:${customerToSegmentRelation.name}]->(:${segmentEntity.name}) <-[:${offerToSegmentRelation.name}]-(:${offerEntity.name}) -[:${offerToProductRelation.name}]->(product:${productEntity.name} {sku: '$sku'}) - RETURN product; + RETURN DISTINCT product; """.trimIndent(), transaction) { statementResult -> - if (statementResult.hasNext()) { - Either.right(productEntity.createEntity(statementResult.single().get("product").asMap())) - } else { - Either.left(NotFoundError(type = productEntity.name, id = sku)) + + val products = statementResult + .list { productEntity.createEntity(it["product"].asMap()) } + .toList() + when(products.size) { + 0 -> NotFoundError(type = productEntity.name, id = sku).left() + 1 -> products.single().right() + else -> { + logger.warn("Found multiple products: {} with same sku:{} for customerId: {}", products, sku, customerId) + products.first().right() + } } } } From c085127ff82c6f99433b67264e3f16ffb42bc3d0 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 7 Nov 2019 11:25:46 +0100 Subject: [PATCH 25/27] Updated prime fix version --- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index 786f826c1..fffb141a4 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.72.0" +version = "1.72.1" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 32dfc4e34..86bf5ef82 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.72.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.72.1,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From e66ce870f3a3f7b835f64c50727fcc5530312e9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 11:45:40 +0100 Subject: [PATCH 26/27] Make the fix case insensitive (allowing lowercase F values) --- .../ostelco/simcards/inventory/SimInventoryCallbackService.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt index 842d39879..8f70d1602 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt @@ -30,7 +30,7 @@ class SimInventoryCallbackService(val dao: SimInventoryDAO) : SmDpPlusCallbackSe // Remove padding in ICCIDs with odd number of digits. // The database don't recognize those and will only get confused when // trying to use ICCIDs with trailing Fs as keys. - val iccid = incomingIccid.trimEnd('F') + val iccid = incomingIccid.toUpperCase().trimEnd('F') // If we can't find the ICCID, then cry foul and log an error message // that will get the ops team's attention asap! From dfac29b88760d333ac7fb2e60848de645fb419e2 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 8 Nov 2019 08:35:00 +0100 Subject: [PATCH 27/27] Using admin tool, identify customer using `jsonPayload.mdc.customerIdentity` in the logs --- .../support/resources/HoustonResources.kt | 20 +++++------ .../kotlin/org/ostelco/prime/dsl/Syntax.kt | 6 ++++ .../ostelco/prime/storage/graph/Neo4jStore.kt | 33 +++++++++++-------- .../org/ostelco/prime/storage/graph/Schema.kt | 4 +-- .../prime/ocs/notifications/Notifications.kt | 6 ++-- .../org/ostelco/prime/storage/Variants.kt | 5 +-- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- .../org/ostelco/tools/prime/admin/Main.kt | 7 ++++ .../prime/admin/actions/CustomerActions.kt | 28 +++++++++++++++- 10 files changed, 78 insertions(+), 35 deletions(-) diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt index ba4e2d87f..325250b61 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt @@ -8,7 +8,6 @@ import io.dropwizard.auth.Auth import org.ostelco.prime.apierror.ApiError import org.ostelco.prime.apierror.ApiErrorCode import org.ostelco.prime.apierror.ApiErrorCode.FAILED_TO_FETCH_AUDIT_LOGS -import org.ostelco.prime.apierror.ApiErrorMapper import org.ostelco.prime.apierror.InternalServerError import org.ostelco.prime.apierror.NotFoundError import org.ostelco.prime.apierror.responseBuilder @@ -32,7 +31,6 @@ import java.util.regex.Pattern import javax.validation.constraints.NotNull import javax.ws.rs.DELETE import javax.ws.rs.GET -import javax.ws.rs.POST import javax.ws.rs.PUT import javax.ws.rs.Path import javax.ws.rs.PathParam @@ -115,7 +113,7 @@ class ProfilesResource { private fun getAllScanInformation(customerId: String): Either> { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.getAllScanInformation(identity = identity) }.mapLeft { NotFoundError("Failed to fetch scan information.", ApiErrorCode.FAILED_TO_FETCH_SCAN_INFORMATION, it) @@ -162,10 +160,8 @@ class ProfilesResource { private fun getProfileListForMsisdn(msisdn: String): Either> { return try { - storage.getCustomerForMsisdn(msisdn).mapLeft { + storage.getCustomersForMsisdn(msisdn).mapLeft { NotFoundError("Failed to fetch profile.", ApiErrorCode.FAILED_TO_FETCH_CUSTOMER, it) - }.map { - listOf(it) } } catch (e: Exception) { logger.error("Failed to fetch profile for msisdn $msisdn", e) @@ -176,7 +172,7 @@ class ProfilesResource { // TODO: Reuse the one from SubscriberDAO private fun getSubscriptions(customerId: String): Either> { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.getSubscriptions(identity) }.mapLeft { NotFoundError("Failed to get subscriptions.", ApiErrorCode.FAILED_TO_FETCH_SUBSCRIPTIONS, it) @@ -216,7 +212,7 @@ class BundlesResource { // TODO: Reuse the one from SubscriberDAO private fun getBundles(customerId: String): Either> { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.getBundles(identity) }.mapLeft { NotFoundError("Failed to get bundles. ${it.message}", ApiErrorCode.FAILED_TO_FETCH_BUNDLES) @@ -256,7 +252,7 @@ class PurchaseResource { // TODO: Reuse the one from SubscriberDAO private fun getPurchaseHistory(customerId: String): Either> { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.getPurchaseRecords(identity) }.bimap( { NotFoundError("Failed to get purchase history.", ApiErrorCode.FAILED_TO_FETCH_PAYMENT_HISTORY, it) }, @@ -306,7 +302,7 @@ class RefundResource { private fun refundPurchase(customerId: String, purchaseRecordId: String, reason: String): Either { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.refundPurchase(identity, purchaseRecordId, reason) }.mapLeft { when (it) { @@ -350,7 +346,7 @@ class ContextResource { // TODO: Reuse the one from SubscriberDAO private fun getContext(customerId: String): Either { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.getCustomer(identity).map { customer -> storage.getAllRegionDetails(identity = identity) .fold( @@ -459,7 +455,7 @@ class CustomerResource { // TODO: Reuse the one from SubscriberDAO private fun removeCustomer(customerId: String): Either { return try { - storage.getIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> storage.removeCustomer(identity) }.mapLeft { NotFoundError("Failed to remove customer.", ApiErrorCode.FAILED_TO_REMOVE_CUSTOMER, it) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt index 0cf01877b..76dc11393 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt @@ -225,6 +225,12 @@ infix fun Customer.Companion.referredBy(customer: CustomerContext) = fromId = customer.id ) +infix fun Customer.Companion.withSubscription(subscription: SubscriptionContext) = + RelatedToClause( + relationType = subscriptionRelation, + toId = subscription.id + ) + // // ExCustomer // diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index acd86d8a0..11344bf27 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -33,6 +33,7 @@ import org.ostelco.prime.dsl.withId import org.ostelco.prime.dsl.withKyc import org.ostelco.prime.dsl.withMsisdn import org.ostelco.prime.dsl.withSku +import org.ostelco.prime.dsl.withSubscription import org.ostelco.prime.dsl.writeTransaction import org.ostelco.prime.ekyc.DaveKycService import org.ostelco.prime.ekyc.MyInfoKycService @@ -2100,21 +2101,11 @@ object Neo4jStoreSingleton : GraphStore { // Balance (Customer - Subscription - Bundle) // - override fun getCustomerForMsisdn(msisdn: String): Either = readTransaction { - read(""" - MATCH (customer:${customerEntity.name})-[:${subscriptionRelation.name}]->(subscription:${subscriptionEntity.name} {msisdn: '$msisdn'}) - RETURN customer - """.trimIndent(), - transaction) { - if (it.hasNext()) - Either.right(customerEntity.createEntity(it.single().get("customer").asMap())) - else - Either.left(NotFoundError(type = customerEntity.name, id = msisdn)) - } + override fun getCustomersForMsisdn(msisdn: String): Either> = readTransaction { + get(Customer withSubscription (Subscription withMsisdn msisdn)) } - - override fun getIdentityForCustomerId(id: String): Either = readTransaction { + override fun getAnyIdentityForCustomerId(id: String): Either = readTransaction { read(""" MATCH (:${customerEntity.name} { id:'$id' })<-[r:${identifiesRelation.name}]-(identity:${identityEntity.name}) RETURN identity, r.provider as provider @@ -2135,7 +2126,7 @@ object Neo4jStoreSingleton : GraphStore { read(""" MATCH (c:${customerEntity.name})<-[r:${identifiesRelation.name}]-(identity:${identityEntity.name}) WHERE c.contactEmail contains '$queryString' or c.nickname contains '$queryString' or c.id contains '$queryString' - RETURN c, identity, r.provider as provider + RETURN identity, r.provider as provider """.trimIndent(), transaction) { if (it.hasNext()) { @@ -2152,6 +2143,20 @@ object Neo4jStoreSingleton : GraphStore { } } + override fun getAllIdentities(): Either> = readTransaction { + read(""" + MATCH (:${customerEntity.name})<-[r:${identifiesRelation.name}]-(identity:${identityEntity.name}) + RETURN identity, r.provider as provider + """.trimIndent(), + transaction) { statementResult -> + statementResult.list { record -> + val identity = identityEntity.createEntity(record.get("identity").asMap()) + val provider = record.get("provider").asString() + ModelIdentity(id = identity.id, type = identity.type, provider = provider) + }.right() + } + } + // // For metrics // diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt index 0768af777..1ef0a5de1 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt @@ -548,7 +548,7 @@ object ObjectHandler { private fun toSimpleMap(map: Map, prefix: String = ""): Map { val outputMap: MutableMap = LinkedHashMap() - map.forEach { key, value -> + map.forEach { (key, value) -> when (value) { is Map<*, *> -> outputMap.putAll(toSimpleMap(value as Map, "$prefix$key$SEPARATOR")) is List<*> -> println("Skipping list value: $value for key: $key") @@ -569,7 +569,7 @@ object ObjectHandler { internal fun toNestedMap(map: Map): Map { val outputMap: MutableMap = LinkedHashMap() - map.forEach { key, value -> + map.forEach { (key, value) -> if (key.contains(SEPARATOR)) { val keys = key.split(SEPARATOR) var loopMap = outputMap diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/notifications/Notifications.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/notifications/Notifications.kt index 2f8f0af9f..633a2e1c0 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/notifications/Notifications.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/notifications/Notifications.kt @@ -14,8 +14,10 @@ object Notifications { val lowBalanceThreshold = ConfigRegistry.config.lowBalanceThreshold if ((balance < lowBalanceThreshold) && ((balance + reserved) > lowBalanceThreshold)) { // TODO martin : Title and message should differ depending on subscription - storage.getCustomerForMsisdn(msisdn).map { customer -> - appNotifier.notify(customer.id, "OYA", "You have less then " + lowBalanceThreshold / 1000000 + "Mb data left") + storage.getCustomersForMsisdn(msisdn).map { customers -> + customers.forEach { customer -> + appNotifier.notify(customer.id, "OYA", "You have less then " + lowBalanceThreshold / 1000000 + "Mb data left") + } } } } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt index 1c3e7c92e..5adbf6026 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt @@ -195,10 +195,11 @@ data class ConsumptionResult(val msisdnAnalyticsId: String, val granted: Long, v interface AdminGraphStore { - fun getCustomerForMsisdn(msisdn: String): Either + fun getCustomersForMsisdn(msisdn: String): Either> - fun getIdentityForCustomerId(id: String): Either + fun getAnyIdentityForCustomerId(id: String): Either fun getIdentitiesFor(queryString: String): Either> + fun getAllIdentities(): Either> /** * Link Customer to MSISDN diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index fffb141a4..1392467a2 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.72.1" +version = "1.72.2" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 86bf5ef82..bbc1c99d0 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.72.1,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.72.2,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml diff --git a/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/Main.kt b/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/Main.kt index a1730b088..25fb58cd9 100644 --- a/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/Main.kt +++ b/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/Main.kt @@ -6,6 +6,7 @@ import org.ostelco.tools.prime.admin.actions.approveRegionForCustomer import org.ostelco.tools.prime.admin.actions.createCustomer import org.ostelco.tools.prime.admin.actions.createSubscription import org.ostelco.tools.prime.admin.actions.getAllRegionDetails +import org.ostelco.tools.prime.admin.actions.identifyCustomer import org.ostelco.tools.prime.admin.actions.print import org.ostelco.tools.prime.admin.actions.printLeft import org.ostelco.tools.prime.admin.actions.setBalance @@ -121,4 +122,10 @@ fun doActions() { } +fun debug() { + + // identify customer using `jsonPayload.mdc.customerIdentity` in the logs + identifyCustomer(setOf("")) +} + data class SimProfileData(val iccId: String, val msisdn: String) \ No newline at end of file diff --git a/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt b/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt index 343e3d658..1c8635d3a 100644 --- a/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt +++ b/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt @@ -7,6 +7,7 @@ import arrow.core.left import arrow.core.right import arrow.effects.IO import arrow.instances.either.monad.monad +import org.apache.commons.codec.digest.DigestUtils import org.ostelco.prime.dsl.withId import org.ostelco.prime.dsl.writeTransaction import org.ostelco.prime.model.Bundle @@ -126,4 +127,29 @@ private fun emailAsIdentity(email: String) = Identity( id = email, type = "EMAIL", provider = "email" -) \ No newline at end of file +) + +fun identifyCustomer(idDigests: Set) = IO { + Either.monad().binding { + adminStore + // get all Identity values + .getAllIdentities() + .bind() + // map to + .map { String(Base64.getEncoder().encode(DigestUtils.sha256(it.id))) to it } + .toMap() + // filter on idDigests we want to find + .filterKeys { idDigests.contains(it) } + // find customer for each filtered Identity + .forEach { (idDigest, identity) -> + adminStore.getCustomer(identity).bimap( + { println("No Customer found for digest: $idDigest. - ${it.message}") }, + { println("Found Customer ID: ${it.id} for digest: $idDigest") } + ) + } + }.fix() + } + .unsafeRunSync() + .mapLeft { + println(it.message) + } \ No newline at end of file