From 4016f74a54eef5e8a8d1aca39cce2e7e2a541d61 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 2 Oct 2019 15:50:08 +0200 Subject: [PATCH 01/56] Remove the no longer used metrics from ocs gateway These metrics are no longer in use. --- .../datasource/protobuf/GrpcDataSource.java | 5 - .../datasource/protobuf/ProtobufDataSource.kt | 9 - .../ostelco/ocsgw/metrics/OcsgwMetrics.java | 176 ------------------ 3 files changed, 190 deletions(-) delete mode 100644 ocsgw/src/main/java/org/ostelco/ocsgw/metrics/OcsgwMetrics.java diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java index 7fe7194c4..ce35cc0f4 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java @@ -16,7 +16,6 @@ import org.ostelco.ocs.api.CreditControlRequestType; import org.ostelco.ocs.api.OcsServiceGrpc; import org.ostelco.ocsgw.datasource.DataSource; -import org.ostelco.ocsgw.metrics.OcsgwMetrics; import org.ostelco.ocsgw.utils.EventConsumer; import org.ostelco.ocsgw.utils.EventProducer; import org.slf4j.Logger; @@ -53,8 +52,6 @@ public class GrpcDataSource implements DataSource { private ServiceAccountJwtAccessCredentials jwtAccessCredentials; - private OcsgwMetrics ocsgwAnalytics; - private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); private ScheduledFuture reconnectStreamFuture = null; @@ -93,7 +90,6 @@ public GrpcDataSource( final String serviceAccountFile = "/config/" + System.getenv("SERVICE_FILE"); jwtAccessCredentials = ServiceAccountJwtAccessCredentials.fromStream(new FileInputStream(serviceAccountFile)); - ocsgwAnalytics = new OcsgwMetrics(metricsServerHostname, jwtAccessCredentials, protobufDataSource); producer = new EventProducer<>(requestQueue); } @@ -104,7 +100,6 @@ public void init() { initCreditControlRequestStream(); initActivateStream(); initKeepAlive(); - ocsgwAnalytics.initAnalyticsRequestStream(); setupEventConsumer(); } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt index bf4fc0939..ddf8e645f 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt @@ -15,8 +15,6 @@ import org.ostelco.diameter.model.SessionContext import org.ostelco.ocs.api.* import org.ostelco.ocsgw.OcsServer import org.ostelco.ocsgw.converter.ProtobufToDiameterConverter -import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport -import org.ostelco.prime.metrics.api.User import java.util.* import java.util.concurrent.ConcurrentHashMap @@ -107,13 +105,6 @@ class ProtobufDataSource { } } - fun getAnalyticsReport(): OcsgwAnalyticsReport { - val builder = OcsgwAnalyticsReport.newBuilder().setActiveSessions(sessionIdMap.size) - builder.keepAlive = false - sessionIdMap.forEach { msisdn, (_, _, _, apn, mccMnc) -> builder.addUsers(User.newBuilder().setApn(apn).setMccMnc(mccMnc).setMsisdn(msisdn).build()) } - return builder.build() - } - /** * A user will be blocked if one of the MSCC in the request could not be filled in the answer * or if the resultCode is not DIAMETER_SUCCESS diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/metrics/OcsgwMetrics.java b/ocsgw/src/main/java/org/ostelco/ocsgw/metrics/OcsgwMetrics.java deleted file mode 100644 index 7f59e74f2..000000000 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/metrics/OcsgwMetrics.java +++ /dev/null @@ -1,176 +0,0 @@ -package org.ostelco.ocsgw.metrics; - -import com.google.auth.oauth2.ServiceAccountJwtAccessCredentials; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.StatusRuntimeException; -import io.grpc.auth.MoreCallCredentials; -import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; -import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; -import io.grpc.stub.StreamObserver; -import org.ostelco.ocsgw.datasource.protobuf.ProtobufDataSource; -import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply; -import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport; -import org.ostelco.prime.metrics.api.OcsgwAnalyticsServiceGrpc; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.net.ssl.SSLException; -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.concurrent.Callable; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public class OcsgwMetrics { - - private static final Logger LOG = LoggerFactory.getLogger(OcsgwMetrics.class); - - private static final int KEEP_ALIVE_TIMEOUT_IN_MINUTES = 1; - - private static final int KEEP_ALIVE_TIME_IN_MINUTES = 20; - - private OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceStub ocsgwAnalyticsServiceStub; - - private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); - - private ScheduledFuture initAnalyticsFuture = null; - - private ScheduledFuture sendAnalyticsFuture = null; - - private StreamObserver ocsgwAnalyticsReportStream; - - private ManagedChannel grpcChannel; - - private ServiceAccountJwtAccessCredentials credentials; - - private String metricsServerHostname; - - private ProtobufDataSource protobufDataSource; - - public OcsgwMetrics( - String metricsServerHostname, - ServiceAccountJwtAccessCredentials serviceAccountJwtAccessCredentials, - ProtobufDataSource protobufDataSource) { - - this.protobufDataSource = protobufDataSource; - credentials = serviceAccountJwtAccessCredentials; - this.metricsServerHostname = metricsServerHostname; - } - - private void setupChannel() { - - ManagedChannelBuilder channelBuilder; - - final boolean disableTls = Boolean.valueOf(System.getenv("DISABLE_TLS")); - - try { - if (disableTls) { - channelBuilder = ManagedChannelBuilder - .forTarget(metricsServerHostname) - .usePlaintext(); - } else { - final NettyChannelBuilder nettyChannelBuilder = NettyChannelBuilder - .forTarget(metricsServerHostname); - - - channelBuilder = Files.exists(Paths.get("/cert/metrics.crt")) - ? nettyChannelBuilder.sslContext( - GrpcSslContexts.forClient().trustManager(new File("/cert/metrics.crt")).build()) - .useTransportSecurity() - :nettyChannelBuilder.useTransportSecurity(); - - } - - if (grpcChannel != null) { - - grpcChannel.shutdownNow(); - try { - boolean isShutdown = grpcChannel.awaitTermination(3, TimeUnit.SECONDS); - LOG.info("grpcChannel is shutdown : " + isShutdown); - } catch (InterruptedException e) { - LOG.info("Error shutting down gRPC channel"); - } - } - - grpcChannel = channelBuilder - .keepAliveWithoutCalls(true) -// .keepAliveTimeout(KEEP_ALIVE_TIMEOUT_IN_MINUTES, TimeUnit.MINUTES) -// .keepAliveTime(KEEP_ALIVE_TIME_IN_MINUTES, TimeUnit.MINUTES) - .build(); - - ocsgwAnalyticsServiceStub = OcsgwAnalyticsServiceGrpc.newStub(grpcChannel) - .withCallCredentials(MoreCallCredentials.from(credentials));; - - } catch (SSLException e) { - LOG.warn("Failed to setup gRPC channel", e); - } - } - - public void initAnalyticsRequestStream() { - - setupChannel(); - - ocsgwAnalyticsReportStream = ocsgwAnalyticsServiceStub.ocsgwAnalyticsEvent( - new StreamObserver() { - - @Override - public void onNext(OcsgwAnalyticsReply value) { - // Ignore reply from Prime - } - - @Override - public void onError(Throwable t) { - LOG.error("AnalyticsRequestObserver error", t); - if (t instanceof StatusRuntimeException) { - reconnectAnalyticsReportStream(); - } - } - - @Override - public void onCompleted() { - // Nothing to do here - } - } - ); - initAutoReportAnalyticsReport(); - } - - - private void sendAnalyticsReport(OcsgwAnalyticsReport report) { - if (report != null) { - ocsgwAnalyticsReportStream.onNext(report); - } - } - - private void reconnectAnalyticsReportStream() { - LOG.debug("reconnectAnalyticsReportStream called"); - - if (initAnalyticsFuture != null) { - initAnalyticsFuture.cancel(true); - } - - LOG.debug("Schedule new Callable initAnalyticsRequest"); - initAnalyticsFuture = executorService.schedule((Callable) () -> { - initAnalyticsRequestStream(); - return "Called!"; - }, - 5, - TimeUnit.SECONDS); - } - - private void initAutoReportAnalyticsReport() { - - if (sendAnalyticsFuture == null) { - sendAnalyticsFuture = executorService.scheduleAtFixedRate(() -> { - sendAnalyticsReport(protobufDataSource.getAnalyticsReport()); - }, - 0, - 5, - TimeUnit.SECONDS); - } - } -} \ No newline at end of file From 24ba2bec85f5687ef9bedf864f3bd93656d9e2a3 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Thu, 3 Oct 2019 12:13:00 +0200 Subject: [PATCH 02/56] Removing no longer used metrics endpoints --- analytics-grpc-api/README.md | 1 - analytics-grpc-api/build.gradle.kts | 58 ------------------- .../src/main/proto/analytics.proto | 20 ------- .../src/main/proto/prime_metrics.proto | 27 --------- docker-compose.esp.yaml | 1 - docker-compose.seagull.yaml | 1 - docker-compose.yaml | 1 - ocsgw/docker-compose.yaml | 1 - .../main/java/org/ostelco/ocsgw/OcsServer.kt | 3 +- .../datasource/protobuf/GrpcDataSource.java | 4 +- .../org/ostelco/ocsgw/utils/AppConfig.java | 4 -- 11 files changed, 2 insertions(+), 119 deletions(-) delete mode 100644 analytics-grpc-api/README.md delete mode 100644 analytics-grpc-api/build.gradle.kts delete mode 100644 analytics-grpc-api/src/main/proto/analytics.proto delete mode 100644 analytics-grpc-api/src/main/proto/prime_metrics.proto diff --git a/analytics-grpc-api/README.md b/analytics-grpc-api/README.md deleted file mode 100644 index 6663d5b8f..000000000 --- a/analytics-grpc-api/README.md +++ /dev/null @@ -1 +0,0 @@ -# Module Analytics API diff --git a/analytics-grpc-api/build.gradle.kts b/analytics-grpc-api/build.gradle.kts deleted file mode 100644 index c551dc056..000000000 --- a/analytics-grpc-api/build.gradle.kts +++ /dev/null @@ -1,58 +0,0 @@ -import com.google.protobuf.gradle.generateProtoTasks -import com.google.protobuf.gradle.id -import com.google.protobuf.gradle.ofSourceSet -import com.google.protobuf.gradle.plugins -import com.google.protobuf.gradle.protobuf -import com.google.protobuf.gradle.protoc -import org.ostelco.prime.gradle.Version - -plugins { - `java-library` - id("com.google.protobuf") - idea -} - -dependencies { - api("io.grpc:grpc-netty-shaded:${Version.grpc}") - api("io.grpc:grpc-protobuf:${Version.grpc}") - api("io.grpc:grpc-stub:${Version.grpc}") - api("io.grpc:grpc-core:${Version.grpc}") - implementation("com.google.protobuf:protobuf-java:${Version.protoc}") - implementation("com.google.protobuf:protobuf-java-util:${Version.protoc}") - implementation("javax.annotation:javax.annotation-api:${Version.javaxAnnotation}") -} - -var protobufGeneratedFilesBaseDir: String = "" - -protobuf { - protoc { artifact = "com.google.protobuf:protoc:${Version.protoc}" } - plugins { - id("grpc") { - artifact = "io.grpc:protoc-gen-grpc-java:${Version.grpc}" - } - } - generateProtoTasks { - ofSourceSet("main").forEach { - it.plugins { - id("grpc") - } - } - } - protobufGeneratedFilesBaseDir = generatedFilesBaseDir -} - -java { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 -} - -idea { - module { - sourceDirs.addAll(files("${protobufGeneratedFilesBaseDir}/main/java")) - sourceDirs.addAll(files("${protobufGeneratedFilesBaseDir}/main/grpc")) - } -} - -tasks.clean { - delete(protobufGeneratedFilesBaseDir) -} diff --git a/analytics-grpc-api/src/main/proto/analytics.proto b/analytics-grpc-api/src/main/proto/analytics.proto deleted file mode 100644 index c1760772a..000000000 --- a/analytics-grpc-api/src/main/proto/analytics.proto +++ /dev/null @@ -1,20 +0,0 @@ -syntax = "proto3"; - -package org.ostelco.analytics.api; - -option java_multiple_files = true; -option java_package = "org.ostelco.analytics.api"; - -import "google/protobuf/timestamp.proto"; - -// This is used only to report to Analytics engine by Prime via Google Cloud Pub/Sub. -message User { - string msisdn = 1; - string apn = 2; - string mccMnc = 3; -} - -message ActiveUsersInfo { - repeated User users = 1; - google.protobuf.Timestamp timestamp = 2; -} diff --git a/analytics-grpc-api/src/main/proto/prime_metrics.proto b/analytics-grpc-api/src/main/proto/prime_metrics.proto deleted file mode 100644 index 318e4bdfb..000000000 --- a/analytics-grpc-api/src/main/proto/prime_metrics.proto +++ /dev/null @@ -1,27 +0,0 @@ -syntax = "proto3"; - -package org.ostelco.prime.metrics.api; - -option java_multiple_files = true; -option java_package = "org.ostelco.prime.metrics.api"; - -// This is used to report Analytics events from OCSgw to Prime - -service OcsgwAnalyticsService { - rpc OcsgwAnalyticsEvent (stream OcsgwAnalyticsReport) returns (OcsgwAnalyticsReply) {} -} - -message OcsgwAnalyticsReport { - uint32 activeSessions = 1; - repeated User users = 2; - bool keepAlive = 3; -} - -message User { - string msisdn = 1; - string apn = 2; - string mccMnc = 3; -} - -message OcsgwAnalyticsReply { -} \ No newline at end of file diff --git a/docker-compose.esp.yaml b/docker-compose.esp.yaml index db2967f11..68cd49b77 100644 --- a/docker-compose.esp.yaml +++ b/docker-compose.esp.yaml @@ -90,7 +90,6 @@ services: command: ["./wait_including_esps.sh"] environment: - OCS_GRPC_SERVER=ocs.dev.ostelco.org - - METRICS_GRPC_SERVER=metrics.dev.ostelco.org - SERVICE_FILE=prime-service-account.json - GOOGLE_CLOUD_PROJECT=${GCP_PROJECT_ID} - PUBSUB_EMULATOR_HOST=pubsub-emulator:8085 diff --git a/docker-compose.seagull.yaml b/docker-compose.seagull.yaml index 903e1b0a6..005e5d5af 100644 --- a/docker-compose.seagull.yaml +++ b/docker-compose.seagull.yaml @@ -51,7 +51,6 @@ services: environment: - DISABLE_TLS=true - OCS_GRPC_SERVER=prime:8082 - - METRICS_GRPC_SERVER=prime:8083 - SERVICE_FILE=prime-service-account.json - GOOGLE_CLOUD_PROJECT=${GCP_PROJECT_ID} - PUBSUB_EMULATOR_HOST=pubsub-emulator:8085 diff --git a/docker-compose.yaml b/docker-compose.yaml index 95737c45e..e4dd3d44c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -59,7 +59,6 @@ services: environment: - DISABLE_TLS=true - OCS_GRPC_SERVER=prime:8082 - - METRICS_GRPC_SERVER=prime:8083 - SERVICE_FILE=prime-service-account.json - GOOGLE_CLOUD_PROJECT=${GCP_PROJECT_ID} - PUBSUB_EMULATOR_HOST=pubsub-emulator:8085 diff --git a/ocsgw/docker-compose.yaml b/ocsgw/docker-compose.yaml index 103ebf4fe..c9aa67234 100644 --- a/ocsgw/docker-compose.yaml +++ b/ocsgw/docker-compose.yaml @@ -7,7 +7,6 @@ services: command: ["./start_dev.sh"] environment: - OCS_GRPC_SERVER=ocs.dev.oya.world - - METRICS_GRPC_SERVER=metrics.dev.oya.world - SERVICE_FILE=prime-service-account.json - GOOGLE_CLOUD_PROJECT=${GCP_PROJECT_ID} - PUBSUB_PROJECT_ID=${GCP_PROJECT_ID} diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.kt b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.kt index a7e4521fe..5db93c017 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.kt +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.kt @@ -164,8 +164,7 @@ object OcsServer { appConfig: AppConfig): GrpcDataSource = GrpcDataSource( protobufDataSource, - appConfig.grpcServer, - appConfig.metricsServer) + appConfig.grpcServer) private fun getPubSubDataSource( protobufDataSource: ProtobufDataSource, diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java index ce35cc0f4..2b64774d0 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java @@ -73,8 +73,7 @@ public class GrpcDataSource implements DataSource { */ public GrpcDataSource( final ProtobufDataSource protobufDataSource, - final String ocsServerHostname, - final String metricsServerHostname) throws IOException { + final String ocsServerHostname) throws IOException { this.protobufDataSource = protobufDataSource; @@ -82,7 +81,6 @@ public GrpcDataSource( LOG.info("Created GrpcDataSource"); LOG.info("ocsServerHostname : {}", ocsServerHostname); - LOG.info("metricsServerHostname : {}", metricsServerHostname); // Not using the standard GOOGLE_APPLICATION_CREDENTIALS for this // as we need to download the file using container credentials in diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java b/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java index 9482d2181..d94d5b1ac 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/utils/AppConfig.java @@ -65,10 +65,6 @@ public String getGrpcServer() { return getEnvProperty("OCS_GRPC_SERVER"); } - public String getMetricsServer() { - return getEnvProperty("METRICS_GRPC_SERVER"); - } - public String getPubSubProjectId() { return getEnvProperty("PUBSUB_PROJECT_ID"); } From dd0a9c3e9d31504724b7e3b746fb8bb597e632d3 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Thu, 3 Oct 2019 13:25:47 +0200 Subject: [PATCH 03/56] Remove metrics endpoint The endpoint it no longer used, as we are now running the OCS gateway in the cloud. --- .circleci/config.yml | 12 +--- .../prime/analytics/AnalyticsGrpcServer.kt | 56 ---------------- .../prime/analytics/AnalyticsGrpcService.kt | 65 ------------------- .../prime/analytics/AnalyticsModule.kt | 9 --- .../publishers/ActiveUsersPublisher.kt | 38 ----------- dataflow-pipelines/build.gradle.kts | 2 - docker-compose.esp.yaml | 20 ------ docs/TEST.md | 11 +--- ocsgw/build.gradle.kts | 1 - ocsgw/script/wait_including_esps.sh | 8 --- prime-modules/build.gradle.kts | 1 - prime/infra/LEGACY.md | 20 +----- run-full-regression-test.sh | 15 ----- settings.gradle.kts | 2 - 14 files changed, 5 insertions(+), 255 deletions(-) delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt delete mode 100644 analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt diff --git a/.circleci/config.yml b/.circleci/config.yml index 74d73980d..6e090a086 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,8 +70,6 @@ jobs: command: | scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org cp certs/ocs.dev.ostelco.org/nginx.crt ocsgw/cert/ocs.crt - scripts/generate-selfsigned-ssl-certs.sh metrics.dev.ostelco.org - cp certs/metrics.dev.ostelco.org/nginx.crt ocsgw/cert/metrics.crt - run: name: Acceptance Tests command: docker-compose up --build --abort-on-container-exit @@ -255,13 +253,10 @@ jobs: echo $GOOGLE_DEV_ENDPOINTS_CREDENTIALS > ${HOME}/gcloud-service-key.json gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json - sed -i 's/GCP_PROJECT_ID/'${DEV_PROJECT}'/g' prime/infra/dev/ocs-api.yaml - sed -i 's/GCP_PROJECT_ID/'${DEV_PROJECT}'/g' prime/infra/dev/metrics-api.yaml + sed -i 's/GCP_PROJECT_ID/'${DEV_PROJECT}'/g' prime/infra/dev/ocs-api.yaml python -m grpc_tools.protoc --include_imports --include_source_info --proto_path=ocs-grpc-api/src/main/proto --descriptor_set_out=ocs_descriptor.pb ocs.proto - python -m grpc_tools.protoc --include_imports --include_source_info --proto_path=analytics-grpc-api/src/main/proto --descriptor_set_out=metrics_descriptor.pb prime_metrics.proto gcloud endpoints services deploy ocs_descriptor.pb prime/infra/dev/ocs-api.yaml - gcloud endpoints services deploy metrics_descriptor.pb prime/infra/dev/metrics-api.yaml gcloud endpoints services deploy prime/infra/dev/prime-customer-api.yaml gcloud endpoints services deploy prime/infra/dev/prime-webhooks.yaml gcloud endpoints services deploy prime/infra/dev/prime-houston-api.yaml @@ -322,13 +317,10 @@ jobs: echo $GOOGLE_PROD_ENDPOINTS_CREDENTIALS > ${HOME}/gcloud-service-key.json gcloud auth activate-service-account --key-file=${HOME}/gcloud-service-key.json - sed -i 's/GCP_PROJECT_ID/'${PROD_PROJECT}'/g' prime/infra/prod/ocs-api.yaml - sed -i 's/GCP_PROJECT_ID/'${PROD_PROJECT}'/g' prime/infra/prod/metrics-api.yaml + sed -i 's/GCP_PROJECT_ID/'${PROD_PROJECT}'/g' prime/infra/prod/ocs-api.yaml python -m grpc_tools.protoc --include_imports --include_source_info --proto_path=ocs-grpc-api/src/main/proto --descriptor_set_out=ocs_descriptor.pb ocs.proto - python -m grpc_tools.protoc --include_imports --include_source_info --proto_path=analytics-grpc-api/src/main/proto --descriptor_set_out=metrics_descriptor.pb prime_metrics.proto gcloud endpoints services deploy ocs_descriptor.pb prime/infra/prod/ocs-api.yaml - gcloud endpoints services deploy metrics_descriptor.pb prime/infra/prod/metrics-api.yaml gcloud endpoints services deploy prime/infra/prod/prime-customer-api.yaml gcloud endpoints services deploy prime/infra/prod/prime-webhooks.yaml gcloud endpoints services deploy prime/infra/prod/prime-houston-api.yaml diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt deleted file mode 100644 index 10c30752d..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcServer.kt +++ /dev/null @@ -1,56 +0,0 @@ -package org.ostelco.prime.analytics - -import io.dropwizard.lifecycle.Managed -import io.grpc.BindableService -import io.grpc.Server -import io.grpc.ServerBuilder -import org.ostelco.prime.getLogger -import java.io.IOException - -/** - * This is Analytics Server running on gRPC protocol. - * Its startup and shutdown are managed by Dropwizard's lifecycle - * through the Managed interface. - * - */ -class AnalyticsGrpcServer(private val port: Int, service: BindableService) : Managed { - - private val logger by getLogger() - - // may add Transport Security with Certificates if needed. - // may add executor for control over number of threads - private val server: Server = ServerBuilder.forPort(port).addService(service).build() - - /** - * Startup is managed by Dropwizard's lifecycle. - * - * @throws IOException ... sometimes, perhaps. - */ - override fun start() { - server.start() - logger.info("Analytics Server started, listening for incoming gRPC traffic on {}", port) - } - - /** - * Shutdown is managed by Dropwizard's lifecycle. - * - * @throws InterruptedException When something goes wrong. - */ - override fun stop() { - logger.info("Stopping Analytics Server listening for gRPC traffic on {}", port) - server.shutdown() - blockUntilShutdown() - } - - /** - * Used for unit testing - */ - fun forceStop() { - logger.info("Stopping forcefully Analytics Server listening for gRPC traffic on {}", port) - server.shutdownNow() - } - - private fun blockUntilShutdown() { - server.awaitTermination() - } -} diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt deleted file mode 100644 index d7f4f84b4..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsGrpcService.kt +++ /dev/null @@ -1,65 +0,0 @@ -package org.ostelco.prime.analytics - -import io.grpc.stub.StreamObserver -import org.ostelco.prime.analytics.PrimeMetric.ACTIVE_SESSIONS -import org.ostelco.prime.analytics.metrics.CustomMetricsRegistry - -import org.ostelco.prime.analytics.publishers.ActiveUsersPublisher -import org.ostelco.prime.getLogger -import org.ostelco.prime.metrics.api.OcsgwAnalyticsReply -import org.ostelco.prime.metrics.api.OcsgwAnalyticsReport -import org.ostelco.prime.metrics.api.OcsgwAnalyticsServiceGrpc -import java.util.* - - -/** - * Serves incoming gRPC analytics requests. - * - * It's implemented as a subclass of [OcsServiceGrpc.OcsServiceImplBase] overriding - * methods that together implements the protocol described in the analytics protobuf - * file: analytics.proto - *` - * service OcsgwAnalyticsService { - * rpc OcsgwAnalyticsEvent (stream OcsgwAnalyticsReport) returns (OcsgwAnalyticsReply) {} - * } - */ - -class AnalyticsGrpcService : OcsgwAnalyticsServiceGrpc.OcsgwAnalyticsServiceImplBase() { - - private val logger by getLogger() - - /** - * Handles the OcsgwAnalyticsEvent message. - */ - override fun ocsgwAnalyticsEvent(ocsgwAnalyticsReply: StreamObserver): StreamObserver { - val streamId = newUniqueStreamId() - return StreamObserverForStreamWithId(streamId) - } - - private fun newUniqueStreamId(): String { - return UUID.randomUUID().toString() - } - - private inner class StreamObserverForStreamWithId internal constructor(private val streamId: String) : StreamObserver { - - /** - * This method gets called every time a new active session count is sent - * from the OCS GW. - * @param request provides current active session as a counter with a timestamp - */ - override fun onNext(request: OcsgwAnalyticsReport) { - if (!request.keepAlive) { - CustomMetricsRegistry.updateMetricValue(ACTIVE_SESSIONS, request.activeSessions.toLong()) - ActiveUsersPublisher.publish(request.usersList) - } - } - - override fun onError(t: Throwable) { - // TODO vihang: handle onError for stream observers - } - - override fun onCompleted() { - logger.info("AnalyticsGrpcService with streamId: {} completed", streamId) - } - } -} diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt index 7eb7c8cb2..77a0b468a 100644 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt +++ b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/AnalyticsModule.kt @@ -20,12 +20,7 @@ class AnalyticsModule : PrimeModule { CustomMetricsRegistry.init(env.metrics()) - val server = AnalyticsGrpcServer(8083, AnalyticsGrpcService()) - - env.lifecycle().manage(server) - // dropwizard starts Analytics events publisher - env.lifecycle().manage(ActiveUsersPublisher) env.lifecycle().manage(DataConsumptionInfoPublisher) env.lifecycle().manage(PurchasePublisher) env.lifecycle().manage(RefundPublisher) @@ -47,10 +42,6 @@ data class AnalyticsConfig( @JsonProperty("purchaseInfoTopicId") val purchaseInfoTopicId: String, - @NotBlank - @JsonProperty("activeUsersTopicId") - val activeUsersTopicId: String, - @NotBlank @JsonProperty("simProvisioningTopicId") val simProvisioningTopicId: String, diff --git a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt b/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt deleted file mode 100644 index c31dceff3..000000000 --- a/analytics-module/src/main/kotlin/org/ostelco/prime/analytics/publishers/ActiveUsersPublisher.kt +++ /dev/null @@ -1,38 +0,0 @@ -package org.ostelco.prime.analytics.publishers - -import com.google.protobuf.ByteString -import com.google.protobuf.util.JsonFormat -import com.google.protobuf.util.Timestamps -import com.google.pubsub.v1.PubsubMessage -import org.ostelco.analytics.api.ActiveUsersInfo -import org.ostelco.prime.analytics.ConfigRegistry -import org.ostelco.prime.metrics.api.User -import java.time.Instant - -/** - * This class publishes the active users information events to the Google Cloud Pub/Sub. - */ -object ActiveUsersPublisher : - PubSubPublisher by DelegatePubSubPublisher(topicId = ConfigRegistry.config.activeUsersTopicId) { - - private val jsonPrinter = JsonFormat.printer().includingDefaultValueFields() - - private fun convertToJson(activeUsersInfo: ActiveUsersInfo): ByteString = - ByteString.copyFromUtf8(jsonPrinter.print(activeUsersInfo)) - - fun publish(userList: List) { - val timestamp = Instant.now().toEpochMilli() - val activeUsersInfoBuilder = ActiveUsersInfo.newBuilder().setTimestamp(Timestamps.fromMillis(timestamp)) - for (user in userList) { - val userBuilder = org.ostelco.analytics.api.User.newBuilder() - activeUsersInfoBuilder.addUsers(userBuilder.setApn(user.apn).setMccMnc(user.mccMnc).setMsisdn(user.msisdn).build()) - } - - val pubsubMessage = PubsubMessage.newBuilder() - .setData(convertToJson(activeUsersInfoBuilder.build())) - .build() - - // schedule a message to be published, messages are automatically batched - publishPubSubMessage(pubsubMessage) - } -} diff --git a/dataflow-pipelines/build.gradle.kts b/dataflow-pipelines/build.gradle.kts index 3581f69d3..ba0b0978e 100644 --- a/dataflow-pipelines/build.gradle.kts +++ b/dataflow-pipelines/build.gradle.kts @@ -11,8 +11,6 @@ plugins { dependencies { implementation(kotlin("stdlib-jdk8")) - implementation(project(":analytics-grpc-api")) - implementation("com.google.cloud:google-cloud-pubsub:${Version.googleCloudPubSub}") implementation("org.apache.beam:beam-sdks-java-core:${Version.beam}") diff --git a/docker-compose.esp.yaml b/docker-compose.esp.yaml index 68cd49b77..d854704b7 100644 --- a/docker-compose.esp.yaml +++ b/docker-compose.esp.yaml @@ -62,26 +62,6 @@ services: ipv4_address: 172.16.238.4 default: - metrics-esp: - container_name: metrics-esp - image: gcr.io/endpoints-release/endpoints-runtime:1 - volumes: - - "./prime/config:/esp" - - "./certs/metrics.dev.ostelco.org:/etc/nginx/ssl" - command: > - --service=metrics.dev.ostelco.org - --rollout_strategy=managed - --http2_port=80 - --ssl_port=443 - --backend=grpc://172.16.238.5:8083 - --service_account_key=/esp/prime-service-account.json - networks: - net: - aliases: - - "metrics.dev.ostelco.org" - ipv4_address: 172.16.238.6 - default: - ocsgw: container_name: ocsgw build: ocsgw diff --git a/docs/TEST.md b/docs/TEST.md index d7abc5c5a..b29b53be3 100644 --- a/docs/TEST.md +++ b/docs/TEST.md @@ -19,16 +19,7 @@ grep -i prime-service-account $(find . -name '.gitignore') | awk -F: '{print $1} cd certs/ocs.dev.ostelco.org openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=ocs.dev.ostelco.org' cp nginx.crt ../../ocsgw/cert/ocs.crt -``` - * Create self-signed certificate for nginx with domain as `metrics.dev.ostelco.org` and place them at following location: - * In `certs/metrics.dev.ostelco.org`, keep `nginx.key` and `nginx.cert`. - * In `ocsgw/cert`, keep `metrics.cert`. - -```bash -cd certs/metrics.dev.ostelco.org -openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.dev.ostelco.org' -cp nginx.crt ../../ocsgw/cert/metrics.crt -``` +`` * Set Stripe API key as env variable - `STRIPE_API_KEY`. Note: It is the key denoted as "Secret key" that shuld be set in this env variable. diff --git a/ocsgw/build.gradle.kts b/ocsgw/build.gradle.kts index 7e66859d4..12518ab58 100644 --- a/ocsgw/build.gradle.kts +++ b/ocsgw/build.gradle.kts @@ -12,7 +12,6 @@ dependencies { implementation(kotlin("stdlib-jdk8")) implementation(project(":ocs-grpc-api")) - implementation(project(":analytics-grpc-api")) implementation(project(":diameter-stack")) implementation(project(":diameter-ha")) diff --git a/ocsgw/script/wait_including_esps.sh b/ocsgw/script/wait_including_esps.sh index 3d56017b1..6e9747165 100755 --- a/ocsgw/script/wait_including_esps.sh +++ b/ocsgw/script/wait_including_esps.sh @@ -18,14 +18,6 @@ done echo "ESP launched" -echo "OCSGW waiting Metrics ESP to launch on 80..." - -while ! nc -z metrics.dev.ostelco.org 80; do - sleep 0.1 # wait for 1/10 of the second before check again -done - -echo "Metrics ESP launched" - # Start app for testing exec java \ -Dfile.encoding=UTF-8 \ diff --git a/prime-modules/build.gradle.kts b/prime-modules/build.gradle.kts index 176019bb7..ca4c82436 100644 --- a/prime-modules/build.gradle.kts +++ b/prime-modules/build.gradle.kts @@ -17,7 +17,6 @@ dependencies { api("com.fasterxml.jackson.module:jackson-module-kotlin:${Version.jackson}") api(project(":ocs-grpc-api")) - api(project(":analytics-grpc-api")) api(project(":model")) api("io.dropwizard:dropwizard-core:${Version.dropwizard}") diff --git a/prime/infra/LEGACY.md b/prime/infra/LEGACY.md index eb009d4a6..627432d6c 100644 --- a/prime/infra/LEGACY.md +++ b/prime/infra/LEGACY.md @@ -80,7 +80,7 @@ Reference: ## Endpoint -Generate self-contained protobuf descriptor file - `ocs_descriptor.pb` & `metrics_descriptor.pb` +Generate self-contained protobuf descriptor file - `ocs_descriptor.pb` ```bash pyenv versions @@ -94,19 +94,12 @@ python -m grpc_tools.protoc \ --descriptor_set_out=ocs_descriptor.pb \ ocs.proto -python -m grpc_tools.protoc \ - --include_imports \ - --include_source_info \ - --proto_path=analytics-grpc-api/src/main/proto \ - --descriptor_set_out=metrics_descriptor.pb \ - prime_metrics.proto ``` Deploy endpoints ```bash gcloud endpoints services deploy ocs_descriptor.pb prime/infra/prod/ocs-api.yaml -gcloud endpoints services deploy metrics_descriptor.pb prime/infra/prod/metrics-api.yaml ``` ## Deployment & Service @@ -152,7 +145,7 @@ kubectl describe service prime-service gcloud endpoints services deploy prime/infra/prod/prime-customer-api.yaml ``` -## SSL secrets for api.ostelco.org, ocs.ostelco.org & metrics.ostelco.org +## SSL secrets for api.ostelco.org, ocs.ostelco.org The endpoints runtime expects the SSL configuration to be named as `nginx.crt` and `nginx.key`. Sample command to create the secret: ```bash @@ -163,7 +156,6 @@ kubectl create secret generic api-ostelco-ssl \ The secret for ... * `api.ostelco.org` is in `api-ostelco-ssl` * `ocs.ostelco.org` is in `ocs-ostelco-ssl` - * `metrics.ostelco.org` is in `metrics-ostelco-ssl` # For Dev cluster @@ -210,20 +202,12 @@ python -m grpc_tools.protoc \ --proto_path=ocs-grpc-api/src/main/proto \ --descriptor_set_out=ocs_descriptor.pb \ ocs.proto - -python -m grpc_tools.protoc \ - --include_imports \ - --include_source_info \ - --proto_path=analytics-grpc-api/src/main/proto \ - --descriptor_set_out=metrics_descriptor.pb \ - prime_metrics.proto ``` Deploy endpoints ```bash gcloud endpoints services deploy ocs_descriptor.pb prime/infra/dev/ocs-api.yaml -gcloud endpoints services deploy metrics_descriptor.pb prime/infra/dev/metrics-api.yaml ``` * Client API HTTP endpoint diff --git a/run-full-regression-test.sh b/run-full-regression-test.sh index 265e8016c..9b3e868a9 100755 --- a/run-full-regression-test.sh +++ b/run-full-regression-test.sh @@ -50,21 +50,6 @@ if [[ ! -f "ocsgw/cert/ocs.crt" ]] ; then cp nginx.crt ../../ocsgw/cert/ocs.crt) fi - -# -# If necessary, Create self-signed certificate for nginx with domain -# as `metrics.dev.ostelco.org` and place them at following location: -# * In `certs/metrics.dev.ostelco.org`, keep `nginx.key` and `nginx.cert`. -# In `ocsgw/cert`, keep `metrics.cert`. -# - -if [[ ! -f "ocsgw/cert/metrics.crt" ]] ; then - (cd certs/metrics.dev.ostelco.org; - openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout ./nginx.key -out ./nginx.crt -subj '/CN=metrics.dev.ostelco.org' ; - cp nginx.crt ../../ocsgw/cert/metrics.crt) -fi - - # # Then build (or not, we're using gradle) the whole system # diff --git a/settings.gradle.kts b/settings.gradle.kts index a795c0a98..41114a341 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -2,7 +2,6 @@ rootProject.name = "ostelco-core" include(":acceptance-tests") include(":admin-endpoint") -include(":analytics-grpc-api") include(":analytics-module") include(":app-notifier") include(":appleid-auth-service") @@ -51,7 +50,6 @@ include(":sim-administration:sm-dp-plus-emulator") project(":acceptance-tests").projectDir = File("$rootDir/acceptance-tests") project(":admin-endpoint").projectDir = File("$rootDir/admin-endpoint") -project(":analytics-grpc-api").projectDir = File("$rootDir/analytics-grpc-api") project(":analytics-module").projectDir = File("$rootDir/analytics-module") project(":app-notifier").projectDir = File("$rootDir/app-notifier") project(":appleid-auth-service").projectDir = File("$rootDir/appleid-auth-service") From cfc002e93258e277c56bae84ce652b260678c4cc Mon Sep 17 00:00:00 2001 From: mpeterss Date: Thu, 3 Oct 2019 13:58:49 +0200 Subject: [PATCH 04/56] Removing more analytics from ocsgw --- dataflow-pipelines/build.gradle.kts | 2 +- prime/config/config.yaml | 1 - prime/config/test.yaml | 1 - prime/src/integration-test/resources/config.yaml | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/dataflow-pipelines/build.gradle.kts b/dataflow-pipelines/build.gradle.kts index ba0b0978e..c736922b3 100644 --- a/dataflow-pipelines/build.gradle.kts +++ b/dataflow-pipelines/build.gradle.kts @@ -10,7 +10,7 @@ plugins { dependencies { implementation(kotlin("stdlib-jdk8")) - + implementation("com.google.cloud:google-cloud-pubsub:${Version.googleCloudPubSub}") implementation("org.apache.beam:beam-sdks-java-core:${Version.beam}") diff --git a/prime/config/config.yaml b/prime/config/config.yaml index e13b9a8cd..b66434e05 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -75,7 +75,6 @@ modules: projectId: ${GCP_PROJECT_ID} dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info - activeUsersTopicId: active-users simProvisioningTopicId: sim-provisioning subscriptionStatusUpdateTopicId: subscription-status-update refundsTopicId: analytics-refunds diff --git a/prime/config/test.yaml b/prime/config/test.yaml index 4c7f9a4de..83ebc1889 100644 --- a/prime/config/test.yaml +++ b/prime/config/test.yaml @@ -60,7 +60,6 @@ modules: projectId: ${GCP_PROJECT_ID} dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info - activeUsersTopicId: active-users simProvisioningTopicId: sim-provisioning subscriptionStatusUpdateTopicId: subscription-status-update refundsTopicId: analytics-refunds diff --git a/prime/src/integration-test/resources/config.yaml b/prime/src/integration-test/resources/config.yaml index 54bed830d..fce75f97a 100644 --- a/prime/src/integration-test/resources/config.yaml +++ b/prime/src/integration-test/resources/config.yaml @@ -23,7 +23,6 @@ modules: projectId: ${GCP_PROJECT_ID} dataTrafficTopicId: data-traffic purchaseInfoTopicId: purchase-info - activeUsersTopicId: active-users simProvisioningTopicId: sim-provisioning subscriptionStatusUpdateTopicId: subscription-status-update refundsTopicId: analytics-refunds From 6b4be690a721b33203de623aaaf25d84487f2b78 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 21 Nov 2019 14:54:07 +0100 Subject: [PATCH 05/56] Adds Postgres sql script to add timestamps to 'sim_entries' table --- .../simmanager/schema/add-timestamps.sql | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 sim-administration/simmanager/schema/add-timestamps.sql diff --git a/sim-administration/simmanager/schema/add-timestamps.sql b/sim-administration/simmanager/schema/add-timestamps.sql new file mode 100644 index 000000000..e89362a4b --- /dev/null +++ b/sim-administration/simmanager/schema/add-timestamps.sql @@ -0,0 +1,56 @@ +-- +-- Add timstamp column for hlrstate +-- +alter table sim_entries add column tshlrstate timestamp; +alter table sim_entries alter column tshlrstate set not null default now(); +create or replace function hlrstate_changed() +returns trigger as $$ +begin + if new.hlrstate != old.hlrstate then + new.tshlrstate := now(); + end if; +return new; +end; +$$ language plpgsql; +create trigger trigger_hlrstate +before update on sim_entries +for each row +execute procedure hlrstate_changed(); + +-- +-- Add timstamp column for smdpplusstate +-- +alter table sim_entries add column tssmdpplusstate timestamp; +alter table sim_entries alter column tssmdpplusstate not null set default now(); +create or replace function smdpplusstate_changed() +returns trigger as $$ +begin + if new.smdpplusstate != old.smdpplusstate then + new.tssmdpplusstate := now(); + end if; +return new; +end; +$$ language plpgsql; +create trigger trigger_smdpplusstate +before update on sim_entries +for each row +execute procedure smdpplusstate_changed(); + +-- +-- Add timstamp column for provisionstate +-- +alter table sim_entries add column tsprovisionstate timestamp; +alter table sim_entries alter column tsprovisionstate not null set default now(); +create or replace function provisionstate_changed() +returns trigger as $$ +begin + if new.provisionstate != old.provisionstate then + new.tsprovisionstate := now(); + end if; +return new; +end; +$$ language plpgsql; +create trigger trigger_provisionstate +before update on sim_entries +for each row +execute procedure provisionstate_changed(); From 1d73073d1a26a779781b16f5b05c6883fe625c05 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 21 Nov 2019 15:11:53 +0100 Subject: [PATCH 06/56] Improved version of Postgres sql script to add timestamps --- .../simmanager/schema/add-timestamps.sql | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/sim-administration/simmanager/schema/add-timestamps.sql b/sim-administration/simmanager/schema/add-timestamps.sql index e89362a4b..4701f3a5f 100644 --- a/sim-administration/simmanager/schema/add-timestamps.sql +++ b/sim-administration/simmanager/schema/add-timestamps.sql @@ -2,55 +2,52 @@ -- Add timstamp column for hlrstate -- alter table sim_entries add column tshlrstate timestamp; -alter table sim_entries alter column tshlrstate set not null default now(); +alter table sim_entries alter column tshlrstate set default now(); create or replace function hlrstate_changed() returns trigger as $$ begin - if new.hlrstate != old.hlrstate then - new.tshlrstate := now(); - end if; -return new; + new.tshlrstate := now(); + return new; end; $$ language plpgsql; create trigger trigger_hlrstate before update on sim_entries for each row +when (old.hlrstate is distinct from new.hlrstate) execute procedure hlrstate_changed(); -- -- Add timstamp column for smdpplusstate -- alter table sim_entries add column tssmdpplusstate timestamp; -alter table sim_entries alter column tssmdpplusstate not null set default now(); +alter table sim_entries alter column tssmdpplusstate set default now(); create or replace function smdpplusstate_changed() returns trigger as $$ begin - if new.smdpplusstate != old.smdpplusstate then - new.tssmdpplusstate := now(); - end if; -return new; + new.tssmdpplusstate := now(); + return new; end; $$ language plpgsql; create trigger trigger_smdpplusstate before update on sim_entries for each row +when (old.smdpplusstate is distinct from new.smdpplusstate) execute procedure smdpplusstate_changed(); -- -- Add timstamp column for provisionstate -- alter table sim_entries add column tsprovisionstate timestamp; -alter table sim_entries alter column tsprovisionstate not null set default now(); +alter table sim_entries alter column tsprovisionstate set default now(); create or replace function provisionstate_changed() returns trigger as $$ begin - if new.provisionstate != old.provisionstate then - new.tsprovisionstate := now(); - end if; -return new; + new.tsprovisionstate := now(); + return new; end; $$ language plpgsql; create trigger trigger_provisionstate before update on sim_entries for each row +when (old.provisionstate is distinct from new.provisionstate) execute procedure provisionstate_changed(); From 747e8f9c97a97092f8affd66a62bb7592806d4e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Cederl=C3=B6f?= Date: Fri, 22 Nov 2019 14:35:40 +0100 Subject: [PATCH 07/56] Fix incorrectly fixed merge conflict --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc4c19f5d..9b4fe3d1b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,11 +66,11 @@ jobs: - . # generating selfsigned certs. Needed for docker compose tests - - run: - name: Generate self signed certs - command: | - scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org - cp certs/ocs.dev.ostelco.org/nginx.crt ocsgw/cert/ocs.crt +# - run: +# name: Generate self signed certs +# command: | +# scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org +# cp certs/ocs.dev.ostelco.org/nginx.crt ocsgw/cert/ocs.crt - run: name: Acceptance Tests command: docker-compose up --build --abort-on-container-exit From b31e04641ec2a1d2b1e305f71183451c46be1ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Cederl=C3=B6f?= Date: Fri, 22 Nov 2019 14:36:30 +0100 Subject: [PATCH 08/56] Fix whitespace --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b4fe3d1b..0f4289542 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ jobs: # - run: # name: Generate self signed certs # command: | -# scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org +# scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org # cp certs/ocs.dev.ostelco.org/nginx.crt ocsgw/cert/ocs.crt - run: name: Acceptance Tests From fae809ae062971f6f5be2a5cd0856d21fbfec598 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 22 Nov 2019 14:36:59 +0100 Subject: [PATCH 09/56] Refactors DB schema files a bit and adds some documentation --- sim-administration/postgres/README.md | 20 ++++++ .../postgres/add-timestamps.sql | 62 ++++++++++++++++++ sim-administration/postgres/init.sql | 65 ++++++++++++++++--- sim-administration/simmanager/SCHEMA.md | 4 ++ .../simmanager/schema/add-timestamps.sql | 53 --------------- 5 files changed, 142 insertions(+), 62 deletions(-) create mode 100644 sim-administration/postgres/README.md create mode 100644 sim-administration/postgres/add-timestamps.sql create mode 100644 sim-administration/simmanager/SCHEMA.md delete mode 100644 sim-administration/simmanager/schema/add-timestamps.sql diff --git a/sim-administration/postgres/README.md b/sim-administration/postgres/README.md new file mode 100644 index 000000000..df3cdbe8d --- /dev/null +++ b/sim-administration/postgres/README.md @@ -0,0 +1,20 @@ +# SIM Manager DB schema + +This directory contains the DB schema for the SIM manager PostgreSQL DB, and +Docker build file for acceptance tests involving the SIM manager DB. + +## DB schema + + * [Main schema definition](./init.sql) + * [Schema file for updating existing DBs with timestamps](./add-timestamps.sql) + * [Schema file for integration tests](../simmanager/src/integration-test/resources/init.sql) + +## Acceptance tests and the Docker image + +Referenced from the main [Docker compose](../../docker-compose.yaml) file and will be built +automatically as part of running the acceptance tests. + +## Integration test + +The [DB schema](../simmanager/src/integration-test/resources/init.sql) used in SIM manager +integration tests must be updated as needed on changes to the main [DB schema](./init.sql). diff --git a/sim-administration/postgres/add-timestamps.sql b/sim-administration/postgres/add-timestamps.sql new file mode 100644 index 000000000..359ddc154 --- /dev/null +++ b/sim-administration/postgres/add-timestamps.sql @@ -0,0 +1,62 @@ +/** + * Script for adding timestamp to existing DB. + * + * For new DB, use the 'init.sql' script. + */ + +-- +-- Add timstamp column for hlrState. +-- +alter table sim_entries add column tsHlrState timestamp; +alter table sim_entries alter column tsHlrState set default now(); +create or replace function hlrState_changed() +returns trigger as $$ +begin + new.tsHlrState := now(); + return new; +end; +$$ language plpgsql; + +create trigger trigger_hlrState +before update on sim_entries +for each row +when (old.hlrState is distinct from new.hlrState) +execute procedure hlrState_changed(); + +-- +-- Add timstamp column for smdpPlusState. +-- +alter table sim_entries add column tsSmdpPlusState timestamp; +alter table sim_entries alter column tsSmdpPlusState set default now(); +create or replace function smdpPlusState_changed() +returns trigger as $$ +begin + new.tsSmdpPlusState := now(); + return new; +end; +$$ language plpgsql; + +create trigger trigger_smdpPlusState +before update on sim_entries +for each row +when (old.smdpPlusState is distinct from new.smdpPlusState) +execute procedure smdpPlusState_changed(); + +-- +-- Add timstamp column for provisionState. +-- +alter table sim_entries add column tsProvisionState timestamp; +alter table sim_entries alter column tsProvisionState set default now(); +create or replace function provisionState_changed() +returns trigger as $$ +begin + new.tsProvisionState := now(); + return new; +end; +$$ language plpgsql; + +create trigger trigger_provisionState +before update on sim_entries +for each row +when (old.provisionState is distinct from new.provisionState) +execute procedure provisionState_changed(); diff --git a/sim-administration/postgres/init.sql b/sim-administration/postgres/init.sql index 00a3dc260..61d20e818 100644 --- a/sim-administration/postgres/init.sql +++ b/sim-administration/postgres/init.sql @@ -1,3 +1,10 @@ +/** + * SIM Manager DB schema. + */ + +-- +-- Create tables. +-- create table sim_import_batches (id bigserial primary key, status text, endedAt bigint, @@ -22,6 +29,9 @@ create table sim_entries (id bigserial primary key, pin2 varchar(4), puk1 varchar(8), puk2 varchar(8), + tsHlrState timestamp default now(), + tsSmdpPlusState timestamp default now(), + tsProvisionState timestamp default now(), UNIQUE (imsi), UNIQUE (iccid)); create table hlr_adapters (id bigserial primary key, @@ -35,16 +45,53 @@ create table sim_vendors_permitted_hlrs (id bigserial primary key, hlrId bigserial, UNIQUE (profileVendorId, hlrId)); +-- +-- Add trigger for updating tshlrState timestamp on changes to hlrState. +-- +create or replace function hlrState_changed() +returns trigger as $$ +begin + new.tsHlrState := now(); + return new; +end; +$$ language plpgsql; + +create trigger trigger_hlrState +before update on sim_entries +for each row +when (old.hlrState is distinct from new.hlrState) +execute procedure hlrState_changed(); +-- +-- Add trigger for updating tsSmdpPlusState timestamp on changes to smdpPlusState. +-- +create or replace function smdpPlusState_changed() +returns trigger as $$ +begin + new.tsSmdpPlusState := now(); + return new; +end; +$$ language plpgsql; --- dao.addProfileVendorAdapter("Bar") -INSERT INTO profile_vendor_adapters(name) VALUES ('Bar'); +create trigger trigger_smdpPlusState +before update on sim_entries +for each row +when (old.smdpPlusState is distinct from new.smdpPlusState) +execute procedure smdpPlusState_changed(); --- dao.addHssEntry("Foo") -INSERT INTO hlr_adapters(name) VALUES ('Foo'); +-- +-- Add trigger for updating tsProvisionState timestamp on chages to provisionState. +-- +create or replace function provisionState_changed() +returns trigger as $$ +begin + new.tsProvisionState := now(); + return new; +end; +$$ language plpgsql; --- dao.permitVendorForHssByNames(profileVendor = "Bar", hssName = "Foo") --- val profileVendorAdapter = getProfileVendorAdapterByName(profileVendor) --- val hlrAdapter = getHssEntryByName(hssName) --- storeSimVendorForHssPermission(profileVendorAdapter.id, hlrAdapter.id) -INSERT INTO sim_vendors_permitted_hlrs(profileVendorid, hlrId) VALUES (1, 1) +create trigger trigger_provisionState +before update on sim_entries +for each row +when (old.provisionState is distinct from new.provisionState) +execute procedure provisionState_changed(); diff --git a/sim-administration/simmanager/SCHEMA.md b/sim-administration/simmanager/SCHEMA.md new file mode 100644 index 000000000..e67b7e2cf --- /dev/null +++ b/sim-administration/simmanager/SCHEMA.md @@ -0,0 +1,4 @@ +# SIM Manager DB schema + +The schema file for the SIM Manager is located in the [../postgres/](../postgres/) directory. +See the [README](../postgres/README.md) file there for more information. diff --git a/sim-administration/simmanager/schema/add-timestamps.sql b/sim-administration/simmanager/schema/add-timestamps.sql deleted file mode 100644 index 4701f3a5f..000000000 --- a/sim-administration/simmanager/schema/add-timestamps.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- Add timstamp column for hlrstate --- -alter table sim_entries add column tshlrstate timestamp; -alter table sim_entries alter column tshlrstate set default now(); -create or replace function hlrstate_changed() -returns trigger as $$ -begin - new.tshlrstate := now(); - return new; -end; -$$ language plpgsql; -create trigger trigger_hlrstate -before update on sim_entries -for each row -when (old.hlrstate is distinct from new.hlrstate) -execute procedure hlrstate_changed(); - --- --- Add timstamp column for smdpplusstate --- -alter table sim_entries add column tssmdpplusstate timestamp; -alter table sim_entries alter column tssmdpplusstate set default now(); -create or replace function smdpplusstate_changed() -returns trigger as $$ -begin - new.tssmdpplusstate := now(); - return new; -end; -$$ language plpgsql; -create trigger trigger_smdpplusstate -before update on sim_entries -for each row -when (old.smdpplusstate is distinct from new.smdpplusstate) -execute procedure smdpplusstate_changed(); - --- --- Add timstamp column for provisionstate --- -alter table sim_entries add column tsprovisionstate timestamp; -alter table sim_entries alter column tsprovisionstate set default now(); -create or replace function provisionstate_changed() -returns trigger as $$ -begin - new.tsprovisionstate := now(); - return new; -end; -$$ language plpgsql; -create trigger trigger_provisionstate -before update on sim_entries -for each row -when (old.provisionstate is distinct from new.provisionstate) -execute procedure provisionstate_changed(); From be324a0b643487f4521a13c0c82abe6d110e1404 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 25 Nov 2019 13:49:19 +0100 Subject: [PATCH 10/56] Add new sim provisioning API to support endpoint --- prime/infra/dev/prime-houston-api.yaml | 32 +++++++++++++++++++++++++ prime/infra/prod/prime-houston-api.yaml | 32 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/prime/infra/dev/prime-houston-api.yaml b/prime/infra/dev/prime-houston-api.yaml index e815a2d45..dbf82b1b6 100644 --- a/prime/infra/dev/prime-houston-api.yaml +++ b/prime/infra/dev/prime-houston-api.yaml @@ -250,6 +250,38 @@ paths: description: "The customerId of the customer" required: true type: string + "/support/simprofile/{id}": + post: + description: "Provision SIM Profile for the user." + produces: + - application/json + operationId: "provisionSimProfile" + responses: + 201: + description: "Provisioned SIM profile for this user." + schema: + $ref: '#/definitions/SimProfile' + 400: + description: "Not allowed for this region, or missing parameters." + 500: + description: "Service Unavailable" + security: + - auth0_jwt: [] + parameters: + - name: id + in: path + description: "The customerId of the user" + required: true + type: string + - name: regionCode + in: query + description: "Region code" + required: true + type: string + - name: profileType + in: query + description: "Profile Type" + type: string definitions: Context: diff --git a/prime/infra/prod/prime-houston-api.yaml b/prime/infra/prod/prime-houston-api.yaml index 46640bf78..76fbd6d21 100644 --- a/prime/infra/prod/prime-houston-api.yaml +++ b/prime/infra/prod/prime-houston-api.yaml @@ -250,6 +250,38 @@ paths: description: "The customerId of the customer" required: true type: string + "/support/simprofile/{id}": + post: + description: "Provision SIM Profile for the user." + produces: + - application/json + operationId: "provisionSimProfile" + responses: + 201: + description: "Provisioned SIM profile for this user." + schema: + $ref: '#/definitions/SimProfile' + 400: + description: "Not allowed for this region, or missing parameters." + 500: + description: "Service Unavailable" + security: + - auth0_jwt: [] + parameters: + - name: id + in: path + description: "The customerId of the user" + required: true + type: string + - name: regionCode + in: query + description: "Region code" + required: true + type: string + - name: profileType + in: query + description: "Profile Type" + type: string definitions: Context: From b8a77f351888c2c2a47c123191e6685d220bf935 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 25 Nov 2019 13:54:06 +0100 Subject: [PATCH 11/56] Add new API to provision SIM via support UI --- .../ostelco/prime/support/SupportModule.kt | 1 + .../support/resources/HoustonResources.kt | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/SupportModule.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/SupportModule.kt index 1b4246085..29a4e8804 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/SupportModule.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/SupportModule.kt @@ -18,5 +18,6 @@ class SupportModule : PrimeModule { jerseySever.register(ContextResource()) jerseySever.register(AuditLogResource()) jerseySever.register(CustomerResource()) + jerseySever.register(SimProfilesResource()) } } \ No newline at end of file diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt index 14f9d3af0..58275f2c7 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt @@ -12,6 +12,7 @@ import org.ostelco.prime.apierror.InternalServerError import org.ostelco.prime.apierror.NotFoundError import org.ostelco.prime.apierror.responseBuilder import org.ostelco.prime.appnotifier.AppNotifier +import org.ostelco.prime.auditlog.AuditLog import org.ostelco.prime.auth.AccessTokenPrincipal import org.ostelco.prime.getLogger import org.ostelco.prime.model.Bundle @@ -20,6 +21,7 @@ import org.ostelco.prime.model.Customer import org.ostelco.prime.model.Identity import org.ostelco.prime.model.PurchaseRecord import org.ostelco.prime.model.ScanInformation +import org.ostelco.prime.model.SimProfile import org.ostelco.prime.model.Subscription import org.ostelco.prime.module.getResource import org.ostelco.prime.notifications.NOTIFY_OPS_MARKER @@ -27,11 +29,13 @@ import org.ostelco.prime.paymentprocessor.core.ForbiddenError import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.storage.AdminDataSource import org.ostelco.prime.storage.AuditLogStore +import org.ostelco.prime.tracing.EnableTracing import java.util.regex.Pattern import javax.validation.constraints.NotNull import javax.ws.rs.DELETE import javax.ws.rs.GET import javax.ws.rs.PUT +import javax.ws.rs.POST import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces @@ -464,3 +468,56 @@ class CustomerResource { } } } + +/** + * Resource used to provision new sim profiles. + */ +@Path("/support/simprofile") + +class SimProfilesResource { + private val logger by getLogger() + private val storage by lazy { getResource() } + + @EnableTracing + @POST + @Path("{id}") + @Produces(MediaType.APPLICATION_JSON) + fun provisionSimProfile(@Auth token: AccessTokenPrincipal?, + @NotNull + @PathParam("id") + id: String, + @QueryParam("regionCode") + regionCode: String, + @QueryParam("profileType") + profileType: String?): Response = + if (token == null) { + Response.status(Response.Status.UNAUTHORIZED) + } else { + logger.info("${token.name} Creating new SIM profile in region $regionCode & profileType $profileType for customerId: $id") + provisionSimProfile( + customerId =id, + regionCode = regionCode, + profileType = profileType) + .responseBuilder() + }.build() + + private fun provisionSimProfile(customerId: String, regionCode: String, profileType: String?): Either { + return try { + storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> + storage.provisionSimProfile(identity, regionCode, profileType).mapLeft { + AuditLog.error(identity, message = "Failed to provision SIM profile.") + it + }.map { + AuditLog.info(identity, message = "Provisioned new SIM with ICCID ${it.iccId} by support.") + it + } + }.mapLeft { + NotFoundError("Failed to provision SIM profile.", ApiErrorCode.FAILED_TO_PROVISION_SIM_PROFILE, it) + } + } catch (e: Exception) { + logger.error("Failed to provision SIM profile for customer with id - $customerId", e) + Either.left(NotFoundError("Failed to provision SIM profile", ApiErrorCode.FAILED_TO_PROVISION_SIM_PROFILE)) + } + } + +} From 93b9b2b9b57613cbb965cab3f76fd462b95ed464 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 25 Nov 2019 16:23:37 +0100 Subject: [PATCH 12/56] Add tests for the new endpoint --- .../kotlin/org/ostelco/at/jersey/Tests.kt | 51 +++++++++++++++++++ .../support/resources/HoustonResources.kt | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index acf11d708..f5332380d 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -283,6 +283,57 @@ class RegionsTest { StripePayment.deleteCustomer(customerId = customerId) } } + @Test + fun `jersey test - POST support simprofile - Sim profile from houston`() { + + val email = "regions-${randomInt()}@test.com" + var customerId = "" + try { + customerId = createCustomer(name = "Test Single Region User", email = email).id + enableRegion(email = email) + + val dataMap = MultivaluedHashMap() + dataMap["regionCode"] = listOf("no") + dataMap["profileType"] = listOf("iPhone") + + var regionDetailsList: Collection = get { + path = "/regions" + this.email = email + } + + assertEquals(2, regionDetailsList.size, "Customer should have 2 regions") + var receivedRegion = regionDetailsList.first() + assertEquals(APPROVED, receivedRegion.status, "Region status do not match") + val regionCode = receivedRegion.region.id + + val simProfile = post { + path = "/support/simprofile/$customerId" + this.email = email + this.queryParams = mapOf("regionCode" to regionCode, "profileType" to "iphone") + } + + regionDetailsList = get { + path = "/regions" + this.email = email + } + // Find the same Region + receivedRegion = regionDetailsList.find { + it.region.id == regionCode + }!! + + assertEquals( + 1, + receivedRegion.simProfiles.size, + "Should have only one sim profile") + + assertNotNull(receivedRegion.simProfiles.single().iccId) + assertEquals("", receivedRegion.simProfiles.single().alias) + assertNotNull(receivedRegion.simProfiles.single().eSimActivationCode) + assertEquals(simProfile.iccId, receivedRegion.simProfiles.single().iccId) + } finally { + StripePayment.deleteCustomer(customerId = customerId) + } + } } class SubscriptionsTest { diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt index 58275f2c7..192faa9ba 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt @@ -498,7 +498,7 @@ class SimProfilesResource { customerId =id, regionCode = regionCode, profileType = profileType) - .responseBuilder() + .responseBuilder(success = Response.Status.CREATED) }.build() private fun provisionSimProfile(customerId: String, regionCode: String, profileType: String?): Either { From 64656635ccd1a2c9d0ae0c137ad9997e7a0017f4 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 25 Nov 2019 19:46:59 +0100 Subject: [PATCH 13/56] Add alias parameter to the new API --- .../endpoint/resources/SimProfilesResource.kt | 3 ++- .../prime/customer/endpoint/store/SubscriberDAO.kt | 2 +- .../customer/endpoint/store/SubscriberDAOImpl.kt | 4 ++-- .../prime/support/resources/HoustonResources.kt | 11 +++++++---- .../org/ostelco/prime/storage/graph/Neo4jStore.kt | 5 +++-- .../org/ostelco/prime/storage/graph/Neo4jStoreTest.kt | 9 ++++++--- .../main/kotlin/org/ostelco/prime/storage/Variants.kt | 2 +- prime/infra/dev/prime-houston-api.yaml | 4 ++++ prime/infra/prod/prime-houston-api.yaml | 4 ++++ 9 files changed, 30 insertions(+), 14 deletions(-) diff --git a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/SimProfilesResource.kt b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/SimProfilesResource.kt index 5887486a2..f5d625800 100644 --- a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/SimProfilesResource.kt +++ b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/resources/SimProfilesResource.kt @@ -42,7 +42,8 @@ class SimProfilesResource(private val regionCode: String, private val dao: Subsc dao.provisionSimProfile( identity = token.identity, regionCode = regionCode, - profileType = profileType) + profileType = profileType, + alias = "") .responseBuilder() }.build() diff --git a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAO.kt b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAO.kt index 24c108887..0ce509958 100644 --- a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAO.kt +++ b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAO.kt @@ -61,7 +61,7 @@ interface SubscriberDAO { fun getSimProfiles(identity: Identity, regionCode: String): Either> - fun provisionSimProfile(identity: Identity, regionCode: String, profileType: String?): Either + fun provisionSimProfile(identity: Identity, regionCode: String, profileType: String?, alias: String): Either fun updateSimProfile(identity: Identity, regionCode: String, iccId: String, alias: String): Either diff --git a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt index 8c1c69fbb..d140d0537 100644 --- a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt +++ b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt @@ -190,9 +190,9 @@ class SubscriberDAOImpl : SubscriberDAO { } } - override fun provisionSimProfile(identity: Identity, regionCode: String, profileType: String?): Either { + override fun provisionSimProfile(identity: Identity, regionCode: String, profileType: String?, alias: String): Either { return try { - storage.provisionSimProfile(identity, regionCode, profileType).mapLeft { + storage.provisionSimProfile(identity, regionCode, profileType, alias).mapLeft { AuditLog.error(identity, message = "Failed to provision SIM profile.") NotFoundError("Failed to provision SIM profile.", ApiErrorCode.FAILED_TO_PROVISION_SIM_PROFILE, it) } diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt index 192faa9ba..1c5721acb 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt @@ -489,7 +489,9 @@ class SimProfilesResource { @QueryParam("regionCode") regionCode: String, @QueryParam("profileType") - profileType: String?): Response = + profileType: String, + @QueryParam("alias") + alias: String): Response = if (token == null) { Response.status(Response.Status.UNAUTHORIZED) } else { @@ -497,14 +499,15 @@ class SimProfilesResource { provisionSimProfile( customerId =id, regionCode = regionCode, - profileType = profileType) + profileType = profileType, + alias = alias) .responseBuilder(success = Response.Status.CREATED) }.build() - private fun provisionSimProfile(customerId: String, regionCode: String, profileType: String?): Either { + private fun provisionSimProfile(customerId: String, regionCode: String, profileType: String, alias: String): Either { return try { storage.getAnyIdentityForCustomerId(id = customerId).flatMap { identity: Identity -> - storage.provisionSimProfile(identity, regionCode, profileType).mapLeft { + storage.provisionSimProfile(identity, regionCode, profileType, alias).mapLeft { AuditLog.error(identity, message = "Failed to provision SIM profile.") it }.map { diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 4fa299e3a..bf534850d 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -747,7 +747,8 @@ object Neo4jStoreSingleton : GraphStore { override fun provisionSimProfile( identity: ModelIdentity, regionCode: String, - profileType: String?): Either = writeTransaction { + profileType: String?, + alias: String): Either = writeTransaction { IO { Either.monad().binding { val customerId = getCustomerId(identity = identity).bind() @@ -766,7 +767,7 @@ object Neo4jStoreSingleton : GraphStore { val simEntry = simManager.allocateNextEsimProfile(hlr = hssNameLookup.getHssName(region.id.toLowerCase()), phoneType = profileType) .mapLeft { NotFoundError("eSIM profile", id = "Loltel") } .bind() - val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId, requestedOn = utcTimeNow()) + val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId, alias = alias, requestedOn = utcTimeNow()) create { simProfile }.bind() fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() fact { (SimProfile withId simProfile.id) isFor (Region withCode regionCode.toLowerCase()) }.bind() diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt index 164cd0294..0fb287c2f 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt @@ -749,7 +749,8 @@ class Neo4jStoreTest { Neo4jStoreSingleton.provisionSimProfile( identity = IDENTITY, regionCode = REGION_CODE, - profileType = "default") + profileType = "default", + alias = "") .bimap( { fail(it.message) }, { @@ -956,7 +957,8 @@ class Neo4jStoreTest { Neo4jStoreSingleton.provisionSimProfile( identity = IDENTITY, regionCode = REGION_CODE, - profileType = "default") + profileType = "default", + alias = "") .mapLeft { fail(it.message) } // test @@ -1032,7 +1034,8 @@ class Neo4jStoreTest { Neo4jStoreSingleton.provisionSimProfile( identity = IDENTITY, regionCode = REGION_CODE, - profileType = "default") + profileType = "default", + alias = "") .mapLeft { fail(it.message) } // test diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt index 1453eb301..f54c1ef55 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/Variants.kt @@ -108,7 +108,7 @@ interface ClientGraphStore { /** * Provision new SIM Profile for Customer */ - fun provisionSimProfile(identity: Identity, regionCode: String, profileType: String?): Either + fun provisionSimProfile(identity: Identity, regionCode: String, profileType: String?, alias: String): Either /** * Update SIM Profile for Customer diff --git a/prime/infra/dev/prime-houston-api.yaml b/prime/infra/dev/prime-houston-api.yaml index dbf82b1b6..6e18d9f53 100644 --- a/prime/infra/dev/prime-houston-api.yaml +++ b/prime/infra/dev/prime-houston-api.yaml @@ -282,6 +282,10 @@ paths: in: query description: "Profile Type" type: string + - name: alias + in: query + description: "Name for the SIM" + type: string definitions: Context: diff --git a/prime/infra/prod/prime-houston-api.yaml b/prime/infra/prod/prime-houston-api.yaml index 76fbd6d21..bf3b5d320 100644 --- a/prime/infra/prod/prime-houston-api.yaml +++ b/prime/infra/prod/prime-houston-api.yaml @@ -282,6 +282,10 @@ paths: in: query description: "Profile Type" type: string + - name: alias + in: query + description: "Name for the SIM" + type: string definitions: Context: From 6504ca5ef836d353f75b83626e01302a52959497 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Mon, 25 Nov 2019 20:14:31 +0100 Subject: [PATCH 14/56] Add new paramter to the tests --- .../src/main/kotlin/org/ostelco/at/jersey/Tests.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index f5332380d..83c84a105 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -302,14 +302,16 @@ class RegionsTest { } assertEquals(2, regionDetailsList.size, "Customer should have 2 regions") - var receivedRegion = regionDetailsList.first() + var receivedRegion = regionDetailsList.find { + it.status == APPROVED + }!! assertEquals(APPROVED, receivedRegion.status, "Region status do not match") val regionCode = receivedRegion.region.id val simProfile = post { path = "/support/simprofile/$customerId" this.email = email - this.queryParams = mapOf("regionCode" to regionCode, "profileType" to "iphone") + this.queryParams = mapOf("regionCode" to regionCode, "profileType" to "iphone", "alias" to "") } regionDetailsList = get { From f6d3464ef00f6c8e700e4d12c6ab1e309bf1fb7f Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 26 Nov 2019 10:24:56 +0100 Subject: [PATCH 15/56] Add API docs --- .../org/ostelco/prime/support/resources/HoustonResources.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt index 1c5721acb..1ab32db4b 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt @@ -478,6 +478,9 @@ class SimProfilesResource { private val logger by getLogger() private val storage by lazy { getResource() } + /** + * Provision a new SIM card. + */ @EnableTracing @POST @Path("{id}") From f75414eab7a574ae1158bc81584be9cc305dd852 Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Tue, 26 Nov 2019 11:31:02 +0100 Subject: [PATCH 16/56] Cleanup tests --- .../kotlin/org/ostelco/at/jersey/Tests.kt | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt index 83c84a105..3a7df2991 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/jersey/Tests.kt @@ -48,6 +48,7 @@ import kotlin.test.assertFails import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue +import kotlin.test.fail class CustomerTest { @@ -292,23 +293,16 @@ class RegionsTest { customerId = createCustomer(name = "Test Single Region User", email = email).id enableRegion(email = email) - val dataMap = MultivaluedHashMap() - dataMap["regionCode"] = listOf("no") - dataMap["profileType"] = listOf("iPhone") - var regionDetailsList: Collection = get { path = "/regions" this.email = email } assertEquals(2, regionDetailsList.size, "Customer should have 2 regions") - var receivedRegion = regionDetailsList.find { - it.status == APPROVED - }!! - assertEquals(APPROVED, receivedRegion.status, "Region status do not match") + var receivedRegion = regionDetailsList.find { it.status == APPROVED } + ?: fail("Failed to find an approved region.") val regionCode = receivedRegion.region.id - - val simProfile = post { + val simProfile = post { path = "/support/simprofile/$customerId" this.email = email this.queryParams = mapOf("regionCode" to regionCode, "profileType" to "iphone", "alias" to "") @@ -319,9 +313,8 @@ class RegionsTest { this.email = email } // Find the same Region - receivedRegion = regionDetailsList.find { - it.region.id == regionCode - }!! + receivedRegion = regionDetailsList.find { it.region.id == regionCode } + ?: fail("Failed to find the region used to create sim profile.") assertEquals( 1, From 7232c58a65ae9621d969dbb74f679fcb7262dbea Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 26 Nov 2019 11:55:29 +0100 Subject: [PATCH 17/56] Added timestamps in SimProfile in GraphQL spec --- graphql/src/test/resources/customer.graphqls | 5 +++++ prime/config/customer.graphqls | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/graphql/src/test/resources/customer.graphqls b/graphql/src/test/resources/customer.graphqls index f057b6a45..f5e34de8f 100644 --- a/graphql/src/test/resources/customer.graphqls +++ b/graphql/src/test/resources/customer.graphqls @@ -63,6 +63,11 @@ type SimProfile { eSimActivationCode: String! status: SimProfileStatus! alias: String! + requestedOn: String + downloadedOn: String + installedOn: String + installedReportedByAppOn: String + deletedOn: String } enum SimProfileStatus { diff --git a/prime/config/customer.graphqls b/prime/config/customer.graphqls index f057b6a45..f5e34de8f 100644 --- a/prime/config/customer.graphqls +++ b/prime/config/customer.graphqls @@ -63,6 +63,11 @@ type SimProfile { eSimActivationCode: String! status: SimProfileStatus! alias: String! + requestedOn: String + downloadedOn: String + installedOn: String + installedReportedByAppOn: String + deletedOn: String } enum SimProfileStatus { From c18ea75a2a3764ea240d96162db181f74d0dc499 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 27 Nov 2019 14:22:06 +0100 Subject: [PATCH 18/56] Fix the check if the user is in the system We need to wait for the callback from the store before we create the CCA. --- .../ostelco/prime/ocs/core/OnlineCharging.kt | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index 5e1a1cbbf..3028fe5b7 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -80,15 +80,24 @@ object OnlineCharging : OcsAsyncRequestConsumer { responseBuilder.validityTime = 86400 storage.consume(msisdn, 0L, 0L) { storeResult -> storeResult.fold( - { responseBuilder.resultCode = ResultCode.DIAMETER_USER_UNKNOWN }, - { responseBuilder.resultCode = ResultCode.DIAMETER_SUCCESS }) + { + responseBuilder.resultCode = ResultCode.DIAMETER_USER_UNKNOWN + synchronized(OnlineCharging) { + returnCreditControlAnswer(responseBuilder.build()) + } + }, + { + responseBuilder.resultCode = ResultCode.DIAMETER_SUCCESS + synchronized(OnlineCharging) { + returnCreditControlAnswer(responseBuilder.build()) + } + }) } } else { chargeMSCCs(request, msisdn, responseBuilder) - } - - synchronized(OnlineCharging) { - returnCreditControlAnswer(responseBuilder.build()) + synchronized(OnlineCharging) { + returnCreditControlAnswer(responseBuilder.build()) + } } } } From 90ac05ca8cd6aa2998eaba7714c6578ae362ad98 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Wed, 27 Nov 2019 14:42:47 +0100 Subject: [PATCH 19/56] Add missing initial setup data for SIM manager DB for use in acceptance tests --- sim-administration/postgres/Dockerfile | 5 +++-- sim-administration/postgres/README.md | 4 ++++ .../postgres/setup-for-acceptance-test.sql | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 sim-administration/postgres/setup-for-acceptance-test.sql diff --git a/sim-administration/postgres/Dockerfile b/sim-administration/postgres/Dockerfile index bbd35c2c2..3256a7cd1 100644 --- a/sim-administration/postgres/Dockerfile +++ b/sim-administration/postgres/Dockerfile @@ -4,6 +4,7 @@ ENV POSTGRES_USER postgres_user ENV POSTGRES_PASSWORD postgres_password ENV POSTGRES_DB sim-inventory +# Note, as files are applied in alphabetic order care should be +# made to make sure that the 'init.sql' gets applied first. COPY init.sql /docker-entrypoint-initdb.d/ - - +COPY setup-for-acceptance-test.sql /docker-entrypoint-initdb.d/ diff --git a/sim-administration/postgres/README.md b/sim-administration/postgres/README.md index df3cdbe8d..617d829c2 100644 --- a/sim-administration/postgres/README.md +++ b/sim-administration/postgres/README.md @@ -14,6 +14,10 @@ Docker build file for acceptance tests involving the SIM manager DB. Referenced from the main [Docker compose](../../docker-compose.yaml) file and will be built automatically as part of running the acceptance tests. +Initial test data for the acceptance test are located in the +[setup-for-acceptance-test.sql](./setup-for-acceptance-test.sql) and will be added and set +up as part of the building of the Docker image for the tests. + ## Integration test The [DB schema](../simmanager/src/integration-test/resources/init.sql) used in SIM manager diff --git a/sim-administration/postgres/setup-for-acceptance-test.sql b/sim-administration/postgres/setup-for-acceptance-test.sql new file mode 100644 index 000000000..7abdb85d9 --- /dev/null +++ b/sim-administration/postgres/setup-for-acceptance-test.sql @@ -0,0 +1,15 @@ +-- +-- Setup for acceptance tests. +-- + +-- dao.addProfileVendorAdapter("Bar") +INSERT INTO profile_vendor_adapters(name) VALUES ('Bar'); + +-- dao.addHssEntry("Foo") +INSERT INTO hlr_adapters(name) VALUES ('Foo'); + +-- dao.permitVendorForHssByNames(profileVendor = "Bar", hssName = "Foo") +-- val profileVendorAdapter = getProfileVendorAdapterByName(profileVendor) +-- val hlrAdapter = getHssEntryByName(hssName) +-- storeSimVendorForHssPermission(profileVendorAdapter.id, hlrAdapter.id) +INSERT INTO sim_vendors_permitted_hlrs(profileVendorid, hlrId) VALUES (1, 1) From 0a2cf0b4deb96f8a81f8f64fa55d685cd6c4f578 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 27 Nov 2019 14:56:41 +0100 Subject: [PATCH 20/56] Refactor --- .../ostelco/prime/ocs/core/OnlineCharging.kt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index 3028fe5b7..c22413bcd 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -82,26 +82,27 @@ object OnlineCharging : OcsAsyncRequestConsumer { storeResult.fold( { responseBuilder.resultCode = ResultCode.DIAMETER_USER_UNKNOWN - synchronized(OnlineCharging) { - returnCreditControlAnswer(responseBuilder.build()) - } + sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) }, { responseBuilder.resultCode = ResultCode.DIAMETER_SUCCESS - synchronized(OnlineCharging) { - returnCreditControlAnswer(responseBuilder.build()) - } + sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) }) } } else { chargeMSCCs(request, msisdn, responseBuilder) - synchronized(OnlineCharging) { - returnCreditControlAnswer(responseBuilder.build()) - } + sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) } } } + private fun sendCreditControlAnswer(returnCreditControlAnswer: (CreditControlAnswerInfo) -> kotlin.Unit, + responseBuilder: CreditControlAnswerInfo.Builder) { + synchronized(OnlineCharging) { + returnCreditControlAnswer(responseBuilder.build()) + } + } + private suspend fun chargeMSCCs(request: CreditControlRequestInfo, msisdn: String, responseBuilder: CreditControlAnswerInfo.Builder) { From a23171649775a40b5c0af78f7d46991f8a6a7a10 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 27 Nov 2019 15:01:01 +0100 Subject: [PATCH 21/56] Refactor to one call to send answer --- .../kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index c22413bcd..6d4049d64 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -79,15 +79,14 @@ object OnlineCharging : OcsAsyncRequestConsumer { if (request.msccCount == 0) { responseBuilder.validityTime = 86400 storage.consume(msisdn, 0L, 0L) { storeResult -> - storeResult.fold( + responseBuilder.resultCode ==storeResult.fold( { - responseBuilder.resultCode = ResultCode.DIAMETER_USER_UNKNOWN - sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) + ResultCode.DIAMETER_USER_UNKNOWN }, { - responseBuilder.resultCode = ResultCode.DIAMETER_SUCCESS - sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) + ResultCode.DIAMETER_SUCCESS }) + sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) } } else { chargeMSCCs(request, msisdn, responseBuilder) From b7a6c46cd7e85062c4c273a53d5d8f257a92d6e5 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 27 Nov 2019 15:06:23 +0100 Subject: [PATCH 22/56] Refactor --- .../kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index 6d4049d64..d51b5f0c3 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -80,12 +80,8 @@ object OnlineCharging : OcsAsyncRequestConsumer { responseBuilder.validityTime = 86400 storage.consume(msisdn, 0L, 0L) { storeResult -> responseBuilder.resultCode ==storeResult.fold( - { - ResultCode.DIAMETER_USER_UNKNOWN - }, - { - ResultCode.DIAMETER_SUCCESS - }) + { ResultCode.DIAMETER_USER_UNKNOWN }, + { ResultCode.DIAMETER_SUCCESS }) sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) } } else { From baf93447389988c075a26274a5b863e19a08ed48 Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 27 Nov 2019 15:10:28 +0100 Subject: [PATCH 23/56] Fix typo --- .../main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index d51b5f0c3..2f19e77df 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -79,7 +79,7 @@ object OnlineCharging : OcsAsyncRequestConsumer { if (request.msccCount == 0) { responseBuilder.validityTime = 86400 storage.consume(msisdn, 0L, 0L) { storeResult -> - responseBuilder.resultCode ==storeResult.fold( + responseBuilder.resultCode = storeResult.fold( { ResultCode.DIAMETER_USER_UNKNOWN }, { ResultCode.DIAMETER_SUCCESS }) sendCreditControlAnswer(returnCreditControlAnswer, responseBuilder) From 7245674deee51c12925e655ed4ea71463a9311ac Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 28 Nov 2019 10:46:10 +0100 Subject: [PATCH 24/56] Formatting fixes --- sim-administration/postgres/README.md | 25 ++++++++++--------- .../postgres/add-timestamps.sql | 2 +- .../postgres/setup-for-acceptance-test.sql | 2 +- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/sim-administration/postgres/README.md b/sim-administration/postgres/README.md index 617d829c2..ef09f4afd 100644 --- a/sim-administration/postgres/README.md +++ b/sim-administration/postgres/README.md @@ -1,24 +1,25 @@ # SIM Manager DB schema -This directory contains the DB schema for the SIM manager PostgreSQL DB, and -Docker build file for acceptance tests involving the SIM manager DB. +This directory contains the DB schema for the SIM manager PostgreSQL DB, and Docker +build file for acceptance tests involving the SIM manager DB. ## DB schema - * [Main schema definition](./init.sql) - * [Schema file for updating existing DBs with timestamps](./add-timestamps.sql) - * [Schema file for integration tests](../simmanager/src/integration-test/resources/init.sql) +* [Main schema definition](./init.sql) +* [Schema file for updating existing DBs with timestamps](./add-timestamps.sql) +* [Schema file for integration tests](../simmanager/src/integration-test/resources/init.sql) -## Acceptance tests and the Docker image +## Acceptance tests -Referenced from the main [Docker compose](../../docker-compose.yaml) file and will be built -automatically as part of running the acceptance tests. +The file [Dockerfile](./Dockerfile) is used for building the DB image used in acceptance +tests. Referenced from the main [Docker compose](../../docker-compose.yaml) file and will +be built automatically as part of running the acceptance tests. -Initial test data for the acceptance test are located in the -[setup-for-acceptance-test.sql](./setup-for-acceptance-test.sql) and will be added and set -up as part of the building of the Docker image for the tests. +The file [setup-for-acceptance-test.sql](./setup-for-acceptance-test.sql) contains +initial test data for the acceptance tests and will be added and set up when building +the Docker image used in the tests. -## Integration test +## Integration tests The [DB schema](../simmanager/src/integration-test/resources/init.sql) used in SIM manager integration tests must be updated as needed on changes to the main [DB schema](./init.sql). diff --git a/sim-administration/postgres/add-timestamps.sql b/sim-administration/postgres/add-timestamps.sql index 359ddc154..9fd05a6cc 100644 --- a/sim-administration/postgres/add-timestamps.sql +++ b/sim-administration/postgres/add-timestamps.sql @@ -1,5 +1,5 @@ /** - * Script for adding timestamp to existing DB. + * Script for adding timestamps to existing DB. * * For new DB, use the 'init.sql' script. */ diff --git a/sim-administration/postgres/setup-for-acceptance-test.sql b/sim-administration/postgres/setup-for-acceptance-test.sql index 7abdb85d9..a1e7a41e9 100644 --- a/sim-administration/postgres/setup-for-acceptance-test.sql +++ b/sim-administration/postgres/setup-for-acceptance-test.sql @@ -1,5 +1,5 @@ -- --- Setup for acceptance tests. +-- Initial data set for acceptance tests. -- -- dao.addProfileVendorAdapter("Bar") From b39db3baf626bf855b19f61514f4cf4ee1fed1b5 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 28 Nov 2019 11:11:31 +0100 Subject: [PATCH 25/56] Fix missing cleartext reporting of the 'delete subscription' Stripe event --- .../prime/paymentprocessor/subscribers/Reporter.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt index 81a3b11d2..4f8a86dde 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt @@ -272,6 +272,14 @@ object Reporter { format("${email(subscription.customer)} subscribed to ${subscription.plan.id}", event) ) + event.type == "customer.subscription.updated" -> logger.info( + format("${email(subscription.customer)} subscription to ${subscription.plan.id} got updated", + event) + ) + event.type == "customer.subscription.deleted" -> logger.info(NOTIFY_OPS_MARKER, + format("${email(subscription.customer)} subscription to ${subscription.plan.id} got deleted", + event) + ) else -> logger.warn(format("Unhandled Stripe event ${event.type} (cat: Subscription)", event)) } @@ -280,6 +288,8 @@ object Reporter { private fun url(eventId: String): String = "https://dashboard.stripe.com/events/${eventId}" + /* TODO (kmm) Update to use the java.text.NumberFormat API or the new + JSR-354 Currency and Money API. */ private fun currency(amount: Long, currency: String): String = when (currency.toUpperCase()) { "SGD", "USD" -> "\$" From aa8b3914c5497851053e407ac3f017511eb7c5e3 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 28 Nov 2019 11:12:03 +0100 Subject: [PATCH 26/56] Adds customer.subscription.updated event to list of subscribed to Stripe events --- payment-processor/script/update-webhook.sh | 1 + prime/infra/stripe-monitor-task.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/payment-processor/script/update-webhook.sh b/payment-processor/script/update-webhook.sh index c02624e79..6d026b6ca 100755 --- a/payment-processor/script/update-webhook.sh +++ b/payment-processor/script/update-webhook.sh @@ -24,6 +24,7 @@ update_event_list() { -d enabled_events[]="customer.deleted" \ -d enabled_events[]="customer.subscription.created" \ -d enabled_events[]="customer.subscription.deleted" \ + -d enabled_events[]="customer.subscription.updated" \ -d enabled_events[]="invoice.created" \ -d enabled_events[]="invoice.deleted" \ -d enabled_events[]="invoice.finalized" \ diff --git a/prime/infra/stripe-monitor-task.yaml b/prime/infra/stripe-monitor-task.yaml index eeb8e47b4..7cf058489 100644 --- a/prime/infra/stripe-monitor-task.yaml +++ b/prime/infra/stripe-monitor-task.yaml @@ -28,6 +28,7 @@ spec: "customer.deleted", "customer.subscription.created", "customer.subscription.deleted", + "customer.subscription.updated", "invoice.created", "invoice.deleted", "invoice.finalized", From e8d44ca88db5774a3fb5b583fd5df1319017ca2f Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 27 Nov 2019 16:41:29 +0100 Subject: [PATCH 27/56] DSL for operations on Offer --- .../kotlin/org/ostelco/prime/dsl/Syntax.kt | 21 +++++++ .../org/ostelco/prime/dsl/Transactions.kt | 60 ++++++++----------- .../ostelco/prime/storage/graph/Neo4jStore.kt | 32 +++++++--- .../org/ostelco/prime/storage/graph/Schema.kt | 17 +++++- .../prime/storage/graph/model/Model.kt | 5 +- 5 files changed, 90 insertions(+), 45 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt index 98c40d27b..e5250e822 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt @@ -19,6 +19,8 @@ import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.exSubscriptionRelatio import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.forPurchaseByRelation import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.forPurchaseOfRelation import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.identifiesRelation +import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.offerToProductRelation +import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.offerToSegmentRelation import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.referredRelation import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.scanInformationRelation import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.simProfileRegionRelation @@ -29,6 +31,7 @@ import org.ostelco.prime.storage.graph.Neo4jStoreSingleton.subscriptionToBundleR import org.ostelco.prime.storage.graph.RelationType import org.ostelco.prime.storage.graph.model.ExCustomer import org.ostelco.prime.storage.graph.model.Identity +import org.ostelco.prime.storage.graph.model.Offer import org.ostelco.prime.storage.graph.model.Segment import org.ostelco.prime.storage.graph.model.SimProfile import kotlin.reflect.KClass @@ -174,6 +177,19 @@ data class PlanContext(override val id: String) : EntityContext(Plan::clas data class ProductContext(override val id: String) : EntityContext(Product::class, id) data class SegmentContext(override val id: String) : EntityContext(Segment::class, id) +data class OfferContext(override val id: String) : EntityContext(Offer::class, id) { + + infix fun isOfferedTo(segment: SegmentContext) = RelationExpression( + relationType = offerToSegmentRelation, + fromId = id, + toId = segment.id) + + infix fun containsProduct(product: ProductContext) = RelationExpression( + relationType = offerToProductRelation, + fromId = id, + toId = product.id) +} + data class PurchaseRecordContext(override val id: String) : EntityContext(PurchaseRecord::class, id) { infix fun forPurchaseBy(customer: CustomerContext) = RelationExpression( @@ -376,3 +392,8 @@ infix fun PurchaseRecord.Companion.forPurchaseOf(product: ProductContext) = // Segment // infix fun Segment.Companion.withId(id: String): SegmentContext = SegmentContext(id) + +// +// Offer +// +infix fun Offer.Companion.withId(id: String): OfferContext = OfferContext(id) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt index 37f1fcb13..f30ab9ef3 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt @@ -15,8 +15,6 @@ import org.ostelco.prime.storage.graph.EntityRegistry import org.ostelco.prime.storage.graph.EntityStore import org.ostelco.prime.storage.graph.Neo4jClient import org.ostelco.prime.storage.graph.PrimeTransaction -import org.ostelco.prime.storage.graph.Relation -import org.ostelco.prime.storage.graph.RelationRegistry import org.ostelco.prime.storage.graph.RelationStore import org.ostelco.prime.storage.graph.UniqueRelationStore import kotlin.reflect.KClass @@ -162,6 +160,20 @@ class WriteTransaction(override val transaction: PrimeTransaction) : ReadTransac } } } + + fun unlink(expression: () -> RelationExpression): Either { + val relationExpression = expression() + val relationStore = relationExpression.relationType.relationStore + return relationStore?.delete( + fromId = relationExpression.fromId, + toId = relationExpression.toId, + transaction = transaction + ) ?: SystemError( + type = "relationStore", + id = relationExpression.relationType.name, + message = "Missing relation store" + ).left() + } } class JobContext(private val transaction: PrimeTransaction) { @@ -170,17 +182,13 @@ class JobContext(private val transaction: PrimeTransaction) { fun create(obj: () -> E) { result = result.flatMap { - val entity: E = obj() - val entityStore: EntityStore = EntityRegistry.getEntityStore(entity::class) as EntityStore - entityStore.create(entity = entity, transaction = transaction) + WriteTransaction(transaction).create(obj) } } fun update(obj: () -> E) { result = result.flatMap { - val entity: E = obj() - val entityStore: EntityStore = EntityRegistry.getEntityStore(entity::class) as EntityStore - entityStore.update(entity = entity, transaction = transaction) + WriteTransaction(transaction).update(obj) } } @@ -191,33 +199,15 @@ class JobContext(private val transaction: PrimeTransaction) { } } - fun fact(fact: () -> RelationContext) { - val relationContext = fact() - val relationType = RelationRegistry.getRelationType(relationContext.relation) - when (val baseRelationStore = relationType?.relationStore) { - null -> { - } - is RelationStore<*, *, *> -> { - result = result.flatMap { - baseRelationStore.create( - fromId = relationContext.fromId, - toId = relationContext.toId, - transaction = transaction) - } - } - is UniqueRelationStore<*, *, *> -> { - result = result.flatMap { - baseRelationStore.create( - fromId = relationContext.fromId, - toId = relationContext.toId, - transaction = transaction) - } - } + fun fact(expression: () -> RelationExpression) { + result = result.flatMap { + WriteTransaction(transaction).fact(expression) } } -} -data class RelationContext( - val fromId: String, - val relation: Relation, - val toId: String) \ No newline at end of file + fun unlink(expression: () -> RelationExpression) { + result = result.flatMap { + WriteTransaction(transaction).unlink(expression) + } + } +} diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index bf534850d..5e273ea5a 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -2810,11 +2810,20 @@ object Neo4jStoreSingleton : GraphStore { private val segmentEntity = Segment::class.entityType - private val offerToSegmentRelation = RelationType(OFFERED_TO_SEGMENT, offerEntity, segmentEntity, None::class.java) - private val offerToSegmentStore = RelationStore(offerToSegmentRelation) + val offerToSegmentRelation = RelationType( + relation = OFFERED_TO_SEGMENT, + from = offerEntity, + to = segmentEntity, + dataClass = None::class.java) + .also { UniqueRelationStore(it) } - private val offerToProductRelation = RelationType(OFFER_HAS_PRODUCT, offerEntity, productEntity, None::class.java) - private val offerToProductStore = RelationStore(offerToProductRelation) + val offerToProductRelation = RelationType( + relation = OFFER_HAS_PRODUCT, + from = offerEntity, + to = productEntity, + dataClass = None::class.java + ) + .also { UniqueRelationStore(it) } val customerToSegmentRelation = RelationType(BELONG_TO_SEGMENT, customerEntity, segmentEntity, None::class.java) private val customerToSegmentStore = RelationStore(customerToSegmentRelation) @@ -2853,9 +2862,18 @@ object Neo4jStoreSingleton : GraphStore { } private fun WriteTransaction.createOffer(offer: ModelOffer): Either { - return create { Offer(id = offer.id) } - .flatMap { offerToSegmentStore.create(offer.id, offer.segments, transaction) } - .flatMap { offerToProductStore.create(offer.id, offer.products, transaction) } + return IO { + Either.monad().binding { + create { Offer(id = offer.id) }.bind() + for (segmentId in offer.segments) { + fact { (Offer withId offer.id) isOfferedTo (Segment withId segmentId) }.bind() + } + for (sku in offer.products) { + fact { (Offer withId offer.id) containsProduct (Product withSku sku) }.bind() + } + }.fix() + }.unsafeRunSync() + .ifFailedThenRollback(transaction) } // diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt index e9a966925..10eaf9b96 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt @@ -193,7 +193,9 @@ class EntityStore(private val entityType: EntityType) { } } -sealed class BaseRelationStore +sealed class BaseRelationStore { + abstract fun delete(fromId: String, toId: String, transaction: Transaction): Either +} // TODO vihang: check if relation already exists, with allow duplicate boolean flag param class RelationStore(private val relationType: RelationType) : BaseRelationStore() { @@ -317,6 +319,17 @@ class RelationStore(private val relationType // TODO vihang: validate if 'to' node exists Unit.right() } + + override fun delete(fromId: String, toId: String, transaction: Transaction): Either = write(""" + MATCH (:${relationType.from.name} { id: '$fromId'})-[r:${relationType.name}]->(:${relationType.to.name} {id: '$toId'}) + DELETE r + """.trimMargin(), + transaction) { statementResult -> + + Either.cond(statementResult.summary().counters().relationshipsDeleted() > 0, + ifTrue = { Unit }, + ifFalse = { NotDeletedError(relationType.name, "$fromId -> $toId") }) + } } // Removes double apostrophes from key values in a JSON string. @@ -465,7 +478,7 @@ class UniqueRelationStore(private val relati } } - fun delete(fromId: String, toId: String, transaction: Transaction): Either = write(""" + override fun delete(fromId: String, toId: String, transaction: Transaction): Either = write(""" MATCH (:${relationType.from.name} { id: '$fromId'})-[r:${relationType.name}]->(:${relationType.to.name} {id: '$toId'}) DELETE r """.trimMargin(), diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt index 9e32262b2..0ae23d368 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt @@ -61,7 +61,10 @@ data class Segment(override val id: String) : HasId { companion object } -data class Offer(override val id: String) : HasId +data class Offer(override val id: String) : HasId { + + companion object +} data class ExCustomer( override val id:String, From f2b21842527f6597055f8432b6e20f7d37b4cf8f Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 26 Nov 2019 14:19:42 +0100 Subject: [PATCH 28/56] Avoiding Read-Update-Write operations --- .../org/ostelco/prime/dsl/Transactions.kt | 18 +++++++++ .../ostelco/prime/storage/graph/Neo4jStore.kt | 39 ++++++++++--------- .../org/ostelco/prime/storage/graph/Schema.kt | 29 +++++++++++--- 3 files changed, 63 insertions(+), 23 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt index f30ab9ef3..00b892905 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt @@ -18,6 +18,7 @@ import org.ostelco.prime.storage.graph.PrimeTransaction import org.ostelco.prime.storage.graph.RelationStore import org.ostelco.prime.storage.graph.UniqueRelationStore import kotlin.reflect.KClass +import kotlin.reflect.KProperty1 object DSL { @@ -113,6 +114,23 @@ class WriteTransaction(override val transaction: PrimeTransaction) : ReadTransac return entityStore.update(entity = entity, transaction = transaction) } + fun update(entityContext: EntityContext, set: Pair, String?>): Either { + val entityStore: EntityStore = EntityRegistry.getEntityStore(entityContext.entityClass) + return entityStore.update(id = entityContext.id, properties = mapOf(set.first.name to set.second), transaction = transaction) + } + + fun update(entityContext: EntityContext, vararg set: Pair, String?>): Either { + val entityStore: EntityStore = EntityRegistry.getEntityStore(entityContext.entityClass) + val properties = set.map { it.first.name to it.second }.toMap() + return entityStore.update(id = entityContext.id, properties = properties, transaction = transaction) + } + + fun update(entityContext: EntityContext, set: Map, String?>): Either { + val entityStore: EntityStore = EntityRegistry.getEntityStore(entityContext.entityClass) + val properties = set.mapKeys { it.key.name } + return entityStore.update(id = entityContext.id, properties = properties, transaction = transaction) + } + fun delete(entityContext: EntityContext): Either { val entityStore: EntityStore = EntityRegistry.getEntityStore(entityContext.entityClass) return entityStore.delete(id = entityContext.id, transaction = transaction) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 5e273ea5a..e47d2c72f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1,5 +1,7 @@ package org.ostelco.prime.storage.graph +// Some of the model classes cannot be directly used in Graph Store as Entities. +// See documentation in [model/Model.kt] for more details. import arrow.core.Either import arrow.core.Either.Left import arrow.core.Either.Right @@ -144,8 +146,6 @@ import kotlin.collections.component1 import kotlin.collections.component2 import kotlin.collections.set import kotlin.reflect.KClass -// Some of the model classes cannot be directly used in Graph Store as Entities. -// See documentation in [model/Model.kt] for more details. import org.ostelco.prime.model.Identity as ModelIdentity import org.ostelco.prime.model.Offer as ModelOffer import org.ostelco.prime.model.Segment as ModelSegment @@ -441,11 +441,9 @@ object Neo4jStoreSingleton : GraphStore { getCustomer(identity = identity) .flatMap { existingCustomer -> - update { - existingCustomer.copy( - nickname = nickname ?: existingCustomer.nickname, - contactEmail = contactEmail ?: existingCustomer.contactEmail) - }.map { + update(Customer withId existingCustomer.id, + set = *arrayOf(Customer::nickname to nickname, Customer::contactEmail to contactEmail) + ).map { AuditLog.info(customerId = existingCustomer.id, message = "Updated nickname/contactEmail") } } @@ -708,11 +706,17 @@ object Neo4jStoreSingleton : GraphStore { customers.forEach { customer -> AuditLog.info(customerId = customer.id, message = "Sim Profile (iccId = $iccId) is $status") } - when(status) { - DOWNLOADED -> update { simProfile.copy(downloadedOn = utcTimeNow()) }.bind() - INSTALLED -> update { simProfile.copy(installedOn = utcTimeNow()) }.bind() - DELETED -> update { simProfile.copy(deletedOn = utcTimeNow()) }.bind() - else -> logger.warn("Not storing timestamp for simProfile: {} for status: {}", iccId, status) + val timestampField = when(status) { + DOWNLOADED -> SimProfile::downloadedOn + INSTALLED -> SimProfile::installedOn + DELETED -> SimProfile::deletedOn + else -> { + logger.warn("Not storing timestamp for simProfile: {} for status: {}", iccId, status) + null + } + } + if (timestampField != null) { + update(SimProfile withId simProfile.id, set = timestampField to utcTimeNow()).bind() } val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() subscriptions.forEach { subscription -> @@ -918,10 +922,9 @@ object Neo4jStoreSingleton : GraphStore { .firstOrNull { simProfile -> simProfile.iccId == iccId } ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() - val updatedSimProfile = simProfile.copy(alias = alias) - update { updatedSimProfile }.bind() + update(SimProfile withId simProfile.id, set = SimProfile::alias to alias).bind() AuditLog.info(customerId = customerId, message = "Updated alias of SIM Profile (iccId = $iccId)") - updatedSimProfile + simProfile.copy(alias = alias) }.fix() }.unsafeRunSync().ifFailedThenRollback(transaction) } @@ -972,10 +975,10 @@ object Neo4jStoreSingleton : GraphStore { .firstOrNull { simProfile -> simProfile.iccId == iccId } ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() - val updatedSimProfile = simProfile.copy(installedReportedByAppOn = utcTimeNow()) - update { updatedSimProfile }.bind() + val utcTimeNow = utcTimeNow() + update(SimProfile withId simProfile.id, set = SimProfile::installedReportedByAppOn to utcTimeNow).bind() AuditLog.info(customerId = customerId, message = "App reported SIM Profile (iccId = $iccId) as installed.") - updatedSimProfile + simProfile.copy(installedReportedByAppOn = utcTimeNow) }.fix() }.unsafeRunSync().ifFailedThenRollback(transaction) } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt index 10eaf9b96..585d624b0 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt @@ -163,6 +163,24 @@ class EntityStore(private val entityType: EntityType) { } } + fun update(id: String, properties: Map, transaction: Transaction): Either { + + val props = properties + .filterValues { it != null } + .mapValues { it.value.toString() } + val parameters: Map = mapOf("props" to props) + return exists(id, transaction).flatMap { + write(query = """MATCH (node:${entityType.name} { id: '$id' }) SET node += ${'$'}props ;""", + parameters = parameters, + transaction = transaction) { statementResult -> + Either.cond( + test = statementResult.summary().counters().containsUpdates(), // TODO vihang: this is not perfect way to check if updates are applied + ifTrue = {}, + ifFalse = { NotUpdatedError(type = entityType.name, id = id) }) + } + } + } + fun delete(id: String, transaction: Transaction): Either = exists(id, transaction).flatMap { write("""MATCH (node:${entityType.name} {id: '$id'} ) DETACH DELETE node;""", @@ -268,6 +286,7 @@ class RelationStore(private val relationType } } + // TODO vihang: use parameters fun create(fromId: String, toIds: Collection, transaction: Transaction): Either = write(""" MATCH (to:${relationType.to.name}) WHERE to.id in [${toIds.joinToString(",") { "'$it'" }}] @@ -289,6 +308,7 @@ class RelationStore(private val relationType }) } + // TODO vihang: use parameters fun create(fromIds: Collection, toId: String, transaction: Transaction): Either = write(""" MATCH (from:${relationType.from.name}) WHERE from.id in [${fromIds.joinToString(",") { "'$it'" }}] @@ -332,10 +352,6 @@ class RelationStore(private val relationType } } -// Removes double apostrophes from key values in a JSON string. -// Usage: output = re.replace(input, "$1$2$3") -val re = Regex("""([,{])\s*"([^"]+)"\s*(:)""") - class UniqueRelationStore(private val relationType: RelationType) : BaseRelationStore() { init { @@ -383,6 +399,7 @@ class UniqueRelationStore(private val relati { Unit.right() }, { val properties = getProperties(relation as Any) + // TODO vihang: set props using parameter val strProps: String = properties.entries.joinToString(",") { """`${it.key}`: "${it.value}"""" } write(""" @@ -412,7 +429,7 @@ class UniqueRelationStore(private val relati val properties = getProperties(relation as Any) doNotExist(fromId, toId, transaction).fold( { - // TODO vihang: replace setClause with map based settings written by Kjell + // TODO vihang: set props using parameter val setClause: String = properties.entries.fold("") { acc, entry -> """$acc SET r.`${entry.key}` = '${entry.value}' """ } write( """MATCH (fromId:${relationType.from.name} {id: '$fromId'})-[r:${relationType.name}]->(toId:${relationType.to.name} {id: '$toId'}) @@ -425,6 +442,7 @@ class UniqueRelationStore(private val relati } }, { + // TODO vihang: set props using parameter val strProps: String = properties.entries.joinToString(",") { """`${it.key}`: "${it.value}"""" } write(""" @@ -463,6 +481,7 @@ class UniqueRelationStore(private val relati return doNotExist(fromId, toId, transaction).flatMap { val properties = getProperties(relation as Any) + // TODO vihang: set props using parameter val strProps: String = properties.entries.joinToString(",") { """`${it.key}`: "${it.value}"""" } write(""" From 9aede993d50f275bf940f4c17a85bd9ef016595e Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 28 Nov 2019 15:04:07 +0100 Subject: [PATCH 29/56] Refactored Graph Schema and DSL --- .../kotlin/org/ostelco/prime/dsl/Syntax.kt | 9 +- .../org/ostelco/prime/dsl/Transactions.kt | 59 ++++++----- .../ostelco/prime/storage/graph/Neo4jStore.kt | 46 ++++----- .../ostelco/prime/storage/graph/Registry.kt | 28 +----- .../org/ostelco/prime/storage/graph/Schema.kt | 97 ++++++++----------- .../ostelco/prime/storage/graph/SchemaTest.kt | 5 - 6 files changed, 103 insertions(+), 141 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt index e5250e822..8247d3321 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt @@ -44,10 +44,6 @@ data class RelatedToClause( val relationType: RelationType, val toId: String) -data class RelationFromClause( - val relationType: RelationType, - val fromId: String) - data class RelationExpression( val relationType: RelationType, val fromId: String, @@ -128,6 +124,11 @@ data class CustomerContext(override val id: String) : EntityContext(Cu relationType = customerToSegmentRelation, fromId = id, toId = segment.id) + + infix fun belongsToRegion(region: RegionContext) = RelationExpression( + relationType = customerRegionRelation, + fromId = id, + toId = region.id) } data class ExCustomerContext(override val id: String) : EntityContext(ExCustomer::class, id) { diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt index 00b892905..86fd490fe 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Transactions.kt @@ -2,15 +2,12 @@ package org.ostelco.prime.dsl import arrow.core.Either import arrow.core.flatMap -import arrow.core.left import arrow.core.right import org.neo4j.driver.v1.AccessMode.READ import org.neo4j.driver.v1.AccessMode.WRITE import org.ostelco.prime.getLogger import org.ostelco.prime.model.HasId -import org.ostelco.prime.storage.DatabaseError import org.ostelco.prime.storage.StoreError -import org.ostelco.prime.storage.SystemError import org.ostelco.prime.storage.graph.EntityRegistry import org.ostelco.prime.storage.graph.EntityStore import org.ostelco.prime.storage.graph.Neo4jClient @@ -72,7 +69,6 @@ open class ReadTransaction(open val transaction: PrimeTransaction) { fun get(relatedToClause: RelatedToClause): Either> { val entityStore: EntityStore = relatedToClause.relationType.to.entityStore - ?: return DatabaseError(type = "entityStore", id = relatedToClause.relationType.to.name, message = "Missing entity store").left() return entityStore.getRelatedFrom( id = relatedToClause.toId, relationType = relatedToClause.relationType, @@ -81,21 +77,34 @@ open class ReadTransaction(open val transaction: PrimeTransaction) { fun get(relatedFromClause: RelatedFromClause): Either> { val entityStore: EntityStore = relatedFromClause.relationType.from.entityStore - ?: return DatabaseError(type = "entityStore", id = relatedFromClause.relationType.from.name, message = "Missing entity store").left() return entityStore.getRelated( id = relatedFromClause.fromId, relationType = relatedFromClause.relationType, transaction = transaction) } - fun get(relationFromClause: RelationFromClause): Either> { - val entityStore: EntityStore = relationFromClause.relationType.from.entityStore - ?: return DatabaseError(type = "entityStore", id = relationFromClause.relationType.from.name, message = "Missing entity store").left() - return entityStore.getRelations( - id = relationFromClause.fromId, - relationType = relationFromClause.relationType, - transaction = transaction) + fun get(relationExpression: RelationExpression): Either> { + return when (val relationStore = relationExpression.relationType.relationStore) { + is UniqueRelationStore<*, *, *> -> (relationStore as UniqueRelationStore).get( + fromId = relationExpression.fromId, + toId = relationExpression.toId, + transaction = transaction + ).map(::listOf) + is RelationStore<*, *, *> -> (relationStore as RelationStore).get( + fromId = relationExpression.fromId, + toId = relationExpression.toId, + transaction = transaction + ) + } } + + fun get(partialRelationExpression: PartialRelationExpression): Either> = get( + RelationExpression( + relationType = partialRelationExpression.relationType, + fromId = partialRelationExpression.fromId, + toId = partialRelationExpression.toId + ) + ) } class WriteTransaction(override val transaction: PrimeTransaction) : ReadTransaction(transaction = transaction) { @@ -173,24 +182,30 @@ class WriteTransaction(override val transaction: PrimeTransaction) : ReadTransac ) } } - null -> { - SystemError(type = "relationStore", id = relationExpression.relationType.name, message = "Missing relation store").left() - } } } fun unlink(expression: () -> RelationExpression): Either { val relationExpression = expression() - val relationStore = relationExpression.relationType.relationStore - return relationStore?.delete( + val relationStore = relationExpression + .relationType + .relationStore + return relationStore.delete( fromId = relationExpression.fromId, toId = relationExpression.toId, transaction = transaction - ) ?: SystemError( - type = "relationStore", - id = relationExpression.relationType.name, - message = "Missing relation store" - ).left() + ) + } + + fun unlink(partialRelationExpression: PartialRelationExpression): Either { + val relationStore = partialRelationExpression + .relationType + .relationStore + return relationStore.delete( + fromId = partialRelationExpression.fromId, + toId = partialRelationExpression.toId, + transaction = transaction + ) } } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index e47d2c72f..ac73911fd 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -154,9 +154,10 @@ import org.ostelco.prime.paymentprocessor.core.NotFoundError as NotFoundPaymentE enum class Relation( val from: KClass, - val to: KClass) { + val to: KClass, + val isUnique: Boolean = true) { - IDENTIFIES(from = Identity::class, to = Customer::class), // (Identity) -[IDENTIFIES]-> (Customer) + IDENTIFIES(from = Identity::class, to = Customer::class, isUnique = false), // (Identity) -[IDENTIFIES]-> (Customer) HAS_SUBSCRIPTION(from = Customer::class, to = Subscription::class), // (Customer) -[HAS_SUBSCRIPTION]-> (Subscription) @@ -170,7 +171,7 @@ enum class Relation( SUBSCRIBES_TO_PLAN(from = Customer::class, to = Plan::class), // (Customer) -[SUBSCRIBES_TO_PLAN]-> (Plan) - LINKED_TO_BUNDLE(from = Subscription::class, to = Bundle::class), // (Subscription) -[LINKED_TO_BUNDLE]-> (Bundle) + LINKED_TO_BUNDLE(from = Subscription::class, to = Bundle::class, isUnique = false), // (Subscription) -[LINKED_TO_BUNDLE]-> (Bundle) FOR_PURCHASE_BY(from = PurchaseRecord::class, to = Customer::class), // (PurchaseRecord) -[FOR_PURCHASE_BY]-> (Customer) @@ -184,7 +185,7 @@ enum class Relation( BELONG_TO_SEGMENT(from = Customer::class, to = Segment::class), // (Customer) -[BELONG_TO_SEGMENT]-> (Segment) - EKYC_SCAN(from = Customer::class, to = ScanInformation::class), // (Customer) -[EKYC_SCAN]-> (ScanInformation) + EKYC_SCAN(from = Customer::class, to = ScanInformation::class, isUnique = false), // (Customer) -[EKYC_SCAN]-> (ScanInformation) BELONG_TO_REGION(from = Customer::class, to = Region::class), // (Customer) -[BELONG_TO_REGION]-> (Region) @@ -238,77 +239,66 @@ object Neo4jStoreSingleton : GraphStore { from = identityEntity, to = customerEntity, dataClass = Identifies::class.java) - .also { RelationStore(it) } val subscriptionRelation = RelationType( relation = HAS_SUBSCRIPTION, from = customerEntity, to = subscriptionEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val exSubscriptionRelation = RelationType( relation = HAD_SUBSCRIPTION, from = exCustomerEntity, to = subscriptionEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val customerToBundleRelation = RelationType( relation = HAS_BUNDLE, from = customerEntity, to = bundleEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val subscriptionToBundleRelation = RelationType( relation = LINKED_TO_BUNDLE, from = subscriptionEntity, to = bundleEntity, dataClass = SubscriptionToBundle::class.java) - .also { RelationStore(it) } val customerToSimProfileRelation = RelationType( relation = HAS_SIM_PROFILE, from = customerEntity, to = simProfileEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val exCustomerToSimProfileRelation = RelationType( relation = HAD_SIM_PROFILE, from = exCustomerEntity, to = simProfileEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val forPurchaseByRelation = RelationType( relation = FOR_PURCHASE_BY, from = purchaseRecordEntity, to = customerEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val forPurchaseOfRelation = RelationType( relation = FOR_PURCHASE_OF, from = purchaseRecordEntity, to = productEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val referredRelation = RelationType( relation = REFERRED, from = customerEntity, to = customerEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val subscribesToPlanRelation = RelationType( relation = Relation.SUBSCRIBES_TO_PLAN, from = customerEntity, to = planEntity, dataClass = PlanSubscription::class.java) - private val subscribesToPlanRelationStore = UniqueRelationStore(subscribesToPlanRelation) val customerRegionRelation = RelationType( relation = Relation.BELONG_TO_REGION, @@ -322,7 +312,6 @@ object Neo4jStoreSingleton : GraphStore { from = exCustomerEntity, to = regionEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val scanInformationRelation = RelationType( relation = Relation.EKYC_SCAN, @@ -336,14 +325,12 @@ object Neo4jStoreSingleton : GraphStore { from = simProfileEntity, to = regionEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val subscriptionSimProfileRelation = RelationType( relation = Relation.SUBSCRIPTION_UNDER_SIM_PROFILE, from = subscriptionEntity, to = simProfileEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } private val onNewCustomerAction: OnNewCustomerAction = config.onNewCustomerAction.getKtsService() private val allowedRegionsService: AllowedRegionsService = config.allowedRegionsService.getKtsService() @@ -759,8 +746,10 @@ object Neo4jStoreSingleton : GraphStore { val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() validateBundleList(bundles, customerId).bind() val customer = get(Customer withId customerId).bind() - val status = customerRegionRelationStore - .get(fromId = customerId, toId = regionCode.toLowerCase(), transaction = transaction) + val status = customerRegionRelationStore.get( + fromId = customerId, + toId = regionCode.toLowerCase(), + transaction = transaction) .bind() .status isApproved( @@ -2580,8 +2569,10 @@ object Neo4jStoreSingleton : GraphStore { Either.monad().binding { val plan = get(Plan withId planId) .bind() - val planSubscription = subscribesToPlanRelationStore.get(customerId, planId, transaction) + + val planSubscription = get((Customer withId customerId) subscribesTo (Plan withId planId)) .bind() + .single() paymentProcessor.cancelSubscription(planSubscription.subscriptionId, invoiceNow) .mapLeft { NotDeletedError(type = planEntity.name, id = "$customerId -> ${plan.id}", @@ -2590,7 +2581,7 @@ object Neo4jStoreSingleton : GraphStore { Unit.right() }.bind() - subscribesToPlanRelationStore.delete(customerId, planId, transaction) + unlink((Customer withId customerId) subscribesTo (Plan withId planId)) .flatMap { Either.right(plan) }.bind() @@ -2818,15 +2809,12 @@ object Neo4jStoreSingleton : GraphStore { from = offerEntity, to = segmentEntity, dataClass = None::class.java) - .also { UniqueRelationStore(it) } val offerToProductRelation = RelationType( - relation = OFFER_HAS_PRODUCT, - from = offerEntity, - to = productEntity, - dataClass = None::class.java - ) - .also { UniqueRelationStore(it) } + relation = OFFER_HAS_PRODUCT, + from = offerEntity, + to = productEntity, + dataClass = None::class.java) val customerToSegmentRelation = RelationType(BELONG_TO_SEGMENT, customerEntity, segmentEntity, None::class.java) private val customerToSegmentStore = RelationStore(customerToSegmentRelation) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Registry.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Registry.kt index 64f653e40..e9046a8c0 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Registry.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Registry.kt @@ -17,8 +17,7 @@ object EntityRegistry { } } - fun getEntityStore(kClass: KClass): EntityStore = - getEntityType(kClass).entityStore ?: throw Exception("Missing EntityStore for Entity Type: ${kClass.simpleName}") + fun getEntityStore(kClass: KClass): EntityStore = getEntityType(kClass).entityStore } val KClass.entityType: EntityType @@ -26,28 +25,3 @@ val KClass.entityType: EntityType val KClass.entityStore: EntityStore get() = getEntityStore(this) - -object RelationRegistry { - - private val relationTypeMap = mutableMapOf>() - private val relationStoreMap = mutableMapOf>() - - private val relationFromTypeMap = mutableMapOf, RelationType>() - private val relationToTypeMap = mutableMapOf, RelationType>() - - fun register(relation: Relation, relationType: RelationType) { - relationTypeMap[relation] = relationType - relationFromTypeMap[relation.from] = relationType - relationToTypeMap[relation.to] = relationType - } - - fun register(relation: Relation, relationStore: RelationStore) { - relationStoreMap[relation] = relationStore - } - - fun getRelationType(relation: Relation) = relationTypeMap[relation] - - fun getRelationTypeFrom(from: KClass) = relationFromTypeMap[from] - - fun getRelationTypeTo(to: KClass) = relationToTypeMap[to] -} \ No newline at end of file diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt index 585d624b0..2e55c4329 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Schema.kt @@ -35,7 +35,7 @@ data class EntityType( private val dataClass: Class, val name: String = dataClass.simpleName) { - var entityStore: EntityStore? = null + var entityStore: EntityStore = EntityStore(this) fun createEntity(map: Map): ENTITY = ObjectHandler.getObject(map, dataClass) } @@ -46,10 +46,11 @@ data class RelationType( val to: EntityType, private val dataClass: Class) { - init { - RelationRegistry.register(relation, this) + val relationStore: BaseRelationStore = if (relation.isUnique) { + UniqueRelationStore(this) + } else { + RelationStore(this) } - var relationStore: BaseRelationStore? = null val name: String = relation.name @@ -128,24 +129,6 @@ class EntityStore(private val entityType: EntityType) { } } - fun getRelations( - id: String, - relationType: RelationType, - transaction: Transaction): Either> { - - return exists(id, transaction).flatMap { - - read(""" - MATCH (from:${entityType.name} { id: '$id' })-[r:${relationType.name}]-() - return r; - """.trimIndent(), - transaction) { statementResult -> - statementResult.list { record -> relationType.createRelation(record["r"].asMap()) } - .right() - } - } - } - fun update(entity: E, transaction: Transaction): Either { return exists(entity.id, transaction).flatMap { @@ -218,10 +201,6 @@ sealed class BaseRelationStore { // TODO vihang: check if relation already exists, with allow duplicate boolean flag param class RelationStore(private val relationType: RelationType) : BaseRelationStore() { - init { - relationType.relationStore = this - } - fun create(from: FROM, relation: RELATION, to: TO, transaction: Transaction): Either { val properties = getStringProperties(relation as Any) @@ -331,6 +310,25 @@ class RelationStore(private val relationType }) } + fun get( + fromId: String, + toId: String, + transaction: Transaction): Either> { + + return relationType.from.entityStore.exists(fromId, transaction) + .flatMap { relationType.to.entityStore.exists(fromId, transaction) } + .flatMap { + read(""" + MATCH (:${relationType.from.name} { id: '$fromId' })-[r:${relationType.name}]-(:${relationType.to.name} { id: '$toId' }) + return r; + """.trimIndent(), + transaction) { statementResult -> + statementResult.list { record -> relationType.createRelation(record["r"].asMap()) } + .right() + } + } + } + fun removeAll(toId: String, transaction: Transaction): Either = write(""" MATCH (from:${relationType.from.name})-[r:${relationType.name}]->(to:${relationType.to.name} { id: '$toId' }) DELETE r; @@ -354,18 +352,12 @@ class RelationStore(private val relationType class UniqueRelationStore(private val relationType: RelationType) : BaseRelationStore() { - init { - relationType.relationStore = this - } - // If relation does not exists, then it creates new relation. fun createIfAbsent(fromId: String, toId: String, transaction: Transaction): Either { - return (relationType.from.entityStore?.exists(fromId, transaction) - ?: NotFoundError(type = relationType.from.name, id = fromId).left()) + return relationType.from.entityStore.exists(fromId, transaction) .flatMap { - relationType.to.entityStore?.exists(toId, transaction) - ?: NotFoundError(type = relationType.to.name, id = toId).left() + relationType.to.entityStore.exists(toId, transaction) }.flatMap { doNotExist(fromId, toId, transaction).fold( @@ -388,11 +380,9 @@ class UniqueRelationStore(private val relati // If relation does not exists, then it creates new relation. fun createIfAbsent(fromId: String, relation: RELATION, toId: String, transaction: Transaction): Either { - return (relationType.from.entityStore?.exists(fromId, transaction) - ?: NotFoundError(type = relationType.from.name, id = fromId).left()) + return relationType.from.entityStore.exists(fromId, transaction) .flatMap { - relationType.to.entityStore?.exists(toId, transaction) - ?: NotFoundError(type = relationType.to.name, id = toId).left() + relationType.to.entityStore.exists(toId, transaction) }.flatMap { doNotExist(fromId, toId, transaction).fold( @@ -419,11 +409,9 @@ class UniqueRelationStore(private val relati // If relation does not exists, then it creates new relation. Or else updates it. fun createOrUpdate(fromId: String, relation: RELATION, toId: String, transaction: Transaction): Either { - return (relationType.from.entityStore?.exists(fromId, transaction) - ?: NotFoundError(type = relationType.from.name, id = fromId).left()) + return relationType.from.entityStore.exists(fromId, transaction) .flatMap { - relationType.to.entityStore?.exists(toId, transaction) - ?: NotFoundError(type = relationType.to.name, id = toId).left() + relationType.to.entityStore.exists(toId, transaction) }.flatMap { val properties = getProperties(relation as Any) @@ -455,7 +443,8 @@ class UniqueRelationStore(private val relati ifTrue = { Unit }, ifFalse = { NotCreatedError(relationType.name, "$fromId -> $toId") }) } - }) + } + ) } } @@ -497,17 +486,6 @@ class UniqueRelationStore(private val relati } } - override fun delete(fromId: String, toId: String, transaction: Transaction): Either = write(""" - MATCH (:${relationType.from.name} { id: '$fromId'})-[r:${relationType.name}]->(:${relationType.to.name} {id: '$toId'}) - DELETE r - """.trimMargin(), - transaction) { statementResult -> - - Either.cond(statementResult.summary().counters().relationshipsDeleted() == 1, - ifTrue = { Unit }, - ifFalse = { NotDeletedError(relationType.name, "$fromId -> $toId") }) - } - fun get(fromId: String, toId: String, transaction: Transaction): Either = read(""" MATCH (:${relationType.from.name} {id: '$fromId'})-[r:${relationType.name}]->(:${relationType.to.name} {id: '$toId'}) RETURN r @@ -522,6 +500,17 @@ class UniqueRelationStore(private val relati } } + override fun delete(fromId: String, toId: String, transaction: Transaction): Either = write(""" + MATCH (:${relationType.from.name} { id: '$fromId'})-[r:${relationType.name}]->(:${relationType.to.name} {id: '$toId'}) + DELETE r + """.trimMargin(), + transaction) { statementResult -> + + Either.cond(statementResult.summary().counters().relationshipsDeleted() == 1, + ifTrue = { Unit }, + ifFalse = { NotDeletedError(relationType.name, "$fromId -> $toId") }) + } + private fun doNotExist(fromId: String, toId: String, transaction: Transaction): Either = read(""" MATCH (:${relationType.from.name} {id: '$fromId'})-[r:${relationType.name}]->(:${relationType.to.name} {id: '$toId'}) RETURN count(r) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt index 9ae89d000..8c3320e15 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/SchemaTest.kt @@ -150,11 +150,6 @@ class SchemaTest { fromEntityStore.getRelated(aId, relation, transaction).fold( { fail(it.message) }, { assertEquals(listOf(b), it) }) - - // get 'r' from 'a' - fromEntityStore.getRelations(aId, relation, transaction).fold( - { fail(it.message) }, - { assertEquals(listOf(r), it) }) } } From 4bdf14a8c1c95b2baade5b30c5762d05c69fd48b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 28 Nov 2019 20:37:04 +0100 Subject: [PATCH 30/56] Extend connection timeout from default 500ms to 10000ms (ten seconds) to see if that gets rids of the dropouts we're observing occationally --- prime/config/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/prime/config/config.yaml b/prime/config/config.yaml index db550ef6f..6741ac394 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -167,6 +167,7 @@ modules: validationQuery: "/* Prime Health Check */ SELECT 1" httpClient: timeout: 10000ms + connectionTimeout: 10000ms tls: # Default is 500 milliseconds, we need more when debugging. # protocol: TLSv1.2 From 87c9fab88f109f278ced3ccb7e46df6282cb9615 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 29 Nov 2019 11:16:37 +0100 Subject: [PATCH 31/56] Cleanup and refactoring of Stripe error collection and reporting --- .../support/resources/HoustonResources.kt | 4 +- .../ostelco/prime/storage/graph/Neo4jStore.kt | 81 +++++++++------- .../prime/paymentprocessor/StripeMonitor.kt | 4 +- .../StripePaymentProcessor.kt | 19 ++-- .../prime/paymentprocessor/StripeUtils.kt | 94 +++++++++++------- .../resources/StripeMonitorResource.kt | 4 +- .../org/ostelco/prime/apierror/ApiError.kt | 31 ++++-- .../paymentprocessor/core/PaymentError.kt | 97 +++++++++++++++++-- 8 files changed, 235 insertions(+), 99 deletions(-) diff --git a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt index 1ab32db4b..d9731e27b 100644 --- a/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt +++ b/customer-support-endpoint/src/main/kotlin/org/ostelco/prime/support/resources/HoustonResources.kt @@ -25,7 +25,7 @@ import org.ostelco.prime.model.SimProfile import org.ostelco.prime.model.Subscription import org.ostelco.prime.module.getResource import org.ostelco.prime.notifications.NOTIFY_OPS_MARKER -import org.ostelco.prime.paymentprocessor.core.ForbiddenError +import org.ostelco.prime.paymentprocessor.core.InvalidRequestError import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.storage.AdminDataSource import org.ostelco.prime.storage.AuditLogStore @@ -308,7 +308,7 @@ class RefundResource { storage.refundPurchase(identity, purchaseRecordId, reason) }.mapLeft { when (it) { - is ForbiddenError -> org.ostelco.prime.apierror.ForbiddenError("Failed to refund purchase. ${it.description}", ApiErrorCode.FAILED_TO_REFUND_PURCHASE) + is InvalidRequestError -> org.ostelco.prime.apierror.ForbiddenError("Failed to refund purchase. ${it.description}", ApiErrorCode.FAILED_TO_REFUND_PURCHASE) else -> NotFoundError("Failed to refund purchase. ${it.toString()}", ApiErrorCode.FAILED_TO_REFUND_PURCHASE) } } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index e47d2c72f..754cccddd 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -85,8 +85,7 @@ import org.ostelco.prime.module.getResource import org.ostelco.prime.notifications.EmailNotifier import org.ostelco.prime.notifications.NOTIFY_OPS_MARKER import org.ostelco.prime.paymentprocessor.PaymentProcessor -import org.ostelco.prime.paymentprocessor.core.BadGatewayError -import org.ostelco.prime.paymentprocessor.core.ForbiddenError +import org.ostelco.prime.paymentprocessor.core.InvalidRequestError import org.ostelco.prime.paymentprocessor.core.InvoicePaymentInfo import org.ostelco.prime.paymentprocessor.core.PaymentError import org.ostelco.prime.paymentprocessor.core.PaymentStatus @@ -94,7 +93,10 @@ import org.ostelco.prime.paymentprocessor.core.PaymentTransactionInfo import org.ostelco.prime.paymentprocessor.core.PlanAlredyPurchasedError import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.paymentprocessor.core.ProfileInfo +import org.ostelco.prime.paymentprocessor.core.StorePurchaseError import org.ostelco.prime.paymentprocessor.core.SubscriptionDetailsInfo +import org.ostelco.prime.paymentprocessor.core.SubscriptionError +import org.ostelco.prime.paymentprocessor.core.UpdatePurchaseError import org.ostelco.prime.securearchive.SecureArchiveService import org.ostelco.prime.sim.SimManager import org.ostelco.prime.storage.AlreadyExistsError @@ -1282,13 +1284,13 @@ object Neo4jStoreSingleton : GraphStore { .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError( "Failed to get customer data for customer with identity - $identity", - error = it) + internalError = it) }.bind() val product = getProduct(identity, sku) .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Product $sku is unavailable", - error = it) + internalError = it) } .bind() @@ -1328,10 +1330,11 @@ object Neo4jStoreSingleton : GraphStore { will ensure that the invoice will be voided. */ createPurchaseRecord(customer.id, purchaseRecord) .mapLeft { - logger.error("Failed to save purchase record for customer ${customer.id}, invoice-id $invoiceId, invoice will be voided in Stripe") - AuditLog.error(customerId = customer.id, message = "Failed to save purchase record - invoice-id $invoiceId, invoice will be voided in Stripe") - BadGatewayError("Failed to save purchase record", - error = it) + logger.error("Failed to save purchase record for customer ${customer.id}, invoice: $invoiceId, invoice will be voided in Stripe") + AuditLog.error(customerId = customer.id, + message = "Failed to save purchase record - invoice: $invoiceId, invoice will be voided in Stripe") + StorePurchaseError("Failed to save purchase record", + internalError = it) }.bind() /* TODO: While aborting transactions, send a record with "reverted" status. */ @@ -1347,7 +1350,9 @@ object Neo4jStoreSingleton : GraphStore { customerId = customer.id, product = product ).mapLeft { - BadGatewayError(description = it.message, error = it.error).left().bind() + StorePurchaseError(description = it.message, internalError = it.error) + .left() + .bind() }.bind() ProductInfo(product.sku) @@ -1447,8 +1452,8 @@ object Neo4jStoreSingleton : GraphStore { if (sourceId != null) { val sourceDetails = paymentProcessor.getSavedSources(customer.id) .mapLeft { - BadGatewayError("Failed to fetch sources for customer: ${customer.id}", - error = it) + org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for customer: ${customer.id}", + internalError = it) }.bind() if (!sourceDetails.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { paymentProcessor.addSource(customer.id, sourceId) @@ -1462,8 +1467,8 @@ object Neo4jStoreSingleton : GraphStore { taxRegionId = taxRegionId) .mapLeft { AuditLog.error(customerId = customer.id, message = "Failed to subscribe to plan $sku") - BadGatewayError("Failed to subscribe ${customer.id} to plan $sku", - error = it) + SubscriptionError("Failed to subscribe ${customer.id} to plan $sku", + internalError = it) } .bind() }.fix() @@ -1531,7 +1536,7 @@ object Neo4jStoreSingleton : GraphStore { } PaymentStatus.REQUIRES_PAYMENT_METHOD -> { NotCreatedError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", - error = ForbiddenError("Payment method failed")) + error = InvalidRequestError("Payment method failed")) .left().bind() } PaymentStatus.REQUIRES_ACTION, @@ -1580,7 +1585,8 @@ object Neo4jStoreSingleton : GraphStore { if (sourceId != null) { val sourceDetails = paymentProcessor.getSavedSources(customer.id) .mapLeft { - BadGatewayError("Failed to fetch sources for user", error = it) + org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for user", + internalError = it) }.bind() addedSourceId = sourceId @@ -1607,8 +1613,8 @@ object Neo4jStoreSingleton : GraphStore { it }.linkReversalActionToTransaction(transaction) { paymentProcessor.removeInvoice(it.id) - logger.error(NOTIFY_OPS_MARKER, - """Failed to create or pay invoice for customer ${customer.id}, invoice-id: ${it.id}. + logger.warn(NOTIFY_OPS_MARKER, + """Failed to pay invoice for customer ${customer.id}, invoice-id: ${it.id}. Verify that the invoice has been deleted or voided in Stripe dashboard. """.trimIndent()) }.bind() @@ -1616,11 +1622,11 @@ object Neo4jStoreSingleton : GraphStore { /* Force immediate payment of the invoice. */ val invoicePaymentInfo = paymentProcessor.payInvoice(invoice.id) .mapLeft { - logger.error("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") + logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") it }.linkReversalActionToTransaction(transaction) { paymentProcessor.refundCharge(it.chargeId) - logger.error(NOTIFY_OPS_MARKER, + logger.warn(NOTIFY_OPS_MARKER, """Refunded customer ${customer.id} for invoice: ${it.id}. Verify that the invoice has been refunded in Stripe dashboard. """.trimIndent()) @@ -2677,8 +2683,8 @@ object Neo4jStoreSingleton : GraphStore { val purchaseRecords = getPurchaseTransactions(startPadded, endPadded) .mapLeft { - BadGatewayError("Error when fetching purchase records", - error = it) + org.ostelco.prime.paymentprocessor.core.NotFoundError("Error when fetching purchase records", + internalError = it) }.bind() val paymentRecords = getPaymentTransactions(startPadded, endPadded) .bind() @@ -2745,16 +2751,19 @@ object Neo4jStoreSingleton : GraphStore { // For refunds // - private fun checkPurchaseRecordForRefund(purchaseRecord: PurchaseRecord): Either { - if (purchaseRecord.refund != null) { - logger.error("Trying to refund again, ${purchaseRecord.id}, refund ${purchaseRecord.refund?.id}") - return Either.left(ForbiddenError("Trying to refund again")) - } else if (purchaseRecord.product.price.amount == 0) { - logger.error("Trying to refund a free product, ${purchaseRecord.id}") - return Either.left(ForbiddenError("Trying to refund a free purchase")) - } - return Unit.right() - } + private fun checkPurchaseRecordForRefund(purchaseRecord: PurchaseRecord): Either = + if (purchaseRecord.refund != null) { + logger.error("Trying to refund again, ${purchaseRecord.id}, refund ${purchaseRecord.refund?.id}") + InvalidRequestError("Attempt at refunding again the purchase ${purchaseRecord.id} " + + "of product ${purchaseRecord.product.sku}, refund ${purchaseRecord.refund?.id}") + .left() + } else if (purchaseRecord.product.price.amount == 0) { + logger.error("Trying to refund a free product, ${purchaseRecord.id}") + InvalidRequestError("Trying to refund a free purchase of product ${purchaseRecord.product.sku}") + .left() + } else { + Unit.right() + } override fun refundPurchase( identity: ModelIdentity, @@ -2766,13 +2775,13 @@ object Neo4jStoreSingleton : GraphStore { .mapLeft { logger.error("Failed to find customer with identity - $identity") NotFoundPaymentError("Failed to find customer with identity - $identity", - error = it) + internalError = it) }.bind() val purchaseRecord = get(PurchaseRecord withId purchaseRecordId) // If we can't find the record, return not-found .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Purchase Record unavailable", - error = it) + internalError = it) }.bind() checkPurchaseRecordForRefund(purchaseRecord) .bind() @@ -2788,9 +2797,9 @@ object Neo4jStoreSingleton : GraphStore { ) update { changedPurchaseRecord } .mapLeft { - logger.error("failed to update purchase record, for refund $refund.id, chargeId $purchaseRecordId, payment has been refunded in Stripe") - BadGatewayError("Failed to update purchase record for refund ${refund.id}", - error = it) + logger.error("Failed to update purchase record, for refund $refund.id, chargeId $purchaseRecordId, payment has been refunded in Stripe") + UpdatePurchaseError("Failed to update purchase record for refund ${refund.id}", + internalError = it) }.bind() analyticsReporter.reportRefund( diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeMonitor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeMonitor.kt index e739dbff9..cd2ef0e77 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeMonitor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeMonitor.kt @@ -10,7 +10,7 @@ import com.stripe.model.WebhookEndpoint import org.ostelco.prime.getLogger import org.ostelco.prime.notifications.NOTIFY_OPS_MARKER import org.ostelco.prime.paymentprocessor.StripeUtils.either -import org.ostelco.prime.paymentprocessor.core.BadGatewayError +import org.ostelco.prime.paymentprocessor.core.NotFoundError import org.ostelco.prime.paymentprocessor.core.PaymentConfigurationError import org.ostelco.prime.paymentprocessor.core.PaymentError @@ -234,7 +234,7 @@ class StripeMonitor { }.right() else { logger.error("No webhooks found on check for Stripe events state") - BadGatewayError("No webhooks found on check for Stripe events state") + NotFoundError("No webhooks found on check for Stripe events state") .left() } } diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index 3f2f5d2ab..550528b01 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -23,8 +23,8 @@ import com.stripe.net.RequestOptions import org.ostelco.prime.getLogger import org.ostelco.prime.notifications.NOTIFY_OPS_MARKER import org.ostelco.prime.paymentprocessor.StripeUtils.either -import org.ostelco.prime.paymentprocessor.core.BadGatewayError -import org.ostelco.prime.paymentprocessor.core.ForbiddenError +import org.ostelco.prime.paymentprocessor.core.ChargeError +import org.ostelco.prime.paymentprocessor.core.InvoiceError import org.ostelco.prime.paymentprocessor.core.InvoicePaymentInfo import org.ostelco.prime.paymentprocessor.core.InvoiceInfo import org.ostelco.prime.paymentprocessor.core.InvoiceItemInfo @@ -36,6 +36,7 @@ import org.ostelco.prime.paymentprocessor.core.PlanInfo import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.paymentprocessor.core.ProfileInfo import org.ostelco.prime.paymentprocessor.core.SourceDetailsInfo +import org.ostelco.prime.paymentprocessor.core.SourceError import org.ostelco.prime.paymentprocessor.core.SourceInfo import org.ostelco.prime.paymentprocessor.core.SubscriptionDetailsInfo import org.ostelco.prime.paymentprocessor.core.SubscriptionInfo @@ -303,6 +304,7 @@ class StripePaymentProcessor : PaymentProcessor { SubscriptionInfo(id = subscription.id) } + /* TODO (kmm): Fix missing exception handling of the 'Charge.create()' call. */ override fun authorizeCharge(customerId: String, sourceId: String?, amount: Int, currency: String): Either { val errorMessage = "Failed to authorize the charge for customerId $customerId sourceId $sourceId amount $amount currency $currency" return when (amount) { @@ -323,12 +325,13 @@ class StripePaymentProcessor : PaymentProcessor { Either.cond( test = (review == null), ifTrue = { charge.id }, - ifFalse = { ForbiddenError("Review required, $errorMessage $review") } + ifFalse = { ChargeError("Review required, $errorMessage $review") } ) } } } + /* TODO (kmm): Fix missing exception handling of the 'Charge.retrive()' call. */ override fun captureCharge(chargeId: String, customerId: String, amount: Int, currency: String): Either { val errorMessage = "Failed to capture charge for customerId $customerId chargeId $chargeId" return when (amount) { @@ -340,7 +343,7 @@ class StripePaymentProcessor : PaymentProcessor { Either.cond( test = (review == null), ifTrue = { charge }, - ifFalse = { ForbiddenError("Review required, $errorMessage $review") } + ifFalse = { ChargeError("Review required, $errorMessage $review") } ) }.flatMap { charge -> try { @@ -348,7 +351,7 @@ class StripePaymentProcessor : PaymentProcessor { charge.id.right() } catch (e: Exception) { logger.warn(errorMessage, e) - BadGatewayError(errorMessage).left() + ChargeError(errorMessage).left() } } } @@ -378,7 +381,7 @@ class StripePaymentProcessor : PaymentProcessor { is Card -> accountInfo.delete() is Source -> accountInfo.detach() else -> - BadGatewayError("Attempt to remove unsupported account-type $accountInfo") + SourceError("Attempt to remove unsupported account-type $accountInfo") .left() } SourceInfo(sourceId) @@ -448,9 +451,9 @@ class StripePaymentProcessor : PaymentProcessor { removeInvoice(invoice) .fold({ - BadGatewayError(errorMessage, error = it) + InvoiceError(errorMessage, internalError = it) }, { - BadGatewayError(errorMessage) + InvoiceError(errorMessage) }).left() } else { InvoiceInfo(invoice.id).right() diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeUtils.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeUtils.kt index e9bfefc46..7b6190bdb 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeUtils.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripeUtils.kt @@ -1,6 +1,7 @@ package org.ostelco.prime.paymentprocessor import arrow.core.Either +import arrow.core.left import com.stripe.exception.ApiConnectionException import com.stripe.exception.AuthenticationException import com.stripe.exception.CardException @@ -8,9 +9,14 @@ import com.stripe.exception.InvalidRequestException import com.stripe.exception.RateLimitException import com.stripe.exception.StripeException import org.ostelco.prime.getLogger -import org.ostelco.prime.paymentprocessor.core.BadGatewayError -import org.ostelco.prime.paymentprocessor.core.ForbiddenError +import org.ostelco.prime.paymentprocessor.core.ApiConnectionError +import org.ostelco.prime.paymentprocessor.core.AuthenticationError +import org.ostelco.prime.paymentprocessor.core.CardError +import org.ostelco.prime.paymentprocessor.core.GenericError +import org.ostelco.prime.paymentprocessor.core.InvalidRequestError import org.ostelco.prime.paymentprocessor.core.PaymentError +import org.ostelco.prime.paymentprocessor.core.PaymentVendorError +import org.ostelco.prime.paymentprocessor.core.RateLimitError object StripeUtils { @@ -19,38 +25,54 @@ object StripeUtils { /** * Convenience function for handling Stripe I/O errors. */ - fun either(errorDescription: String, action: () -> RETURN): Either { - return try { - Either.right(action()) - } catch (e: CardException) { - // If something is decline with a card purchase, CardException will be caught - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}") - Either.left(ForbiddenError(errorDescription, e.message)) - } catch (e: RateLimitException) { - // Too many requests made to the API too quickly - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}") - Either.left(BadGatewayError(errorDescription, e.message)) - } catch (e: InvalidRequestException) { - // Invalid parameters were supplied to Stripe's API - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}") - Either.left(ForbiddenError(errorDescription, e.message)) - } catch (e: AuthenticationException) { - // Authentication with Stripe's API failed - // (maybe you changed API keys recently) - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) - Either.left(BadGatewayError(errorDescription)) - } catch (e: ApiConnectionException) { - // Network communication with Stripe failed - logger.warn("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) - Either.left(BadGatewayError(errorDescription)) - } catch (e: StripeException) { - // Unknown Stripe error - logger.error("Payment error : $errorDescription , Stripe Error Code: ${e.code}", e) - Either.left(BadGatewayError(errorDescription)) - } catch (e: Exception) { - // Something else happened, could be completely unrelated to Stripe - logger.error(errorDescription, e) - Either.left(BadGatewayError(errorDescription)) - } - } + fun either(description: String, action: () -> RETURN): Either = + try { + Either.right(action()) + } catch (e: CardException) { + /* Card got declined. */ + logger.warn("Payment card error : $description, Stripe codes: ${e.code}, ${e.declineCode}, ${e.statusCode}") + CardError(description = description, + message = e.message, + code = e.code, + declineCode = e.declineCode).left() + } catch (e: RateLimitException) { + /* Too many requests made to the Stripe API at the same time. */ + logger.error("Payment rate limiting error : $description, Stripe codes: ${e.code}, ${e.statusCode}") + RateLimitError(description = description, + message = e.message, + code = e.code).left() + } catch (e: InvalidRequestException) { + /* Invalid parameters were supplied to Stripe's API. */ + logger.error("Payment invalid request : $description, Stripe codes: ${e.code}, ${e.statusCode}") + InvalidRequestError(description = description, + message = e.message, + code = e.code).left() + } catch (e: AuthenticationException) { + /* Authentication with Stripe's API failed. + (Maybe you changed API keys recently?) */ + logger.error("Payment authentication error : $description, Stripe codes: ${e.code}, ${e.statusCode}", + e) + AuthenticationError(description = description, + message = e.message, + code = e.code).left() + } catch (e: ApiConnectionException) { + /* Network communication with Stripe failed. */ + logger.error("Payment API connection error : $description", + e) + ApiConnectionError(description = description, + message = e.message).left() + } catch (e: StripeException) { + /* Unknown Stripe error. */ + logger.error("Payment unknown Stripe error : $description", + e) + PaymentVendorError(description = description, + message = e.message).left() + } catch (e: Exception) { + /* Something else happened, completely unrelated to + Stripe. */ + logger.error("Payment unknown error : $description", + e) + GenericError(description = description, + message = e.message).left() + } } diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/resources/StripeMonitorResource.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/resources/StripeMonitorResource.kt index b678f842e..f6f305ffa 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/resources/StripeMonitorResource.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/resources/StripeMonitorResource.kt @@ -11,7 +11,7 @@ import org.ostelco.prime.getLogger import org.ostelco.prime.jsonmapper.asJson import org.ostelco.prime.paymentprocessor.StripeEventState import org.ostelco.prime.paymentprocessor.StripeMonitor -import org.ostelco.prime.paymentprocessor.core.BadGatewayError +import org.ostelco.prime.paymentprocessor.core.GenericError import org.ostelco.prime.paymentprocessor.core.PaymentError import org.ostelco.prime.paymentprocessor.publishers.StripeEventPublisher import java.time.Instant @@ -148,7 +148,7 @@ class StripeMonitorResource(val monitor: StripeMonitor) { events.size }.toEither { logger.error("Failed to publish retrieved failed events - ${it.message}") - BadGatewayError("Failed to publish failed retrieved events - ${it.message}") + GenericError("Failed to publish failed retrieved events - ${it.message}") } private fun ok(value: Map) = diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt index eebe0ac44..708681923 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt @@ -31,15 +31,32 @@ object ApiErrorMapper { val logger by getLogger() - fun mapPaymentErrorToApiError(description: String, errorCode: ApiErrorCode, paymentError: PaymentError) : ApiError { - logger.error("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) - return when(paymentError) { - is org.ostelco.prime.paymentprocessor.core.PlanAlredyPurchasedError -> ForbiddenError(description, errorCode, paymentError) - is org.ostelco.prime.paymentprocessor.core.ForbiddenError -> ForbiddenError(description, errorCode, paymentError) - // FIXME vihang: remove PaymentError from BadGatewayError - is org.ostelco.prime.paymentprocessor.core.BadGatewayError -> InternalServerError(description, errorCode, paymentError) + /* Log level depends on the type of payment error. */ + fun mapPaymentErrorToApiError(description: String, errorCode: ApiErrorCode, paymentError: PaymentError): ApiError { + if (paymentError is org.ostelco.prime.paymentprocessor.core.CardError) { + logger.warn("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) + } else { + logger.error("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) + } + return when (paymentError) { + /* Self made. */ + is org.ostelco.prime.paymentprocessor.core.ChargeError -> BadRequestError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.InvoiceError -> BadRequestError(description, errorCode, paymentError) is org.ostelco.prime.paymentprocessor.core.NotFoundError -> NotFoundError(description, errorCode, paymentError) is org.ostelco.prime.paymentprocessor.core.PaymentConfigurationError -> InternalServerError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.PlanAlredyPurchasedError -> ForbiddenError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.StorePurchaseError -> BadRequestError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.SourceError -> BadRequestError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.SubscriptionError -> BadRequestError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.UpdatePurchaseError -> BadRequestError(description, errorCode, paymentError) + /* Caused by upstream payment vendor. */ + is org.ostelco.prime.paymentprocessor.core.CardError -> ForbiddenError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.RateLimitError -> InternalServerError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.AuthenticationError -> BadRequestError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.ApiConnectionError -> InternalServerError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.InvalidRequestError -> ForbiddenError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.PaymentVendorError -> InternalServerError(description, errorCode, paymentError) + is org.ostelco.prime.paymentprocessor.core.GenericError -> InternalServerError(description, errorCode, paymentError) } } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt index 34031e8f7..99321e329 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/core/PaymentError.kt @@ -2,14 +2,99 @@ package org.ostelco.prime.paymentprocessor.core import org.ostelco.prime.apierror.InternalError -sealed class PaymentError(val description: String, var message : String? = null, val error: InternalError?) : InternalError() +/** + * For specific Stripe error messages a few 'code' fields are included + * which can provide more details about the cause for the error. + * + * - code : Mainly intended for programmatically handling of + * errors but can be useful for giving more context. + * https://stripe.com/docs/error-codes + * - decline code : For card errors resulting from a card + * issuer decline. Not always set. + * https://stripe.com/docs/declines/codes + * - status code : HTTP status code. + * + * The 'codes' fields are included in the error reporting when they are + * present. + * @param description Error description + * @param message Error description as provided upstream (Stripe) + * @param code A short string indicating the type of error from upstream + * @param declineCode A short string indication the reason for a card + * error from the card issuer + * @param internalError Internal error chaining + */ +sealed class PaymentError(val description: String, + val message: String? = null, + val code: String? = null, + val declineCode: String? = null, + val internalError: InternalError?) : InternalError() -class PlanAlredyPurchasedError(description: String, message: String? = null, error: InternalError? = null) : PaymentError(description, message, error) +class ChargeError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) -class ForbiddenError(description: String, message: String? = null, error: InternalError? = null) : PaymentError(description, message, error) +class InvoiceError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) -class NotFoundError(description: String, message: String? = null, error: InternalError? = null) : PaymentError(description, message, error) +class NotFoundError(description: String, + message: String? = null, + code: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, code, null, internalError) -class BadGatewayError(description: String, message: String? = null, error: InternalError? = null) : PaymentError(description, message, error) +class PaymentConfigurationError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) -class PaymentConfigurationError(description: String, message: String? = null, error: InternalError? = null) : PaymentError(description, message, error) +class PlanAlredyPurchasedError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class StorePurchaseError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class SourceError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class SubscriptionError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class UpdatePurchaseError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class CardError(description: String, + message: String? = null, + code: String? = null, + declineCode: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, code, declineCode, internalError) + +class RateLimitError(description: String, + message: String? = null, + code: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, code, null, internalError) + +class InvalidRequestError(description: String, + message: String? = null, + code: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, code, null, internalError) + +class AuthenticationError(description: String, + message: String? = null, + code: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, code, null, internalError) + +class ApiConnectionError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class PaymentVendorError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) + +class GenericError(description: String, + message: String? = null, + internalError: InternalError? = null) : PaymentError(description, message, null, null, internalError) From c3e42104234f35ab4493369719c93b4b374fc471 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 29 Nov 2019 11:18:12 +0100 Subject: [PATCH 32/56] Adds 'audit-log' logging of purchases and failed purchases --- .../kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 754cccddd..6bf841b61 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1337,6 +1337,10 @@ object Neo4jStoreSingleton : GraphStore { internalError = it) }.bind() + /* Adds purchase to customer history. */ + AuditLog.info(customerId = customer.id, + message = "Purchased product $sku for ${product.price.amount} ${product.price.currency} (invoice: $invoiceId, charge-id: $chargeId)") + /* TODO: While aborting transactions, send a record with "reverted" status. */ analyticsReporter.reportPurchase( customerAnalyticsId = customer.analyticsId, @@ -1623,6 +1627,10 @@ object Neo4jStoreSingleton : GraphStore { val invoicePaymentInfo = paymentProcessor.payInvoice(invoice.id) .mapLeft { logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") + /* Adds failed purchase to customer history. */ + AuditLog.warn(customerId = customer.id, + message = "Failed to complete purchase of product $sku for $price.amount} ${price.currency} " + + "status: ${it.code} decline reason: ${it.declineCode}") it }.linkReversalActionToTransaction(transaction) { paymentProcessor.refundCharge(it.chargeId) From 435359795dcc78787a7f25ef3a32b417462db5aa Mon Sep 17 00:00:00 2001 From: mpeterss Date: Fri, 29 Nov 2019 13:48:30 +0100 Subject: [PATCH 33/56] Use local data source for CCR-T For the CCR-T messages there is no use in waiting for the CCA to return from the OCS. The P-GW has terminated the session so we will use the LocalDataSource to quickly reply and sending consumption to OCS afterwards. This is also a small refactor as we now act differently on each CCR type --- .../datasource/proxy/ProxyDataSource.java | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/proxy/ProxyDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/proxy/ProxyDataSource.java index 010bad9ce..286b02749 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/proxy/ProxyDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/proxy/ProxyDataSource.java @@ -4,6 +4,8 @@ import org.ostelco.ocsgw.datasource.local.LocalDataSource; import org.ostelco.diameter.CreditControlContext; import org.ostelco.ocs.api.CreditControlRequestType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Proxy DataSource is a combination of the Local DataSource and any other @@ -22,6 +24,8 @@ public class ProxyDataSource implements DataSource { private final DataSource secondary; + private static final Logger LOG = LoggerFactory.getLogger(ProxyDataSource.class); + public ProxyDataSource(DataSource dataSource) { secondary = dataSource; } @@ -33,23 +37,36 @@ public void init() { @Override public void handleRequest(CreditControlContext context) { - // CCR-I and CCR-T will always be handled by the secondary DataSource - if (context.getOriginalCreditControlRequest().getRequestTypeAVPValue() - != CreditControlRequestType.UPDATE_REQUEST.getNumber()) { - secondary.handleRequest(context); - } else { - // For CCR-U we will send all requests to both Local and Secondary until the secondary has blocked the msisdn - if (!secondary.isBlocked(context.getCreditControlRequest().getMsisdn())) { - local.handleRequest(context); - // When local datasource will be responding with Answer, Secondary datasource should skip to send Answer to P-GW. - context.setSkipAnswer(true); - secondary.handleRequest(context); - } else { + + switch (context.getOriginalCreditControlRequest().getRequestTypeAVPValue()) { + case CreditControlRequestType.INITIAL_REQUEST_VALUE: secondary.handleRequest(context); - } + break; + case CreditControlRequestType.UPDATE_REQUEST_VALUE: + if (!secondary.isBlocked(context.getCreditControlRequest().getMsisdn())) { + proxyAnswer(context); + } else { + secondary.handleRequest(context); + } + break; + case CreditControlRequestType.TERMINATION_REQUEST_VALUE: + proxyAnswer(context); + break; + default: + LOG.warn("Unknown request type : {}", context.getOriginalCreditControlRequest().getRequestTypeAVPValue()); } } + /** + * Use the local data source to send an answer directly to P-GW. + * Use secondary to report the usage to OCS. + */ + private void proxyAnswer(CreditControlContext context) { + local.handleRequest(context); + context.setSkipAnswer(true); + secondary.handleRequest(context); + } + @Override public boolean isBlocked(final String msisdn) { return secondary.isBlocked(msisdn); From 7137b5294eb04c9e8a5e7091dd029abdd7275c37 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Fri, 29 Nov 2019 14:15:28 +0100 Subject: [PATCH 34/56] Fixes issues raised in PR review --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 27 ++++++++++--------- .../org/ostelco/prime/storage/graph/Util.kt | 12 ++++++++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 7a0b47268..0e52eacf1 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1328,7 +1328,7 @@ object Neo4jStoreSingleton : GraphStore { /* Adds purchase to customer history. */ AuditLog.info(customerId = customer.id, - message = "Purchased product $sku for ${product.price.amount} ${product.price.currency} (invoice: $invoiceId, charge-id: $chargeId)") + message = "Purchased product $sku for ${formatMoney(product.price)} (invoice: $invoiceId, charge-id: $chargeId)") /* TODO: While aborting transactions, send a record with "reverted" status. */ analyticsReporter.reportPurchase( @@ -1528,9 +1528,10 @@ object Neo4jStoreSingleton : GraphStore { PaymentStatus.PAYMENT_SUCCEEDED -> { } PaymentStatus.REQUIRES_PAYMENT_METHOD -> { - NotCreatedError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", - error = InvalidRequestError("Payment method failed")) - .left().bind() + NotCreatedError(type = "Customer subscription to Plan", + id = "$customerId -> ${plan.id}", + error = InvalidRequestError("Payment method failed") + ).left().bind() } PaymentStatus.REQUIRES_ACTION, PaymentStatus.TRIAL_START -> { @@ -1606,10 +1607,10 @@ object Neo4jStoreSingleton : GraphStore { it }.linkReversalActionToTransaction(transaction) { paymentProcessor.removeInvoice(it.id) - logger.warn(NOTIFY_OPS_MARKER, - """Failed to pay invoice for customer ${customer.id}, invoice-id: ${it.id}. - Verify that the invoice has been deleted or voided in Stripe dashboard. - """.trimIndent()) + logger.warn(NOTIFY_OPS_MARKER, """ + Failed to pay invoice for customer ${customer.id}, invoice-id: ${it.id}. + Verify that the invoice has been deleted or voided in Stripe dashboard. + """.trimIndent()) }.bind() /* Force immediate payment of the invoice. */ @@ -1618,15 +1619,15 @@ object Neo4jStoreSingleton : GraphStore { logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") /* Adds failed purchase to customer history. */ AuditLog.warn(customerId = customer.id, - message = "Failed to complete purchase of product $sku for $price.amount} ${price.currency} " + + message = "Failed to complete purchase of product $sku for ${formatMoney(price)} " + "status: ${it.code} decline reason: ${it.declineCode}") it }.linkReversalActionToTransaction(transaction) { paymentProcessor.refundCharge(it.chargeId) - logger.warn(NOTIFY_OPS_MARKER, - """Refunded customer ${customer.id} for invoice: ${it.id}. - Verify that the invoice has been refunded in Stripe dashboard. - """.trimIndent()) + logger.warn(NOTIFY_OPS_MARKER, """ + Refunded customer ${customer.id} for invoice: ${it.id}. + Verify that the invoice has been refunded in Stripe dashboard. + """.trimIndent()) }.bind() invoicePaymentInfo diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Util.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Util.kt index ecb007b58..a4cc3ff2f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Util.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Util.kt @@ -51,4 +51,14 @@ fun createProduct(sku: String, taxRegionId: String? = null): Product { emptyMap() } ) -} \ No newline at end of file +} + +/** + * Formatting of amounts. + * TODO (kmm) Update to use the java.text.NumberFormat API or the new + * JSR-354 Currency and Money API. + */ +fun formatMoney(amount: Int, currency: String): String = DecimalFormat("#,###.##") + .format(amount / 100.0) + " ${currency.toUpperCase()}" + +fun formatMoney(price: Price): String = formatMoney(price.amount, price.currency) From cbbd9196e7e5021cee5c1359e5faa9fea26bdebd Mon Sep 17 00:00:00 2001 From: mpeterss Date: Fri, 29 Nov 2019 14:25:47 +0100 Subject: [PATCH 35/56] Do not fetch session when skipping CCA --- .../datasource/protobuf/ProtobufDataSource.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt index d43c921f3..4b3f2cdb9 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt @@ -48,11 +48,11 @@ class ProtobufDataSource { if (ccrContext != null) { ccrContext.logLatency() logger.debug("Found Context for answer msisdn {} requestId [{}] request number {}", ccrContext.creditControlRequest.msisdn, ccrContext.sessionId, ccrContext.creditControlRequest.ccRequestNumber?.integer32) - val session = OcsServer.stack?.getSession(ccrContext.sessionId, ServerCCASession::class.java) - if (session != null && session.isValid) { - removeFromSessionMap(ccrContext) - updateBlockedList(answer, ccrContext.creditControlRequest) - if (!ccrContext.skipAnswer) { + removeFromSessionMap(ccrContext) + updateBlockedList(answer, ccrContext.creditControlRequest) + if (!ccrContext.skipAnswer) { + val session = OcsServer.stack?.getSession(ccrContext.sessionId, ServerCCASession::class.java) + if (session != null && session.isValid) { val cca = createCreditControlAnswer(answer) try { session.sendCreditControlAnswer(ccrContext.createCCA(cca)) @@ -65,9 +65,9 @@ class ProtobufDataSource { } catch (e: OverloadException) { logger.error("Failed to send Credit-Control-Answer msisdn {} requestId {}", answer.msisdn, answer.requestId, e) } + } else { + logger.warn("No session found for [{}] [{}] [{}]", answer.msisdn, answer.requestId, answer.requestNumber) } - } else { - logger.warn("No session found for [{}] [{}] [{}]", answer.msisdn, answer.requestId, answer.requestNumber) } } else { logger.warn("Missing CreditControlContext for [{}] [{}] [{}]", answer.msisdn, answer.requestId, answer.requestNumber) From a2bd5e9d1d7498bca145fc5a30f0effe2326b386 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 10:03:31 +0100 Subject: [PATCH 36/56] Corrected log --- .../src/main/kotlin/org/ostelco/simcards/admin/SimManager.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/SimManager.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/SimManager.kt index eb3a9aeb8..3b9bc59e4 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/SimManager.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/SimManager.kt @@ -26,7 +26,7 @@ object SimManagerSingleton : SimManager { return simInventoryApi.findSimProfileByIccid(hlrName = hlr, iccid = iccId) .map { simEntry -> mapToModelSimEntry(simEntry) } .mapLeft { - logger.error("Failed to get SIM Profile, hlr = {}, ICCID = {}, description: {}", iccId, hlr, it.description) + logger.error("Failed to get SIM Profile, hlr = {}, ICCID = {}, description: {}", hlr, iccId, it.description) it.description } } From dbf0e79c3a54146e76237aab979f1ca72d9997c0 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 11:13:30 +0100 Subject: [PATCH 37/56] Updated all dependencies --- README.md | 2 +- build.gradle.kts | 2 +- .../kotlin/org/ostelco/prime/gradle/Version.kt | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f7c9f7870..66bb5439d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.3.60-blue.svg)](http://kotlinlang.org/) +[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.3.61-blue.svg)](http://kotlinlang.org/) [![Prime version](https://img.shields.io/github/tag/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/tags) [![GitHub license](https://img.shields.io/github/license/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/blob/master/LICENSE) diff --git a/build.gradle.kts b/build.gradle.kts index 1d43ce22f..253f90f17 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,7 +10,7 @@ plugins { id("project-report") id("com.github.ben-manes.versions") version "0.27.0" jacoco - kotlin("jvm") version "1.3.60" apply false + kotlin("jvm") version "1.3.61" apply false id("com.google.protobuf") version "0.8.10" apply false id("com.github.johnrengelman.shadow") version "5.2.0" apply false idea diff --git a/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt b/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt index 138716a32..5d967162f 100644 --- a/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt +++ b/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt @@ -5,11 +5,11 @@ object Version { const val arrow = "0.8.2" - const val byteBuddy = "1.10.3" + const val byteBuddy = "1.10.4" const val csv = "1.7" const val cxf = "3.3.4" const val dockerComposeJunitRule = "1.3.0" - const val dropwizard = "1.3.16" + const val dropwizard = "1.3.17" const val metrics = "4.1.1" const val firebase = "6.11.0" @@ -33,23 +33,23 @@ object Version { const val jdbi3 = "3.11.1" const val jjwt = "0.10.7" const val junit5 = "5.5.2" - const val kotlin = "1.3.60" - const val kotlinXCoroutines = "1.3.2" - const val mockito = "3.1.0" + const val kotlin = "1.3.61" + const val kotlinXCoroutines = "1.3.2-1.3.60" + const val mockito = "3.2.0" const val mockitoKotlin = "2.2.0" const val neo4jDriver = "1.7.5" - const val neo4j = "3.5.12" + const val neo4j = "3.5.13" const val opencensus = "0.24.0" const val postgresql = "42.2.8" // See comment in ./sim-administration/simmanager/build.gradle const val prometheusDropwizard = "2.2.0" - const val protoc = "3.10.0" + const val protoc = "3.11.0" const val slf4j = "1.7.29" // IMPORTANT: When Stripe SDK library version is updated, check if the Stripe API version has changed. // If so, then update API version in Stripe Web Console for callback Webhooks. - const val stripe = "15.4.0" + const val stripe = "15.7.0" const val swagger = "2.1.0" const val swaggerCodegen = "2.4.10" - const val testcontainers = "1.12.3" + const val testcontainers = "1.12.4" const val tink = "1.2.2" const val zxing = "3.4.0" } \ No newline at end of file From da66c5949defd250882fd6d6ec9fda74cb203d1e Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 13:42:59 +0100 Subject: [PATCH 38/56] Minor fixes in ocs-ktc --- ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt | 6 +----- .../kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt | 4 ++-- .../ostelco/prime/ocs/consumption/grpc/OcsGrpcServerTest.kt | 1 + .../ostelco/prime/ocs/consumption/pubsub/OcsPubSubTest.kt | 1 + .../kotlin/org/ostelco/prime/ocs/core/OnlineChargingTest.kt | 1 + 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt index bd8051836..37870500f 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/OcsModule.kt @@ -13,6 +13,7 @@ import org.ostelco.prime.ocs.consumption.pubsub.PubSubClient import org.ostelco.prime.ocs.core.OnlineCharging @JsonTypeName("ocs") +@ExperimentalUnsignedTypes class OcsModule : PrimeModule { @JsonProperty @@ -46,11 +47,6 @@ class OcsModule : PrimeModule { } } -data class Rate( - val serviceId: Long, - val ratingGroup: Long, - val rate: String) - data class PubSubChannel( val projectId: String, val activateTopicId: String, diff --git a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt index 2f19e77df..909704ff4 100644 --- a/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt +++ b/ocs-ktc/src/main/kotlin/org/ostelco/prime/ocs/core/OnlineCharging.kt @@ -91,7 +91,7 @@ object OnlineCharging : OcsAsyncRequestConsumer { } } - private fun sendCreditControlAnswer(returnCreditControlAnswer: (CreditControlAnswerInfo) -> kotlin.Unit, + private fun sendCreditControlAnswer(returnCreditControlAnswer: (CreditControlAnswerInfo) -> Unit, responseBuilder: CreditControlAnswerInfo.Builder) { synchronized(OnlineCharging) { returnCreditControlAnswer(responseBuilder.build()) @@ -144,7 +144,7 @@ object OnlineCharging : OcsAsyncRequestConsumer { sgsnMccMnc = getUserLocationMccMnc(request), apn = request.serviceInformation.psInformation.calledStationId, imsiMccMnc = request.serviceInformation.psInformation.imsiMccMnc) - .bimap( + .fold( { consumptionResult -> consumptionResultHandler(consumptionResult) }, { consumptionRequest -> consumeRequestHandler(consumptionRequest) } ) diff --git a/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/grpc/OcsGrpcServerTest.kt b/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/grpc/OcsGrpcServerTest.kt index 208ecc1bd..14bf18bbd 100644 --- a/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/grpc/OcsGrpcServerTest.kt +++ b/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/grpc/OcsGrpcServerTest.kt @@ -18,6 +18,7 @@ import java.util.concurrent.CountDownLatch import kotlin.system.measureTimeMillis import kotlin.test.AfterTest +@ExperimentalUnsignedTypes class OcsGrpcServerTest { private lateinit var server: OcsGrpcServer diff --git a/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/pubsub/OcsPubSubTest.kt b/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/pubsub/OcsPubSubTest.kt index 8da50bfb7..c30725495 100644 --- a/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/pubsub/OcsPubSubTest.kt +++ b/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/consumption/pubsub/OcsPubSubTest.kt @@ -27,6 +27,7 @@ private const val CCR_SUBSCRIPTION = "ocs-ccr-sub" private const val CCA_SUBSCRIPTION = "ocsgw-cca-sub" private const val ACTIVATE_SUBSCRIPTION = "ocsgw-activate-sub" +@ExperimentalUnsignedTypes class OcsPubSubTest { private val logger by getLogger() diff --git a/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/core/OnlineChargingTest.kt b/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/core/OnlineChargingTest.kt index 3b86f9bf9..027bbb6f4 100644 --- a/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/core/OnlineChargingTest.kt +++ b/ocs-ktc/src/test/kotlin/org/ostelco/prime/ocs/core/OnlineChargingTest.kt @@ -13,6 +13,7 @@ import java.util.concurrent.CountDownLatch import kotlin.system.measureTimeMillis import kotlin.test.fail +@ExperimentalUnsignedTypes class OnlineChargingTest { @Ignore From e8e74f37409b809fcc20fd47bb811bd35b86f40a Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 12:15:05 +0100 Subject: [PATCH 39/56] Updated Arrow to 0.10.* --- .../org/ostelco/prime/gradle/Version.kt | 2 +- .../ostelco/prime/storage/graph/Neo4jStore.kt | 796 +++++++++--------- prime-modules/build.gradle.kts | 6 +- scaninfo-shredder/build.gradle.kts | 6 +- .../hss-adapter/build.gradle.kts | 4 +- .../simmanager/build.gradle.kts | 4 +- 6 files changed, 392 insertions(+), 426 deletions(-) diff --git a/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt b/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt index 5d967162f..287f28ee4 100644 --- a/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt +++ b/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt @@ -3,7 +3,7 @@ package org.ostelco.prime.gradle object Version { const val assertJ = "3.14.0" - const val arrow = "0.8.2" + const val arrow = "0.10.3" const val byteBuddy = "1.10.4" const val csv = "1.7" diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 0e52eacf1..2ff1ac833 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1,19 +1,21 @@ package org.ostelco.prime.storage.graph -// Some of the model classes cannot be directly used in Graph Store as Entities. -// See documentation in [model/Model.kt] for more details. import arrow.core.Either import arrow.core.Either.Left import arrow.core.Either.Right import arrow.core.EitherOf +import arrow.core.extensions.either.monad.monad +import arrow.core.extensions.fx import arrow.core.fix import arrow.core.flatMap import arrow.core.getOrHandle import arrow.core.left import arrow.core.leftIfNull import arrow.core.right -import arrow.effects.IO -import arrow.instances.either.monad.monad +import arrow.fx.IO +import arrow.fx.extensions.fx +import arrow.fx.extensions.io.monad.monad +import arrow.fx.fix import org.neo4j.driver.v1.Transaction import org.ostelco.prime.analytics.AnalyticsService import org.ostelco.prime.appnotifier.AppNotifier @@ -148,6 +150,9 @@ import kotlin.collections.component1 import kotlin.collections.component2 import kotlin.collections.set import kotlin.reflect.KClass + +// Some of the model classes cannot be directly used in Graph Store as Entities. +// See documentation in [model/Model.kt] for more details. import org.ostelco.prime.model.Identity as ModelIdentity import org.ostelco.prime.model.Offer as ModelOffer import org.ostelco.prime.model.Segment as ModelSegment @@ -404,23 +409,20 @@ object Neo4jStoreSingleton : GraphStore { // Here it runs IO synchronously and returning its result blocking the current thread. // https://arrow-kt.io/docs/patterns/monad_comprehensions/#comprehensions-over-coroutines // https://arrow-kt.io/docs/effects/io/#unsaferunsync - IO { - Either.monad().binding { - validateCreateCustomerParams(customer, referredBy).bind() - val bundleId = UUID.randomUUID().toString() - create { Identity(id = identity.id, type = identity.type) }.bind() - create { customer.copy(createdOn = utcTimeNow()) }.bind() - fact { (Identity withId identity.id) identifies (Customer withId customer.id) using Identifies(provider = identity.provider) }.bind() - create { Bundle(id = bundleId, balance = 0L) }.bind() - fact { (Customer withId customer.id) hasBundle (Bundle withId bundleId) }.bind() - if (referredBy != null) { - fact { (Customer withId referredBy) referred (Customer withId customer.id) }.bind() - } - onNewCustomerAction.apply(identity = identity, customer = customer, transaction = transaction).bind() - AuditLog.info(customerId = customer.id, message = "Customer is created") - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + Either.fx { + validateCreateCustomerParams(customer, referredBy).bind() + val bundleId = UUID.randomUUID().toString() + create { Identity(id = identity.id, type = identity.type) }.bind() + create { customer.copy(createdOn = utcTimeNow()) }.bind() + fact { (Identity withId identity.id) identifies (Customer withId customer.id) using Identifies(provider = identity.provider) }//.bind() + create { Bundle(id = bundleId, balance = 0L) }.bind() + fact { (Customer withId customer.id) hasBundle (Bundle withId bundleId) }.bind() + if (referredBy != null) { + fact { (Customer withId referredBy) referred (Customer withId customer.id) }.bind() + } + onNewCustomerAction.apply(identity = identity, customer = customer, transaction = transaction).bind() + AuditLog.info(customerId = customer.id, message = "Customer is created") + }.ifFailedThenRollback(transaction) } override fun updateCustomer( @@ -440,75 +442,72 @@ object Neo4jStoreSingleton : GraphStore { } override fun removeCustomer(identity: ModelIdentity): Either = writeTransaction { - IO { - Either.monad().binding { - // get customer id - val customer = getCustomer(identity).bind() - val customerId = customer.id - // create ex-customer with same id - create { ExCustomer(id = customerId, terminationDate = LocalDate.now().toString(), createdOn = customer.createdOn) }.bind() - // get all subscriptions and link them to ex-customer - val subscriptions = get(Subscription subscribedBy (Customer withId customerId)).bind() - for (subscription in subscriptions) { - fact { (ExCustomer withId customerId) subscribedTo (Subscription withMsisdn subscription.msisdn) }.bind() - } - // get all SIM profiles and link them to ex-customer. - val simProfiles = get(SimProfile forCustomer (Customer withId customerId)).bind() - val simProfileRegions = mutableSetOf() - for (simProfile in simProfiles) { - fact { (ExCustomer withId customerId) had (SimProfile withId simProfile.id) }.bind() - // also get regions linked to those SimProfiles. - simProfileRegions.addAll(get(Region linkedToSimProfile (SimProfile withId simProfile.id)).bind()) - } - // get Regions linked to Customer - val regions = get(Region linkedToCustomer (Customer withId customerId)).bind() - // TODO vihang: clear eKYC data for Regions without any SimProfile -// val regionsWithoutSimProfile = regions - simProfileRegions -// // Link regions with SIM profiles to ExCustomer -// for (region in simProfileRegions) { -// fact { (ExCustomer withId customerId) belongedTo (Region withCode region.id) }.bind() -// } - // (For now) Link regions to ExCustomer - for (region in regions) { - fact { (ExCustomer withId customerId) belongedTo (Region withCode region.id) }.bind() - } - - // TODO vihang: When we read and then delete, it fails when deserialization does not work. - write(query = """ - MATCH (i:${identityEntity.name} {id:'${identity.id}'})-[:${identifiesRelation.name}]->(c:${customerEntity.name}) - OPTIONAL MATCH (c)-[:${customerToBundleRelation.name}]->(b:${bundleEntity.name}) - OPTIONAL MATCH (c)<-[:${forPurchaseByRelation.name}]-(pr:${purchaseRecordEntity.name}) - OPTIONAL MATCH (c)-[:${scanInformationRelation.name}]->(s:${scanInformationEntity.name}) - DETACH DELETE i, c, b, pr, s; - """.trimIndent(), transaction = transaction) { statementResult -> - Either.cond( - test = statementResult.summary().counters().nodesDeleted() > 0, - ifTrue = {}, - ifFalse = { NotFoundError(type = identityEntity.name, id = identity.id) }) - }.bind() + Either.fx { + // get customer id + val customer = getCustomer(identity).bind() + val customerId = customer.id + // create ex-customer with same id + create { ExCustomer(id = customerId, terminationDate = LocalDate.now().toString(), createdOn = customer.createdOn) }.bind() + // get all subscriptions and link them to ex-customer + val subscriptions = get(Subscription subscribedBy (Customer withId customerId)).bind() + for (subscription in subscriptions) { + fact { (ExCustomer withId customerId) subscribedTo (Subscription withMsisdn subscription.msisdn) }.bind() + } + // get all SIM profiles and link them to ex-customer. + val simProfiles = get(SimProfile forCustomer (Customer withId customerId)).bind() + val simProfileRegions = mutableSetOf() + for (simProfile in simProfiles) { + fact { (ExCustomer withId customerId) had (SimProfile withId simProfile.id) }.bind() + // also get regions linked to those SimProfiles. + simProfileRegions.addAll(get(Region linkedToSimProfile (SimProfile withId simProfile.id)).bind()) + } + // get Regions linked to Customer + val regions = get(Region linkedToCustomer (Customer withId customerId)).bind() + // TODO vihang: clear eKYC data for Regions without any SimProfile +// val regionsWithoutSimProfile = regions - simProfileRegions +// // Link regions with SIM profiles to ExCustomer +// for (region in simProfileRegions) { +// fact { (ExCustomer withId customerId) belongedTo (Region withCode region.id) }.bind() +// } + // (For now) Link regions to ExCustomer + for (region in regions) { + fact { (ExCustomer withId customerId) belongedTo (Region withCode region.id) }.bind() + } - /* If removal of payment profile fails, then the customer will be deleted - in neo4j but will still be present in payment backend. In that case the - profile must be removed from the payment backend manually. */ - paymentProcessor.removePaymentProfile(customerId) - .map { - Unit - }.flatMapLeft { - if (it is org.ostelco.prime.paymentprocessor.core.NotFoundError) { - /* Ignore. Customer has not bought products yet. */ - Unit.right() - } else { - logger.error(NOTIFY_OPS_MARKER, - "Removing corresponding payment profile when removing customer $customerId " + - "failed with error ${it.message} : ${it.description}") - NotDeletedError(type = "Payment profile for customer", - id = customerId, - error = it).left() - } - }.bind() - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + // TODO vihang: When we read and then delete, it fails when deserialization does not work. + write(query = """ + MATCH (i:${identityEntity.name} {id:'${identity.id}'})-[:${identifiesRelation.name}]->(c:${customerEntity.name}) + OPTIONAL MATCH (c)-[:${customerToBundleRelation.name}]->(b:${bundleEntity.name}) + OPTIONAL MATCH (c)<-[:${forPurchaseByRelation.name}]-(pr:${purchaseRecordEntity.name}) + OPTIONAL MATCH (c)-[:${scanInformationRelation.name}]->(s:${scanInformationEntity.name}) + DETACH DELETE i, c, b, pr, s; + """.trimIndent(), transaction = transaction) { statementResult -> + Either.cond( + test = statementResult.summary().counters().nodesDeleted() > 0, + ifTrue = {}, + ifFalse = { NotFoundError(type = identityEntity.name, id = identity.id) }) + }.bind() + + /* If removal of payment profile fails, then the customer will be deleted + in neo4j but will still be present in payment backend. In that case the + profile must be removed from the payment backend manually. */ + paymentProcessor.removePaymentProfile(customerId) + .map { + Unit + }.flatMapLeft { + if (it is org.ostelco.prime.paymentprocessor.core.NotFoundError) { + /* Ignore. Customer has not bought products yet. */ + Unit.right() + } else { + logger.error(NOTIFY_OPS_MARKER, + "Removing corresponding payment profile when removing customer $customerId " + + "failed with error ${it.message} : ${it.description}") + NotDeletedError(type = "Payment profile for customer", + id = customerId, + error = it).left() + } + }.bind() + }.ifFailedThenRollback(transaction) } // @@ -683,38 +682,36 @@ object Neo4jStoreSingleton : GraphStore { fun subscribeToSimProfileStatusUpdates() { simManager.addSimProfileStatusUpdateListener { iccId, status -> writeTransaction { - IO { - Either.monad().binding { - logger.info("Received status {} for iccId {}", status, iccId) - val simProfiles = getSimProfilesUsingIccId(iccId = iccId, transaction = transaction) - if (simProfiles.size != 1) { - logger.warn("Found {} SIM Profiles with iccId {}", simProfiles.size, iccId) + Either.fx { + logger.info("Received status {} for iccId {}", status, iccId) + val simProfiles = getSimProfilesUsingIccId(iccId = iccId, transaction = transaction) + if (simProfiles.size != 1) { + logger.warn("Found {} SIM Profiles with iccId {}", simProfiles.size, iccId) + } + simProfiles.forEach { simProfile -> + val customers = get(Customer withSimProfile (SimProfile withId simProfile.id)).bind() + customers.forEach { customer -> + AuditLog.info(customerId = customer.id, message = "Sim Profile (iccId = $iccId) is $status") } - simProfiles.forEach { simProfile -> - val customers = get(Customer withSimProfile (SimProfile withId simProfile.id)).bind() - customers.forEach { customer -> - AuditLog.info(customerId = customer.id, message = "Sim Profile (iccId = $iccId) is $status") - } - val timestampField = when(status) { - DOWNLOADED -> SimProfile::downloadedOn - INSTALLED -> SimProfile::installedOn - DELETED -> SimProfile::deletedOn - else -> { - logger.warn("Not storing timestamp for simProfile: {} for status: {}", iccId, status) - null - } - } - if (timestampField != null) { - update(SimProfile withId simProfile.id, set = timestampField to utcTimeNow()).bind() - } - val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() - subscriptions.forEach { subscription -> - logger.info("Notify status {} for subscription.analyticsId {}", status, subscription.analyticsId) - analyticsReporter.reportSubscriptionStatusUpdate(subscription.analyticsId, status) + val timestampField = when(status) { + DOWNLOADED -> SimProfile::downloadedOn + INSTALLED -> SimProfile::installedOn + DELETED -> SimProfile::deletedOn + else -> { + logger.warn("Not storing timestamp for simProfile: {} for status: {}", iccId, status) + null } } - }.fix() - }.unsafeRunSync() + if (timestampField != null) { + update(SimProfile withId simProfile.id, set = timestampField to utcTimeNow()).bind() + } + val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() + subscriptions.forEach { subscription -> + logger.info("Notify status {} for subscription.analyticsId {}", status, subscription.analyticsId) + analyticsReporter.reportSubscriptionStatusUpdate(subscription.analyticsId, status) + } + } + } // Skipping transaction rollback since it is just updating timestamps } } @@ -742,72 +739,69 @@ object Neo4jStoreSingleton : GraphStore { regionCode: String, profileType: String?, alias: String): Either = writeTransaction { - IO { - Either.monad().binding { - val customerId = getCustomerId(identity = identity).bind() - val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() - validateBundleList(bundles, customerId).bind() - val customer = get(Customer withId customerId).bind() - val status = customerRegionRelationStore.get( - fromId = customerId, - toId = regionCode.toLowerCase(), - transaction = transaction) - .bind() - .status - isApproved( - status = status, - customerId = customerId, - regionCode = regionCode.toLowerCase()).bind() - val region = get(Region withCode regionCode.toLowerCase()).bind() - val simEntry = simManager.allocateNextEsimProfile(hlr = hssNameLookup.getHssName(region.id.toLowerCase()), phoneType = profileType) - .mapLeft { NotFoundError("eSIM profile", id = "Loltel") } - .bind() - val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId, alias = alias, requestedOn = utcTimeNow()) - create { simProfile }.bind() - fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() - fact { (SimProfile withId simProfile.id) isFor (Region withCode regionCode.toLowerCase()) }.bind() - simEntry.msisdnList.forEach { msisdn -> - create { Subscription(msisdn = msisdn) }.bind() - val subscription = get(Subscription withMsisdn msisdn).bind() - - // Report the new provisioning to analytics - analyticsReporter.reportSimProvisioning( - subscriptionAnalyticsId = subscription.analyticsId, - customerAnalyticsId = customer.analyticsId, - regionCode = regionCode - ) + Either.fx { + val customerId = getCustomerId(identity = identity).bind() + val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() + validateBundleList(bundles, customerId).bind() + val customer = get(Customer withId customerId).bind() + val status = customerRegionRelationStore.get( + fromId = customerId, + toId = regionCode.toLowerCase(), + transaction = transaction) + .bind() + .status + isApproved( + status = status, + customerId = customerId, + regionCode = regionCode.toLowerCase()).bind() + val region = get(Region withCode regionCode.toLowerCase()).bind() + val simEntry = simManager.allocateNextEsimProfile(hlr = hssNameLookup.getHssName(region.id.toLowerCase()), phoneType = profileType) + .mapLeft { NotFoundError("eSIM profile", id = "Loltel") } + .bind() + val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId, alias = alias, requestedOn = utcTimeNow()) + create { simProfile }.bind() + fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() + fact { (SimProfile withId simProfile.id) isFor (Region withCode regionCode.toLowerCase()) }.bind() + simEntry.msisdnList.forEach { msisdn -> + create { Subscription(msisdn = msisdn) }.bind() + val subscription = get(Subscription withMsisdn msisdn).bind() + + // Report the new provisioning to analytics + analyticsReporter.reportSimProvisioning( + subscriptionAnalyticsId = subscription.analyticsId, + customerAnalyticsId = customer.analyticsId, + regionCode = regionCode + ) - bundles.forEach { bundle -> - fact { (Subscription withMsisdn msisdn) consumesFrom (Bundle withId bundle.id) using SubscriptionToBundle() }.bind() - } - fact { (Customer withId customerId) subscribesTo (Subscription withMsisdn msisdn) }.bind() - fact { (Subscription withMsisdn msisdn) isUnder (SimProfile withId simProfile.id) }.bind() - } - if (!setOf("android", "iphone", "test").contains(profileType)) { - emailNotifier.sendESimQrCodeEmail( - email = customer.contactEmail, - name = customer.nickname, - qrCode = simEntry.eSimActivationCode) - .mapLeft { - logger.error(NOTIFY_OPS_MARKER, "Failed to send email to {}", customer.contactEmail) - AuditLog.warn(customerId = customerId, message = "Failed to send email with QR code of provisioned SIM Profile") - } + bundles.forEach { bundle -> + fact { (Subscription withMsisdn msisdn) consumesFrom (Bundle withId bundle.id) using SubscriptionToBundle() }.bind() } - AuditLog.info(customerId = customerId, message = "Provisioned SIM Profile") - ModelSimProfile( - iccId = simEntry.iccId, - alias = simProfile.alias, - eSimActivationCode = simEntry.eSimActivationCode, - status = simEntry.status, - requestedOn = simProfile.requestedOn, - downloadedOn = simProfile.downloadedOn, - installedOn = simProfile.installedOn, - installedReportedByAppOn = simProfile.installedReportedByAppOn, - deletedOn = simProfile.deletedOn - ) - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + fact { (Customer withId customerId) subscribesTo (Subscription withMsisdn msisdn) }.bind() + fact { (Subscription withMsisdn msisdn) isUnder (SimProfile withId simProfile.id) }.bind() + } + if (!setOf("android", "iphone", "test").contains(profileType)) { + emailNotifier.sendESimQrCodeEmail( + email = customer.contactEmail, + name = customer.nickname, + qrCode = simEntry.eSimActivationCode) + .mapLeft { + logger.error(NOTIFY_OPS_MARKER, "Failed to send email to {}", customer.contactEmail) + AuditLog.warn(customerId = customerId, message = "Failed to send email with QR code of provisioned SIM Profile") + } + } + AuditLog.info(customerId = customerId, message = "Provisioned SIM Profile") + ModelSimProfile( + iccId = simEntry.iccId, + alias = simProfile.alias, + eSimActivationCode = simEntry.eSimActivationCode, + status = simEntry.status, + requestedOn = simProfile.requestedOn, + downloadedOn = simProfile.downloadedOn, + installedOn = simProfile.installedOn, + installedReportedByAppOn = simProfile.installedReportedByAppOn, + deletedOn = simProfile.deletedOn + ) + }.ifFailedThenRollback(transaction) } private fun isApproved( @@ -832,109 +826,52 @@ object Neo4jStoreSingleton : GraphStore { val map = mutableMapOf() val simProfiles = readTransaction { - IO { - Either.monad().binding { + Either.fx> { - val customerId = getCustomerId(identity = identity).bind() - val simProfiles = get(SimProfile forCustomer (Customer withId customerId)) - .bind() - if (regionCode == null) { - simProfiles.forEach { simProfile -> - val region = get(Region linkedToSimProfile (SimProfile withId simProfile.id)) - .bind() - .firstOrNull() - if (region != null) { - map[simProfile.id] = region.id - } + val customerId = getCustomerId(identity = identity).bind() + val simProfiles = get(SimProfile forCustomer (Customer withId customerId)) + .bind() + if (regionCode == null) { + simProfiles.forEach { simProfile -> + val region = get(Region linkedToSimProfile (SimProfile withId simProfile.id)) + .bind() + .firstOrNull() + if (region != null) { + map[simProfile.id] = region.id } } - simProfiles - }.fix() - }.unsafeRunSync() + } + simProfiles + } } - return IO { - Either.monad().binding { - simProfiles.bind().map { simProfile -> - - val regionId = (regionCode ?: map[simProfile.id]) - - val simEntry = if (regionId != null) { - simManager.getSimProfile( - hlr = hssNameLookup.getHssName(regionId), - iccId = simProfile.iccId) - .getOrHandle { error -> - logger.warn("SimProfile not found in SIM Manager DB. region: {}, iccId: {}, error: {}", regionId, simProfile.iccId, error) - SimEntry( - iccId = simProfile.iccId, - status = NOT_READY, - eSimActivationCode = "Dummy eSIM", - msisdnList = emptyList() - ) - } - } else { - logger.warn("SimProfile not linked to any region. iccId: {}", simProfile.iccId) - SimEntry( - iccId = simProfile.iccId, - status = NOT_READY, - eSimActivationCode = "Dummy eSIM", - msisdnList = emptyList() - ) - } - - ModelSimProfile( + return Either.fx { + simProfiles.bind().map { simProfile -> + + val regionId = (regionCode ?: map[simProfile.id]) + + val simEntry = if (regionId != null) { + simManager.getSimProfile( + hlr = hssNameLookup.getHssName(regionId), + iccId = simProfile.iccId) + .getOrHandle { error -> + logger.warn("SimProfile not found in SIM Manager DB. region: {}, iccId: {}, error: {}", regionId, simProfile.iccId, error) + SimEntry( + iccId = simProfile.iccId, + status = NOT_READY, + eSimActivationCode = "Dummy eSIM", + msisdnList = emptyList() + ) + } + } else { + logger.warn("SimProfile not linked to any region. iccId: {}", simProfile.iccId) + SimEntry( iccId = simProfile.iccId, - alias = simProfile.alias, - eSimActivationCode = simEntry.eSimActivationCode, - status = simEntry.status, - requestedOn = simProfile.requestedOn, - downloadedOn = simProfile.downloadedOn, - installedOn = simProfile.installedOn, - installedReportedByAppOn = simProfile.installedReportedByAppOn, - deletedOn = simProfile.deletedOn + status = NOT_READY, + eSimActivationCode = "Dummy eSIM", + msisdnList = emptyList() ) } - }.fix() - }.unsafeRunSync() - } - - override fun updateSimProfile( - identity: ModelIdentity, - regionCode: String, - iccId: String, - alias: String): Either { - val simProfileEither = writeTransaction { - IO { - Either.monad().binding { - - val customerId = getCustomerId(identity = identity).bind() - val simProfile = get(SimProfile forCustomer (Customer withId customerId)) - .bind() - .firstOrNull { simProfile -> simProfile.iccId == iccId } - ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() - - update(SimProfile withId simProfile.id, set = SimProfile::alias to alias).bind() - AuditLog.info(customerId = customerId, message = "Updated alias of SIM Profile (iccId = $iccId)") - simProfile.copy(alias = alias) - }.fix() - }.unsafeRunSync().ifFailedThenRollback(transaction) - } - - return IO { - Either.monad().binding { - val simProfile = simProfileEither.bind() - val simEntry = simManager.getSimProfile( - hlr = hssNameLookup.getHssName(regionCode), - iccId = iccId) - .getOrHandle { error -> - logger.warn("SimProfile not found in SIM Manager DB. region: {}, iccId: {}, error: {}", regionCode, simProfile.iccId, error) - SimEntry( - iccId = simProfile.iccId, - status = NOT_READY, - eSimActivationCode = "Dummy eSIM", - msisdnList = emptyList() - ) - } ModelSimProfile( iccId = simProfile.iccId, @@ -947,8 +884,58 @@ object Neo4jStoreSingleton : GraphStore { installedReportedByAppOn = simProfile.installedReportedByAppOn, deletedOn = simProfile.deletedOn ) - }.fix() - }.unsafeRunSync() + } + } + } + + override fun updateSimProfile( + identity: ModelIdentity, + regionCode: String, + iccId: String, + alias: String): Either { + + val simProfileEither = writeTransaction { + Either.fx { + + val customerId = getCustomerId(identity = identity).bind() + val simProfile = get(SimProfile forCustomer (Customer withId customerId)) + .bind() + .firstOrNull { simProfile -> simProfile.iccId == iccId } + ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() + + update(SimProfile withId simProfile.id, set = SimProfile::alias to alias).bind() + AuditLog.info(customerId = customerId, message = "Updated alias of SIM Profile (iccId = $iccId)") + simProfile.copy(alias = alias) + }.ifFailedThenRollback(transaction) + } + + return Either.fx { + val simProfile = simProfileEither.bind() + val simEntry = simManager.getSimProfile( + hlr = hssNameLookup.getHssName(regionCode), + iccId = iccId) + .getOrHandle { error -> + logger.warn("SimProfile not found in SIM Manager DB. region: {}, iccId: {}, error: {}", regionCode, simProfile.iccId, error) + SimEntry( + iccId = simProfile.iccId, + status = NOT_READY, + eSimActivationCode = "Dummy eSIM", + msisdnList = emptyList() + ) + } + + ModelSimProfile( + iccId = simProfile.iccId, + alias = simProfile.alias, + eSimActivationCode = simEntry.eSimActivationCode, + status = simEntry.status, + requestedOn = simProfile.requestedOn, + downloadedOn = simProfile.downloadedOn, + installedOn = simProfile.installedOn, + installedReportedByAppOn = simProfile.installedReportedByAppOn, + deletedOn = simProfile.deletedOn + ) + } } override fun markSimProfileAsInstalled( @@ -957,52 +944,49 @@ object Neo4jStoreSingleton : GraphStore { iccId: String): Either { val simProfileEither = writeTransaction { - IO { - Either.monad().binding { + Either.fx { - val customerId = getCustomerId(identity = identity).bind() - val simProfile = get(SimProfile forCustomer (Customer withId customerId)) - .bind() - .firstOrNull { simProfile -> simProfile.iccId == iccId } - ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() - - val utcTimeNow = utcTimeNow() - update(SimProfile withId simProfile.id, set = SimProfile::installedReportedByAppOn to utcTimeNow).bind() - AuditLog.info(customerId = customerId, message = "App reported SIM Profile (iccId = $iccId) as installed.") - simProfile.copy(installedReportedByAppOn = utcTimeNow) - }.fix() - }.unsafeRunSync().ifFailedThenRollback(transaction) + val customerId = getCustomerId(identity = identity).bind() + val simProfile = get(SimProfile forCustomer (Customer withId customerId)) + .bind() + .firstOrNull { simProfile -> simProfile.iccId == iccId } + ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() + + val utcTimeNow = utcTimeNow() + update(SimProfile withId simProfile.id, set = SimProfile::installedReportedByAppOn to utcTimeNow).bind() + AuditLog.info(customerId = customerId, message = "App reported SIM Profile (iccId = $iccId) as installed.") + simProfile.copy(installedReportedByAppOn = utcTimeNow) + + }.ifFailedThenRollback(transaction) } - return IO { - Either.monad().binding { - val simProfile = simProfileEither.bind() - val simEntry = simManager.getSimProfile( - hlr = hssNameLookup.getHssName(regionCode), - iccId = iccId) - .getOrHandle { error -> - logger.warn("SimProfile not found in SIM Manager DB. region: {}, iccId: {}, error: {}", regionCode, simProfile.iccId, error) - SimEntry( - iccId = simProfile.iccId, - status = NOT_READY, - eSimActivationCode = "Dummy eSIM", - msisdnList = emptyList() - ) - } + return Either.fx { + val simProfile = simProfileEither.bind() + val simEntry = simManager.getSimProfile( + hlr = hssNameLookup.getHssName(regionCode), + iccId = iccId) + .getOrHandle { error -> + logger.warn("SimProfile not found in SIM Manager DB. region: {}, iccId: {}, error: {}", regionCode, simProfile.iccId, error) + SimEntry( + iccId = simProfile.iccId, + status = NOT_READY, + eSimActivationCode = "Dummy eSIM", + msisdnList = emptyList() + ) + } - ModelSimProfile( - iccId = simProfile.iccId, - alias = simProfile.alias, - eSimActivationCode = simEntry.eSimActivationCode, - status = simEntry.status, - requestedOn = simProfile.requestedOn, - downloadedOn = simProfile.downloadedOn, - installedOn = simProfile.installedOn, - installedReportedByAppOn = simProfile.installedReportedByAppOn, - deletedOn = simProfile.deletedOn - ) - }.fix() - }.unsafeRunSync() + ModelSimProfile( + iccId = simProfile.iccId, + alias = simProfile.alias, + eSimActivationCode = simEntry.eSimActivationCode, + status = simEntry.status, + requestedOn = simProfile.requestedOn, + downloadedOn = simProfile.downloadedOn, + installedOn = simProfile.installedOn, + installedReportedByAppOn = simProfile.installedReportedByAppOn, + deletedOn = simProfile.deletedOn + ) + } } override fun sendEmailWithActivationQrCode( @@ -1011,58 +995,53 @@ object Neo4jStoreSingleton : GraphStore { iccId: String): Either { val infoEither = readTransaction { - IO { - Either.monad().binding { + Either.fx> { - val customer = getCustomer(identity = identity).bind() - val simProfile = get(SimProfile forCustomer (Customer withId customer.id)) - .bind() - .firstOrNull { simProfile -> simProfile.iccId == iccId } - ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() + val customer = getCustomer(identity = identity).bind() + val simProfile = get(SimProfile forCustomer (Customer withId customer.id)) + .bind() + .firstOrNull { simProfile -> simProfile.iccId == iccId } + ?: NotFoundError(type = simProfileEntity.name, id = iccId).left().bind() - Pair(customer, simProfile) - }.fix() - }.unsafeRunSync() + Pair(customer, simProfile) + } } - return IO { - Either.monad().binding { - val (customer, simProfile) = infoEither.bind() - val simEntry = simManager.getSimProfile( - hlr = hssNameLookup.getHssName(regionCode), - iccId = iccId) - .mapLeft { - NotFoundError(type = simProfileEntity.name, id = simProfile.iccId) - } - .bind() + return Either.fx { + val (customer, simProfile) = infoEither.bind() + val simEntry = simManager.getSimProfile( + hlr = hssNameLookup.getHssName(regionCode), + iccId = iccId) + .mapLeft { + NotFoundError(type = simProfileEntity.name, id = simProfile.iccId) + } + .bind() - emailNotifier.sendESimQrCodeEmail( - email = customer.contactEmail, - name = customer.nickname, - qrCode = simEntry.eSimActivationCode) - .mapLeft { - SystemError(type = "EMAIL", id = customer.contactEmail, message = "Failed to send email") - } - .bind() + emailNotifier.sendESimQrCodeEmail( + email = customer.contactEmail, + name = customer.nickname, + qrCode = simEntry.eSimActivationCode) + .mapLeft { + SystemError(type = "EMAIL", id = customer.contactEmail, message = "Failed to send email") + } + .bind() - ModelSimProfile( - iccId = simEntry.iccId, - eSimActivationCode = simEntry.eSimActivationCode, - status = simEntry.status, - alias = simProfile.alias, - requestedOn = simProfile.requestedOn, - downloadedOn = simProfile.downloadedOn, - installedOn = simProfile.installedOn, - installedReportedByAppOn = simProfile.installedReportedByAppOn, - deletedOn = simProfile.deletedOn - ) - }.fix() - }.unsafeRunSync() + ModelSimProfile( + iccId = simEntry.iccId, + eSimActivationCode = simEntry.eSimActivationCode, + status = simEntry.status, + alias = simProfile.alias, + requestedOn = simProfile.requestedOn, + downloadedOn = simProfile.downloadedOn, + installedOn = simProfile.installedOn, + installedReportedByAppOn = simProfile.installedReportedByAppOn, + deletedOn = simProfile.deletedOn + ) + } } override fun deleteSimProfileWithSubscription(regionCode: String, iccId: String): Either = writeTransaction { - IO { - Either.monad().binding { + Either.fx { val simProfiles = get(SimProfile linkedToRegion (Region withCode regionCode)).bind() simProfiles.forEach { simProfile -> val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() @@ -1071,9 +1050,7 @@ object Neo4jStoreSingleton : GraphStore { } delete(SimProfile withId simProfile.id).bind() } - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + }.ifFailedThenRollback(transaction) } // @@ -1087,54 +1064,51 @@ object Neo4jStoreSingleton : GraphStore { iccId: String, alias: String, msisdn: String): Either = writeTransaction { - IO { - Either.monad().binding { - val customerId = getCustomerId(identity = identity).bind() - val simProfile = SimProfile( - id = UUID.randomUUID().toString(), - iccId = iccId, - alias = alias) - create { simProfile }.bind() - fact { (SimProfile withId simProfile.id) isFor (Region withCode regionCode.toLowerCase()) }.bind() - fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() - - create { Subscription(msisdn) }.bind() - fact { (Subscription withMsisdn msisdn) isUnder (SimProfile withId simProfile.id) }.bind() - fact { (Customer withId customerId) subscribesTo (Subscription withMsisdn msisdn) }.bind() - - val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() - validateBundleList(bundles, customerId).bind() - bundles.forEach { bundle -> - fact { (Subscription withMsisdn msisdn) consumesFrom (Bundle withId bundle.id) using SubscriptionToBundle() }.bind() - } - AuditLog.info(customerId = customerId, message = "Added SIM Profile and Subscription by Admin") - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + Either.fx { + val customerId = getCustomerId(identity = identity).bind() + + val simProfile = SimProfile( + id = UUID.randomUUID().toString(), + iccId = iccId, + alias = alias) + create { simProfile }.bind() + fact { (SimProfile withId simProfile.id) isFor (Region withCode regionCode.toLowerCase()) }.bind() + fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() + + create { Subscription(msisdn) }.bind() + fact { (Subscription withMsisdn msisdn) isUnder (SimProfile withId simProfile.id) }.bind() + fact { (Customer withId customerId) subscribesTo (Subscription withMsisdn msisdn) }.bind() + + val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() + validateBundleList(bundles, customerId).bind() + bundles.forEach { bundle -> + fact { (Subscription withMsisdn msisdn) consumesFrom (Bundle withId bundle.id) using SubscriptionToBundle() }.bind() + } + AuditLog.info(customerId = customerId, message = "Added SIM Profile and Subscription by Admin") + }.ifFailedThenRollback(transaction) } override fun getSubscriptions(identity: ModelIdentity, regionCode: String?): Either> = readTransaction { - IO { - Either.monad().binding { - val customerId = getCustomerId(identity = identity).bind() - if (regionCode == null) { - get(Subscription subscribedBy (Customer withId customerId)).bind() - } else { - read>>(""" - MATCH (:${customerEntity.name} {id: '$customerId'}) - -[:${subscriptionRelation.name}]->(sn:${subscriptionEntity.name}) - -[:${subscriptionSimProfileRelation.name}]->(:${simProfileEntity.name}) - -[:${simProfileRegionRelation.name}]->(:${regionEntity.name} {id: '${regionCode.toLowerCase()}'}) - RETURN sn; - """.trimIndent(), - transaction) { statementResult -> - Either.right(statementResult - .list { subscriptionEntity.createEntity(it["sn"].asMap()) }) - }.bind() - } - }.fix() - }.unsafeRunSync() + + Either.fx> { + val customerId = getCustomerId(identity = identity).bind() + if (regionCode == null) { + get(Subscription subscribedBy (Customer withId customerId)).bind() + } else { + read>>(""" + MATCH (:${customerEntity.name} {id: '$customerId'}) + -[:${subscriptionRelation.name}]->(sn:${subscriptionEntity.name}) + -[:${subscriptionSimProfileRelation.name}]->(:${simProfileEntity.name}) + -[:${simProfileRegionRelation.name}]->(:${regionEntity.name} {id: '${regionCode.toLowerCase()}'}) + RETURN sn; + """.trimIndent(), + transaction) { statementResult -> + Either.right(statementResult + .list { subscriptionEntity.createEntity(it["sn"].asMap()) }) + }.bind() + } + } } // diff --git a/prime-modules/build.gradle.kts b/prime-modules/build.gradle.kts index 324c50aa9..b90ad1b44 100644 --- a/prime-modules/build.gradle.kts +++ b/prime-modules/build.gradle.kts @@ -21,10 +21,8 @@ dependencies { api("io.dropwizard:dropwizard-core:${Version.dropwizard}") - api("io.arrow-kt:arrow-core:${Version.arrow}") - api("io.arrow-kt:arrow-typeclasses:${Version.arrow}") - api("io.arrow-kt:arrow-instances-core:${Version.arrow}") - api("io.arrow-kt:arrow-effects:${Version.arrow}") + api("io.arrow-kt:arrow-fx:${Version.arrow}") + api("io.arrow-kt:arrow-syntax:${Version.arrow}") runtimeOnly("javax.xml.bind:jaxb-api:${Version.jaxb}") runtimeOnly("javax.activation:activation:${Version.javaxActivation}") diff --git a/scaninfo-shredder/build.gradle.kts b/scaninfo-shredder/build.gradle.kts index 451fca29c..873454005 100644 --- a/scaninfo-shredder/build.gradle.kts +++ b/scaninfo-shredder/build.gradle.kts @@ -24,10 +24,8 @@ dependencies { implementation("com.google.cloud:google-cloud-datastore:${Version.googleCloudDataStore}") implementation("io.arrow-kt:arrow-core:${Version.arrow}") - implementation("io.arrow-kt:arrow-typeclasses:${Version.arrow}") - implementation("io.arrow-kt:arrow-instances-core:${Version.arrow}") - implementation("io.arrow-kt:arrow-effects:${Version.arrow}") - + implementation("io.arrow-kt:arrow-syntax:${Version.arrow}") + runtimeOnly("io.dropwizard:dropwizard-json-logging:${Version.dropwizard}") testImplementation("io.dropwizard:dropwizard-testing:${Version.dropwizard}") diff --git a/sim-administration/hss-adapter/build.gradle.kts b/sim-administration/hss-adapter/build.gradle.kts index ce45b2637..a1511a97f 100644 --- a/sim-administration/hss-adapter/build.gradle.kts +++ b/sim-administration/hss-adapter/build.gradle.kts @@ -26,9 +26,7 @@ dependencies { implementation("javax.annotation:javax.annotation-api:${Version.javaxAnnotation}") api("io.arrow-kt:arrow-core:${Version.arrow}") - api("io.arrow-kt:arrow-typeclasses:${Version.arrow}") - api("io.arrow-kt:arrow-instances-core:${Version.arrow}") - api("io.arrow-kt:arrow-effects:${Version.arrow}") + api("io.arrow-kt:arrow-syntax:${Version.arrow}") implementation(kotlin("reflect")) implementation(kotlin("stdlib-jdk8")) diff --git a/sim-administration/simmanager/build.gradle.kts b/sim-administration/simmanager/build.gradle.kts index 9a143c007..0e99167bb 100644 --- a/sim-administration/simmanager/build.gradle.kts +++ b/sim-administration/simmanager/build.gradle.kts @@ -23,9 +23,7 @@ dependencies { // Arrow api("io.arrow-kt:arrow-core:${Version.arrow}") - api("io.arrow-kt:arrow-typeclasses:${Version.arrow}") - api("io.arrow-kt:arrow-instances-core:${Version.arrow}") - api("io.arrow-kt:arrow-effects:${Version.arrow}") + api("io.arrow-kt:arrow-syntax:${Version.arrow}") // Grpc api("io.grpc:grpc-netty-shaded:${Version.grpc}") From ef87c0971f3bb7fade8fe8437677ba4f5603d041 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 14:15:43 +0100 Subject: [PATCH 40/56] Updated scaninfo-store with Arrow Fx --- .../prime/storage/scaninfo/ScanInfoStore.kt | 99 +++++++++---------- 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/scaninfo-datastore/src/main/kotlin/org/ostelco/prime/storage/scaninfo/ScanInfoStore.kt b/scaninfo-datastore/src/main/kotlin/org/ostelco/prime/storage/scaninfo/ScanInfoStore.kt index 86c93e743..94f226b7f 100644 --- a/scaninfo-datastore/src/main/kotlin/org/ostelco/prime/storage/scaninfo/ScanInfoStore.kt +++ b/scaninfo-datastore/src/main/kotlin/org/ostelco/prime/storage/scaninfo/ScanInfoStore.kt @@ -1,11 +1,9 @@ package org.ostelco.prime.storage.scaninfo import arrow.core.Either -import arrow.core.fix +import arrow.core.extensions.fx import arrow.core.left import arrow.core.right -import arrow.effects.IO -import arrow.instances.either.monad.monad import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import com.google.cloud.datastore.Blob @@ -69,21 +67,20 @@ object ScanInformationStoreSingleton : ScanInformationStore { * 2) -//.zip.tk */ override fun upsertVendorScanInformation(customerId: String, countryCode: String, vendorData: MultivaluedMap): Either { - return IO { - Either.monad().binding { - logger.info("Creating createVendorScanInformation for customerId = $customerId") - val vendorScanInformation = createVendorScanInformation(vendorData).bind() - logger.info("Generating data map for customerId = $customerId") - val dataMap = JumioHelper.toDataMap(vendorScanInformation) - secureArchiveService.archiveEncrypted( - customerId = customerId, - regionCodes = listOf(countryCode), - fileName = vendorScanInformation.id, - dataMap = dataMap).bind() - saveScanMetaData(customerId, countryCode, vendorScanInformation).bind() - Unit - }.fix() - }.unsafeRunSync() + + return Either.fx { + logger.info("Creating createVendorScanInformation for customerId = $customerId") + val vendorScanInformation = createVendorScanInformation(vendorData).bind() + logger.info("Generating data map for customerId = $customerId") + val dataMap = JumioHelper.toDataMap(vendorScanInformation) + secureArchiveService.archiveEncrypted( + customerId = customerId, + regionCodes = listOf(countryCode), + fileName = vendorScanInformation.id, + dataMap = dataMap).bind() + saveScanMetaData(customerId, countryCode, vendorScanInformation).bind() + Unit + } } override fun getExtendedStatusInformation(scanInformation: ScanInformation): Map { @@ -223,42 +220,40 @@ object JumioHelper { val scanImageFaceUrl: String? = vendorData.getFirst(JumioScanData.SCAN_IMAGE_FACE.s) val scanLivenessImagesUrl: List? = vendorData[JumioScanData.SCAN_LIVENESS_IMAGES.s] - return IO { - Either.monad().binding { - var result: Pair - if (scanImageUrl != null) { - logger.info("Downloading scan image: $scanImageUrl") - result = downloadFileAsBlob(scanImageUrl, apiToken, apiSecret).bind() - val filename = "id.${getFileExtFromType(result.second)}" - images.put(filename, result.first) - } - if (scanImageBacksideUrl != null) { - logger.info("Downloading scan image back: $scanImageBacksideUrl") - result = downloadFileAsBlob(scanImageBacksideUrl, apiToken, apiSecret).bind() - val filename = "id_backside.${getFileExtFromType(result.second)}" - images.put(filename, result.first) - } - if (scanImageFaceUrl != null) { - logger.info("Downloading Face Image: $scanImageFaceUrl") - result = downloadFileAsBlob(scanImageFaceUrl, apiToken, apiSecret).bind() - val filename = "face.${getFileExtFromType(result.second)}" + return Either.fx { + var result: Pair + if (scanImageUrl != null) { + logger.info("Downloading scan image: $scanImageUrl") + result = downloadFileAsBlob(scanImageUrl, apiToken, apiSecret).bind() + val filename = "id.${getFileExtFromType(result.second)}" + images.put(filename, result.first) + } + if (scanImageBacksideUrl != null) { + logger.info("Downloading scan image back: $scanImageBacksideUrl") + result = downloadFileAsBlob(scanImageBacksideUrl, apiToken, apiSecret).bind() + val filename = "id_backside.${getFileExtFromType(result.second)}" + images.put(filename, result.first) + } + if (scanImageFaceUrl != null) { + logger.info("Downloading Face Image: $scanImageFaceUrl") + result = downloadFileAsBlob(scanImageFaceUrl, apiToken, apiSecret).bind() + val filename = "face.${getFileExtFromType(result.second)}" + images.put(filename, result.first) + } + if (scanLivenessImagesUrl != null) { + val urls = scanLivenessImagesUrl.toMutableList() + urls.sort() // The url list is not in sequence + val flattenedList = flattenList(urls) + var imageIndex = 0 + for (imageUrl in flattenedList) { + logger.info("Downloading Liveness image: $imageUrl") + result = downloadFileAsBlob(imageUrl, apiToken, apiSecret).bind() + val filename = "liveness-${++imageIndex}.${getFileExtFromType(result.second)}" images.put(filename, result.first) } - if (scanLivenessImagesUrl != null) { - val urls = scanLivenessImagesUrl.toMutableList() - urls.sort() // The url list is not in sequence - val flattenedList = flattenList(urls) - var imageIndex = 0 - for (imageUrl in flattenedList) { - logger.info("Downloading Liveness image: $imageUrl") - result = downloadFileAsBlob(imageUrl, apiToken, apiSecret).bind() - val filename = "liveness-${++imageIndex}.${getFileExtFromType(result.second)}" - images.put(filename, result.first) - } - } - VendorScanInformation(scanId, scanReference, scanDetails, images) - }.fix() - }.unsafeRunSync() + } + VendorScanInformation(scanId, scanReference, scanDetails, images) + } } private fun toRegularMap(jsonData: String?): Map? { From 654a8148752d007a2affde18dabe5a7055e1bf52 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 14:17:29 +0100 Subject: [PATCH 41/56] Updated secure-archive with Arrow FX --- .../prime/securearchive/SecureArchiver.kt | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/secure-archive/src/main/kotlin/org/ostelco/prime/securearchive/SecureArchiver.kt b/secure-archive/src/main/kotlin/org/ostelco/prime/securearchive/SecureArchiver.kt index d3fc9e26f..83ba04ab9 100644 --- a/secure-archive/src/main/kotlin/org/ostelco/prime/securearchive/SecureArchiver.kt +++ b/secure-archive/src/main/kotlin/org/ostelco/prime/securearchive/SecureArchiver.kt @@ -1,9 +1,7 @@ package org.ostelco.prime.securearchive import arrow.core.Either -import arrow.core.fix -import arrow.effects.IO -import arrow.instances.either.monad.monad +import arrow.core.extensions.fx import org.ostelco.prime.getLogger import org.ostelco.prime.securearchive.ConfigRegistry.config import org.ostelco.prime.securearchive.util.Encrypter.Companion.getEncrypter @@ -23,31 +21,29 @@ class SecureArchiver : SecureArchiveService { fileName: String, dataMap: Map): Either { - return IO { - Either.monad().binding { - val bucketName = config.storageBucket - logger.info("Generating Plain Zip data for customerId = {}", customerId) - val plainZipData = generateZipFile(fileName, dataMap).bind() - (regionCodes - .filter(config.regions::contains) - + "global") - .map(String::toLowerCase) - .forEach { regionCode -> - logger.info("Encrypt for region: {} for customerId = {}", regionCode, customerId) - val zipData = getEncrypter(regionCode).encrypt(plainZipData) - if (bucketName.isEmpty()) { - val filePath = "${regionCode}_$fileName.zip.tk" - logger.info("No bucket set, saving file locally {}", filePath) - saveLocalFile(filePath, zipData).bind() - } else { - val filePath = "$customerId/${fileName}_${Instant.now()}.zip.tk" - val bucket = "$bucketName-$regionCode" - logger.info("Saving in cloud storage {} --> {}", bucket, filePath) - uploadFileToCloudStorage(bucket, filePath, zipData).bind() - } + return Either.fx { + val bucketName = config.storageBucket + logger.info("Generating Plain Zip data for customerId = {}", customerId) + val plainZipData = generateZipFile(fileName, dataMap).bind() + (regionCodes + .filter(config.regions::contains) + + "global") + .map(String::toLowerCase) + .forEach { regionCode -> + logger.info("Encrypt for region: {} for customerId = {}", regionCode, customerId) + val zipData = getEncrypter(regionCode).encrypt(plainZipData) + if (bucketName.isEmpty()) { + val filePath = "${regionCode}_$fileName.zip.tk" + logger.info("No bucket set, saving file locally {}", filePath) + saveLocalFile(filePath, zipData).bind() + } else { + val filePath = "$customerId/${fileName}_${Instant.now()}.zip.tk" + val bucket = "$bucketName-$regionCode" + logger.info("Saving in cloud storage {} --> {}", bucket, filePath) + uploadFileToCloudStorage(bucket, filePath, zipData).bind() } - Unit - }.fix() - }.unsafeRunSync() + } + Unit + } } } \ No newline at end of file From e1ae79ecae5f2b7f2cc97ccd585533922b540038 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 14:27:40 +0100 Subject: [PATCH 42/56] Changes in SIM admin for compatibility of new version of dependencies --- .../admin/PeriodicProvisioningTask.kt | 8 ++++--- .../inventory/SimInventoryCallbackService.kt | 2 +- .../ProfileVendorAdapterDatum.kt | 23 ++++++++++--------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt index 518b60668..1933da765 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt @@ -126,8 +126,10 @@ class PreallocateProfilesTask( for (i in 1..noOfProfilesToActuallyAllocate) { logger.debug("preprovisioning for profileName='$simProfileName', HSS with ID/metricName ${hssEntry.id}/${hssEntry.name}. Iteration index = $i") val focus = simInventoryDAO.findNextNonProvisionedSimProfileForHss(hssId = hssEntry.id, profile = simProfileName) - focus.mapLeft { error -> return error.left() } - focus.flatMap { simEntry -> preProvisionSimProfile(simEntry, simProfileName, hssEntry).right() } + if (focus.isLeft()) { + return focus + } + focus.flatMap { simEntry -> preProvisionSimProfile(simEntry, simProfileName, hssEntry) } } // TODO: Replace the text "Unit" with a report of what was actually done by the provisioning @@ -174,7 +176,7 @@ class PreallocateProfilesTask( if (profileStats.noOfUnallocatedEntries == 0L) { val msg = "No more unallocated $thisProfilesNameForLogging" logger.error(msg) - return SystemError(msg).left() + SystemError(msg).left() } else { logger.debug("Ready for use: $thisProfilesNameForLogging = ${profileStats.noOfEntriesAvailableForImmediateUse}") if (profileStats.noOfEntriesAvailableForImmediateUse < lowWaterMark) { diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt index 0bdedce07..edf402604 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt @@ -35,7 +35,7 @@ class SimInventoryCallbackService(val dao: SimInventoryDAO) : SmDpPlusCallbackSe // If we can't find the ICCID, then cry foul and log an error message // that will get the ops team's attention asap! val profileQueryResult = dao.getSimProfileByIccid(numericIccId) - profileQueryResult.mapLeft { + if (profileQueryResult.isLeft()) { logger.error(NOTIFY_OPS_MARKER, "Could not find ICCID='$numericIccId' in database while handling downloadProgressinfo callback!!") return diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/profilevendors/ProfileVendorAdapterDatum.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/profilevendors/ProfileVendorAdapterDatum.kt index 8fdc404fe..4a6467fbf 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/profilevendors/ProfileVendorAdapterDatum.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/profilevendors/ProfileVendorAdapterDatum.kt @@ -192,20 +192,21 @@ data class ProfileVendorAdapter( val matchingId = response.matchingId if (matchingId.isNullOrEmpty()) { - return AdapterError("response.matchingId == null or empty").left() - } + AdapterError("response.matchingId == null or empty").left() + } else { - // TODO: This and the next methods should result in - // failures if they return error values, but they probably don't - // Check out what the ideomatic way of doing this is, also apply that - // finding to the activate method below. + // TODO: This and the next methods should result in + // failures if they return error values, but they probably don't + // Check out what the ideomatic way of doing this is, also apply that + // finding to the activate method below. - dao.setSmDpPlusStateAndMatchingId(simEntry.id, SmDpPlusState.RELEASED, matchingId) + dao.setSmDpPlusStateAndMatchingId(simEntry.id, SmDpPlusState.RELEASED, matchingId) - // TODO Do we really want to do this? Do we need the - // sim entry value as a returnv value? If we don't then - // remove the next line. - dao.getSimProfileById(simEntry.id) + // TODO Do we really want to do this? Do we need the + // sim entry value as a returnv value? If we don't then + // remove the next line. + dao.getSimProfileById(simEntry.id) + } } } From c529c1aa3c70333a09d3cf04ad2c40e57a0ba33a Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 14:33:41 +0100 Subject: [PATCH 43/56] Updated SIM admin to Arrow Fx --- .../simcards/inventory/SimInventoryApi.kt | 107 ++++++------- .../simcards/inventory/SimInventoryDAO.kt | 143 ++++++++---------- 2 files changed, 115 insertions(+), 135 deletions(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryApi.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryApi.kt index 2b11816f7..29e771593 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryApi.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryApi.kt @@ -1,12 +1,10 @@ package org.ostelco.simcards.inventory import arrow.core.Either -import arrow.core.fix +import arrow.core.extensions.fx import arrow.core.flatMap import arrow.core.left import arrow.core.right -import arrow.effects.IO -import arrow.instances.either.monad.monad import org.apache.http.impl.client.CloseableHttpClient import org.ostelco.prime.getLogger import org.ostelco.prime.simmanager.DatabaseError @@ -25,22 +23,18 @@ class SimInventoryApi(private val httpClient: CloseableHttpClient, private val logger by getLogger() - fun findSimProfileByIccid(hlrName: String, iccid: String): Either = - IO { - Either.monad().binding { + fun findSimProfileByIccid(hlrName: String, iccid: String): Either = Either.fx { - val simEntry = dao.getSimProfileByIccid(iccid).bind() - checkForValidHlr(hlrName, simEntry) + val simEntry = dao.getSimProfileByIccid(iccid).bind() + checkForValidHlr(hlrName, simEntry) - val config = getProfileVendorConfig(simEntry).bind() + val config = getProfileVendorConfig(simEntry).bind() - // Return the entry found in the database, extended with a - // code represernting the string that will be used by the LPA in the - // UA to talk to the sim vendor's SM-DP+ over the ES9+ protocol. - simEntry.copy(code = "LPA:1\$${config.es9plusEndpoint}\$${simEntry.matchingId}") - - }.fix() - }.unsafeRunSync() + // Return the entry found in the database, extended with a + // code represernting the string that will be used by the LPA in the + // UA to talk to the sim vendor's SM-DP+ over the ES9+ protocol. + simEntry.copy(code = "LPA:1\$${config.es9plusEndpoint}\$${simEntry.matchingId}") + } fun findSimProfileByImsi(hlrName: String, imsi: String): Either = dao.getSimProfileByImsi(imsi) @@ -65,56 +59,51 @@ class SimInventoryApi(private val httpClient: CloseableHttpClient, } } - fun allocateNextEsimProfile(hlrName: String, phoneType: String): Either = - IO { - Either.monad().binding { - logger.info("Allocating new SIM for hlr ${hlrName} and phone-type ${phoneType}") - - val hlrAdapter = dao.getHssEntryByName(hlrName) - .bind() - val profile = getProfileType(hlrName, phoneType) - .bind() - val simEntry = dao.findNextReadyToUseSimProfileForHss(hlrAdapter.id, profile) - .bind() - val config = getProfileVendorConfig(simEntry) - .bind() - - if (simEntry.id == null) { - DatabaseError("simEntry has no id (simEntry=$simEntry)").left().bind() - } + fun allocateNextEsimProfile(hlrName: String, phoneType: String): Either = Either.fx { + + logger.info("Allocating new SIM for hlr ${hlrName} and phone-type ${phoneType}") + + val hlrAdapter = dao.getHssEntryByName(hlrName) + .bind() + val profile = getProfileType(hlrName, phoneType) + .bind() + val simEntry = dao.findNextReadyToUseSimProfileForHss(hlrAdapter.id, profile) + .bind() + val config = getProfileVendorConfig(simEntry) + .bind() - val updatedSimEntry = dao.setProvisionState(simEntry.id, ProvisionState.PROVISIONED) - .bind() + if (simEntry.id == null) { + DatabaseError("simEntry has no id (simEntry=$simEntry)").left().bind() + } - // TODO: Add 'code' field content. - // Original format: LPA:: - // New format: LPA:1$$ */ - updatedSimEntry.copy(code = "LPA:1\$${config.es9plusEndpoint}\$${updatedSimEntry.matchingId}") - }.fix() - }.unsafeRunSync() + val updatedSimEntry = dao.setProvisionState(simEntry.id, ProvisionState.PROVISIONED) + .bind() + // TODO: Add 'code' field content. + // Original format: LPA:: + // New format: LPA:1$$ */ + updatedSimEntry.copy(code = "LPA:1\$${config.es9plusEndpoint}\$${updatedSimEntry.matchingId}") + } fun importBatch(hlrName: String, simVendor: String, csvInputStream: InputStream, - initialHssState: HssState): Either = - IO { - Either.monad().binding { - val profileVendorAdapter = dao.getProfileVendorAdapterDatumByName(simVendor) - .bind() - val hlrAdapter = dao.getHssEntryByName(hlrName) - .bind() - - /* Exits if not true. */ - dao.simVendorIsPermittedForHlr(profileVendorAdapter.id, hlrAdapter.id) - .bind() - dao.importSims(importer = "importer", // TODO: This is a very strange metricName for an importer .-) - hlrId = hlrAdapter.id, - profileVendorId = profileVendorAdapter.id, - csvInputStream = csvInputStream, - initialHssState = initialHssState).bind() - }.fix() - }.unsafeRunSync() + initialHssState: HssState): Either = Either.fx { + + val profileVendorAdapter = dao.getProfileVendorAdapterDatumByName(simVendor) + .bind() + val hlrAdapter = dao.getHssEntryByName(hlrName) + .bind() + + /* Exits if not true. */ + dao.simVendorIsPermittedForHlr(profileVendorAdapter.id, hlrAdapter.id) + .bind() + dao.importSims(importer = "importer", // TODO: This is a very strange metricName for an importer .-) + hlrId = hlrAdapter.id, + profileVendorId = profileVendorAdapter.id, + csvInputStream = csvInputStream, + initialHssState = initialHssState).bind() + } /* Helper functions. */ diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryDAO.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryDAO.kt index 1500ba5f6..f32ef094d 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryDAO.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryDAO.kt @@ -1,12 +1,10 @@ package org.ostelco.simcards.inventory import arrow.core.Either -import arrow.core.fix +import arrow.core.extensions.fx import arrow.core.flatMap import arrow.core.left import arrow.core.right -import arrow.effects.IO -import arrow.instances.either.monad.monad import com.fasterxml.jackson.annotation.JsonProperty import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVParser @@ -200,18 +198,16 @@ class SimInventoryDAO(private val db: SimInventoryDBWrapperImpl) : SimInventoryD * @return true on successful update */ @Transaction - fun permitVendorForHssByNames(profileVendor: String, hssName: String): Either = - IO { - Either.monad().binding { - val profileVendorAdapter = getProfileVendorAdapterDatumByName(profileVendor) - .bind() - val hlrAdapter = getHssEntryByName(hssName) - .bind() - - storeSimVendorForHssPermission(profileVendorAdapter.id, hlrAdapter.id) - .bind() > 0 - }.fix() - }.unsafeRunSync() + fun permitVendorForHssByNames(profileVendor: String, hssName: String): Either = Either.fx { + + val profileVendorAdapter = getProfileVendorAdapterDatumByName(profileVendor) + .bind() + val hlrAdapter = getHssEntryByName(hssName) + .bind() + + storeSimVendorForHssPermission(profileVendorAdapter.id, hlrAdapter.id) + .bind() > 0 + } // // Importing @@ -228,35 +224,33 @@ class SimInventoryDAO(private val db: SimInventoryDBWrapperImpl) : SimInventoryD hlrId: Long, profileVendorId: Long, csvInputStream: InputStream, - initialHssState: HssState = HssState.NOT_ACTIVATED): Either = - IO { - Either.monad().binding { - createNewSimImportBatch(importer = importer, - hssId = hlrId, - profileVendorId = profileVendorId) - .bind() - val batchId = lastInsertedRowId() - .bind() - val values = SimEntryIterator( - profileVendorId = profileVendorId, - hssId = hlrId, - batchId = batchId, - initialHssState = initialHssState, - csvInputStream = csvInputStream) - insertAll(values) - .bind() - // Because "golden numbers" needs special handling, so we're simply marking them - // as reserved. - reserveGoldenNumbersForBatch(batchId) - updateBatchState(id = batchId, - size = values.count.get(), - status = "SUCCESS", // TODO: Use enumeration, not naked string. - endedAt = System.currentTimeMillis()) - .bind() - getBatchInfo(batchId) - .bind() - }.fix() - }.unsafeRunSync() + initialHssState: HssState = HssState.NOT_ACTIVATED): Either = Either.fx { + + createNewSimImportBatch(importer = importer, + hssId = hlrId, + profileVendorId = profileVendorId) + .bind() + val batchId = lastInsertedRowId() + .bind() + val values = SimEntryIterator( + profileVendorId = profileVendorId, + hssId = hlrId, + batchId = batchId, + initialHssState = initialHssState, + csvInputStream = csvInputStream) + insertAll(values) + .bind() + // Because "golden numbers" needs special handling, so we're simply marking them + // as reserved. + reserveGoldenNumbersForBatch(batchId) + updateBatchState(id = batchId, + size = values.count.get(), + status = "SUCCESS", // TODO: Use enumeration, not naked string. + endedAt = System.currentTimeMillis()) + .bind() + getBatchInfo(batchId) + .bind() + } // // Finding next free SIM card for a particular HLR. @@ -267,38 +261,35 @@ class SimInventoryDAO(private val db: SimInventoryDBWrapperImpl) : SimInventoryD */ fun getProfileStats(@Bind("hssId") hssId: Long, @Bind("simProfile") simProfile: String): - Either = - IO { - Either.monad().binding { - - val keyValuePairs = mutableMapOf() - - getProfileStatsAsKeyValuePairs(hssId = hssId, simProfile = simProfile).bind() - .forEach { keyValuePairs.put(it.key, it.value) } - - fun lookup(key: String) = keyValuePairs[key] - ?.right() - ?: NotFoundError("Could not find key $key").left() - - val noOfEntries = - lookup("NO_OF_ENTRIES").bind() - val noOfUnallocatedEntries = - lookup( "NO_OF_UNALLOCATED_ENTRIES").bind() - val noOfReleasedEntries = - lookup("NO_OF_RELEASED_ENTRIES").bind() - val noOfEntriesAvailableForImmediateUse = - lookup("NO_OF_ENTRIES_READY_FOR_IMMEDIATE_USE").bind() - val noOfReservedEntries = - lookup("NO_OF_RESERVED_ENTRIES").bind() - - SimProfileKeyStatistics( - noOfEntries = noOfEntries, - noOfUnallocatedEntries = noOfUnallocatedEntries, - noOfEntriesAvailableForImmediateUse = noOfEntriesAvailableForImmediateUse, - noOfReleasedEntries = noOfReleasedEntries, - noOfReservedEntries = noOfReservedEntries) - }.fix() - }.unsafeRunSync() + Either = Either.fx { + + val keyValuePairs = mutableMapOf() + + getProfileStatsAsKeyValuePairs(hssId = hssId, simProfile = simProfile).bind() + .forEach { keyValuePairs.put(it.key, it.value) } + + fun lookup(key: String) = keyValuePairs[key] + ?.right() + ?: NotFoundError("Could not find key $key").left() + + val noOfEntries = + lookup("NO_OF_ENTRIES").bind() + val noOfUnallocatedEntries = + lookup("NO_OF_UNALLOCATED_ENTRIES").bind() + val noOfReleasedEntries = + lookup("NO_OF_RELEASED_ENTRIES").bind() + val noOfEntriesAvailableForImmediateUse = + lookup("NO_OF_ENTRIES_READY_FOR_IMMEDIATE_USE").bind() + val noOfReservedEntries = + lookup("NO_OF_RESERVED_ENTRIES").bind() + + SimProfileKeyStatistics( + noOfEntries = noOfEntries, + noOfUnallocatedEntries = noOfUnallocatedEntries, + noOfEntriesAvailableForImmediateUse = noOfEntriesAvailableForImmediateUse, + noOfReleasedEntries = noOfReleasedEntries, + noOfReservedEntries = noOfReservedEntries) + } } From babaa5b32e0c544aad0998eddd4677080707fc62 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 14:38:45 +0100 Subject: [PATCH 44/56] Updated prime-admin with Arrow Fx --- .../prime/admin/actions/CustomerActions.kt | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt b/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt index 1c8635d3a..606e99243 100644 --- a/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt +++ b/tools/prime-admin/src/main/kotlin/org/ostelco/tools/prime/admin/actions/CustomerActions.kt @@ -1,12 +1,10 @@ package org.ostelco.tools.prime.admin.actions import arrow.core.Either -import arrow.core.fix +import arrow.core.extensions.fx import arrow.core.flatMap import arrow.core.left import arrow.core.right -import arrow.effects.IO -import arrow.instances.either.monad.monad import org.apache.commons.codec.digest.DigestUtils import org.ostelco.prime.dsl.withId import org.ostelco.prime.dsl.writeTransaction @@ -33,32 +31,27 @@ fun createCustomer(email: String, nickname: String): Either = analyticsId = UUID.randomUUID().toString(), referralId = UUID.randomUUID().toString())) -fun approveRegionForCustomer(email: String, regionCode: String): Either = IO { - Either.monad().binding { +fun approveRegionForCustomer(email: String, regionCode: String): Either = Either.fx { + val customerId = adminStore + .getCustomer(identity = emailAsIdentity(email = email)) + .bind() + .id + + adminStore.approveRegionForCustomer( + customerId = customerId, + regionCode = regionCode + ).bind() +} + +fun addCustomerToSegment(email: String, segmentId: String): Either = writeTransaction { + Either.fx { val customerId = adminStore .getCustomer(identity = emailAsIdentity(email = email)) .bind() .id - - adminStore.approveRegionForCustomer( - customerId = customerId, - regionCode = regionCode - ).bind() - - }.fix() -}.unsafeRunSync() - -fun addCustomerToSegment(email: String, segmentId: String): Either = writeTransaction { - IO { - Either.monad().binding { - val customerId = adminStore - .getCustomer(identity = emailAsIdentity(email = email)) - .bind() - .id - fact { (Customer withId customerId) belongsToSegment (org.ostelco.prime.storage.graph.model.Segment withId segmentId) }.bind() - }.fix() - }.unsafeRunSync() + fact { (Customer withId customerId) belongsToSegment (org.ostelco.prime.storage.graph.model.Segment withId segmentId) }.bind() + } } fun deleteCustomer(email: String) = adminStore @@ -129,8 +122,8 @@ private fun emailAsIdentity(email: String) = Identity( provider = "email" ) -fun identifyCustomer(idDigests: Set) = IO { - Either.monad().binding { +fun identifyCustomer(idDigests: Set) { + Either.fx { adminStore // get all Identity values .getAllIdentities() @@ -147,9 +140,7 @@ fun identifyCustomer(idDigests: Set) = IO { { println("Found Customer ID: ${it.id} for digest: $idDigest") } ) } - }.fix() + }.mapLeft { + println(it.message) } - .unsafeRunSync() - .mapLeft { - println(it.message) - } \ No newline at end of file +} \ No newline at end of file From db7c233010e1e264c31c42d1c33b47caa832e868 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 13:43:31 +0100 Subject: [PATCH 45/56] Updated Neo4jStore with Arrow Fx --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 1602 ++++++++--------- .../prime/storage/graph/Neo4jStoreTest.kt | 124 +- 2 files changed, 832 insertions(+), 894 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 2ff1ac833..c24857657 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -4,7 +4,6 @@ import arrow.core.Either import arrow.core.Either.Left import arrow.core.Either.Right import arrow.core.EitherOf -import arrow.core.extensions.either.monad.monad import arrow.core.extensions.fx import arrow.core.fix import arrow.core.flatMap @@ -12,10 +11,6 @@ import arrow.core.getOrHandle import arrow.core.left import arrow.core.leftIfNull import arrow.core.right -import arrow.fx.IO -import arrow.fx.extensions.fx -import arrow.fx.extensions.io.monad.monad -import arrow.fx.fix import org.neo4j.driver.v1.Transaction import org.ostelco.prime.analytics.AnalyticsService import org.ostelco.prime.appnotifier.AppNotifier @@ -1240,138 +1235,132 @@ object Neo4jStoreSingleton : GraphStore { sku: String, sourceId: String?, saveCard: Boolean): Either = writeTransaction { - IO { - Either.monad().binding { - val customer = getCustomer(identity = identity) + Either.fx { + + val customer = getCustomer(identity = identity) + .mapLeft { + org.ostelco.prime.paymentprocessor.core.NotFoundError( + "Failed to get customer data for customer with identity - $identity", + internalError = it) + }.bind() + + val product = getProduct(identity, sku) + .mapLeft { + org.ostelco.prime.paymentprocessor.core.NotFoundError("Product $sku is unavailable", + internalError = it) + } + .bind() + + if (product.price.amount > 0) { + val (chargeId, invoiceId) = when (product.paymentType) { + SUBSCRIPTION -> { + val subscriptionDetailsInfo = purchasePlan( + customer = customer, + sku = product.sku, + sourceId = sourceId, + saveCard = saveCard, + taxRegionId = product.paymentTaxRegionId) + .bind() + Pair(subscriptionDetailsInfo.chargeId, subscriptionDetailsInfo.invoiceId) + } + else -> { + val invoicePaymentInfo = oneTimePurchase( + customer = customer, + sourceId = sourceId, + saveCard = saveCard, + sku = product.sku, + price = product.price, + taxRegionId = product.paymentTaxRegionId, + productLabel = product.paymentLabel, + transaction = transaction) + .bind() + Pair(invoicePaymentInfo.chargeId, invoicePaymentInfo.id) + } + } + val purchaseRecord = PurchaseRecord( + id = chargeId, + product = product, + timestamp = Instant.now().toEpochMilli(), + properties = mapOf("invoiceId" to invoiceId)) + + /* If this step fails, the previously added 'removeInvoice' call added to the transaction + will ensure that the invoice will be voided. */ + createPurchaseRecord(customer.id, purchaseRecord) .mapLeft { - org.ostelco.prime.paymentprocessor.core.NotFoundError( - "Failed to get customer data for customer with identity - $identity", + logger.error("Failed to save purchase record for customer ${customer.id}, invoice: $invoiceId, invoice will be voided in Stripe") + AuditLog.error(customerId = customer.id, + message = "Failed to save purchase record - invoice: $invoiceId, invoice will be voided in Stripe") + StorePurchaseError("Failed to save purchase record", internalError = it) }.bind() - val product = getProduct(identity, sku) - .mapLeft { - org.ostelco.prime.paymentprocessor.core.NotFoundError("Product $sku is unavailable", - internalError = it) - } - .bind() + /* Adds purchase to customer history. */ + AuditLog.info(customerId = customer.id, + message = "Purchased product $sku for ${formatMoney(product.price)} (invoice: $invoiceId, charge-id: $chargeId)") - if (product.price.amount > 0) { - val (chargeId, invoiceId) = when (product.paymentType) { - SUBSCRIPTION -> { - val subscriptionDetailsInfo = purchasePlan( - customer = customer, - sku = product.sku, - sourceId = sourceId, - saveCard = saveCard, - taxRegionId = product.paymentTaxRegionId) - .bind() - Pair(subscriptionDetailsInfo.chargeId, subscriptionDetailsInfo.invoiceId) - } - else -> { - val invoicePaymentInfo = oneTimePurchase( - customer = customer, - sourceId = sourceId, - saveCard = saveCard, - sku = product.sku, - price = product.price, - taxRegionId = product.paymentTaxRegionId, - productLabel = product.paymentLabel, - transaction = transaction) - .bind() - Pair(invoicePaymentInfo.chargeId, invoicePaymentInfo.id) - } - } - val purchaseRecord = PurchaseRecord( - id = chargeId, - product = product, - timestamp = Instant.now().toEpochMilli(), - properties = mapOf("invoiceId" to invoiceId)) - - /* If this step fails, the previously added 'removeInvoice' call added to the transaction - will ensure that the invoice will be voided. */ - createPurchaseRecord(customer.id, purchaseRecord) - .mapLeft { - logger.error("Failed to save purchase record for customer ${customer.id}, invoice: $invoiceId, invoice will be voided in Stripe") - AuditLog.error(customerId = customer.id, - message = "Failed to save purchase record - invoice: $invoiceId, invoice will be voided in Stripe") - StorePurchaseError("Failed to save purchase record", - internalError = it) - }.bind() - - /* Adds purchase to customer history. */ - AuditLog.info(customerId = customer.id, - message = "Purchased product $sku for ${formatMoney(product.price)} (invoice: $invoiceId, charge-id: $chargeId)") - - /* TODO: While aborting transactions, send a record with "reverted" status. */ - analyticsReporter.reportPurchase( - customerAnalyticsId = customer.analyticsId, - purchaseId = purchaseRecord.id, - sku = product.sku, - priceAmountCents = product.price.amount, - priceCurrency = product.price.currency) - } + /* TODO: While aborting transactions, send a record with "reverted" status. */ + analyticsReporter.reportPurchase( + customerAnalyticsId = customer.analyticsId, + purchaseId = purchaseRecord.id, + sku = product.sku, + priceAmountCents = product.price.amount, + priceCurrency = product.price.currency) + } - applyProduct( - customerId = customer.id, - product = product - ).mapLeft { - StorePurchaseError(description = it.message, internalError = it.error) - .left() - .bind() - }.bind() + applyProduct( + customerId = customer.id, + product = product + ).mapLeft { + StorePurchaseError(description = it.message, internalError = it.error) + }.bind() - ProductInfo(product.sku) - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + ProductInfo(product.sku) + }.ifFailedThenRollback(transaction) } // << END - fun WriteTransaction.applyProduct(customerId: String, product: Product) = IO { - Either.monad().binding { - when (product.productClass) { - MEMBERSHIP -> { - product.segmentIds.forEach { segmentId -> - assignCustomerToSegment(customerId = customerId, - segmentId = segmentId, - transaction = transaction) - .mapLeft { - SystemError( - type = "Customer -> Segment", - id = "$customerId -> $segmentId", - message = "Failed to assign Membership", - error = it) - } - .bind() - } - } - SIMPLE_DATA -> { - /* Topup. */ - simpleDataProduct( - customerId = customerId, - sku = product.sku, - bytes = product.noOfBytes, + fun WriteTransaction.applyProduct(customerId: String, product: Product): Either = Either.fx { + when (product.productClass) { + MEMBERSHIP -> { + product.segmentIds.forEach { segmentId -> + assignCustomerToSegment(customerId = customerId, + segmentId = segmentId, transaction = transaction) .mapLeft { SystemError( - type = "Customer", - id = product.sku, - message = "Failed to update balance for customer: $customerId", + type = "Customer -> Segment", + id = "$customerId -> $segmentId", + message = "Failed to assign Membership", error = it) } .bind() } - else -> { - SystemError( - type = "Product", - id = product.sku, - message = "Missing product class in properties of product: ${product.sku}").left().bind() - } } - }.fix() - }.unsafeRunSync() + SIMPLE_DATA -> { + /* Topup. */ + simpleDataProduct( + customerId = customerId, + sku = product.sku, + bytes = product.noOfBytes, + transaction = transaction) + .mapLeft { + SystemError( + type = "Customer", + id = product.sku, + message = "Failed to update balance for customer: $customerId", + error = it) + } + .bind() + } + else -> { + SystemError( + type = "Product", + id = product.sku, + message = "Missing product class in properties of product: ${product.sku}").left().bind() + } + } + } private fun fetchOrCreatePaymentProfile(customer: Customer): Either = // Fetch/Create stripe payment profile for the customer. @@ -1391,55 +1380,61 @@ object Neo4jStoreSingleton : GraphStore { taxRegionId: String?, sourceId: String?, saveCard: Boolean): Either { - return IO { - Either.monad().binding { - - /* Bail out if subscriber tries to buy an already bought plan. - Note: Already verified above that 'customer' (subscriber) exists. */ - get(PurchaseRecord forPurchaseBy (Customer withId customer.id)) - .map { purchaseRecords -> - if (purchaseRecords.any { x:PurchaseRecord -> x.product.sku == sku }) { - PlanAlredyPurchasedError("A subscription to plan $sku already exists") - .left().bind() - } - } + return Either.fx { - /* A source must be associated with a payment profile with the payment vendor. - Create the profile if it don't exists. */ - fetchOrCreatePaymentProfile(customer) - .bind() + /* Bail out if subscriber tries to buy an already bought plan. + Note: Already verified above that 'customer' (subscriber) exists. */ + get(PurchaseRecord forPurchaseBy (Customer withId customer.id)) + .mapLeft { + StorePurchaseError( + description = "Failed to fetch Purchase Records for Customer", + message = it.message, + internalError = it) + } + .flatMap { purchaseRecords -> + if (purchaseRecords.any { x:PurchaseRecord -> x.product.sku == sku }) { + PlanAlredyPurchasedError("A subscription to plan $sku already exists") + .left() + } else { + Unit.right() + } + }.bind() - /* With recurring payments, the payment card (source) must be stored. The - 'saveCard' parameter is therefore ignored. */ - if (!saveCard) { - logger.warn("Ignoring request for deleting payment source after buying plan $sku for " + - "customer ${customer.id} as stored payment source is required when purchasing a plan") - } + /* A source must be associated with a payment profile with the payment vendor. + Create the profile if it don't exists. */ + fetchOrCreatePaymentProfile(customer) + .bind() - if (sourceId != null) { - val sourceDetails = paymentProcessor.getSavedSources(customer.id) - .mapLeft { - org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for customer: ${customer.id}", - internalError = it) - }.bind() - if (!sourceDetails.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { - paymentProcessor.addSource(customer.id, sourceId) - .bind().id - } - } + /* With recurring payments, the payment card (source) must be stored. The + 'saveCard' parameter is therefore ignored. */ + if (!saveCard) { + logger.warn("Ignoring request for deleting payment source after buying plan $sku for " + + "customer ${customer.id} as stored payment source is required when purchasing a plan") + } - subscribeToPlan( - customerId = customer.id, - planId = sku, - taxRegionId = taxRegionId) + if (sourceId != null) { + val sourceDetails = paymentProcessor.getSavedSources(customer.id) .mapLeft { - AuditLog.error(customerId = customer.id, message = "Failed to subscribe to plan $sku") - SubscriptionError("Failed to subscribe ${customer.id} to plan $sku", + org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for customer: ${customer.id}", internalError = it) - } - .bind() - }.fix() - }.unsafeRunSync() + }.bind() + if (!sourceDetails.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { + paymentProcessor.addSource(customer.id, sourceId) + .bind().id + } + } + + subscribeToPlan( + customerId = customer.id, + planId = sku, + taxRegionId = taxRegionId) + .mapLeft { + AuditLog.error(customerId = customer.id, message = "Failed to subscribe to plan $sku") + SubscriptionError("Failed to subscribe ${customer.id} to plan $sku", + internalError = it) + } + .bind() + } } private fun WriteTransaction.subscribeToPlan( @@ -1448,86 +1443,84 @@ object Neo4jStoreSingleton : GraphStore { taxRegionId: String?, trialEnd: Long = 0L): Either { - return IO { - Either.monad().binding { - val plan = get(Plan withId planId) - .bind() - val profileInfo = paymentProcessor.getPaymentProfile(customerId) - .mapLeft { - NotFoundError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", - error = it) - }.bind() + return Either.fx { + val plan = get(Plan withId planId) + .bind() + val profileInfo = paymentProcessor.getPaymentProfile(customerId) + .mapLeft { + NotFoundError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", + error = it) + }.bind() - /* At this point, we have either: - 1) A new subscription to a plan is being created. - 2) An attempt at buying a previously subscribed to plan but which has not been - paid for. - Both are OK. But in order to handle the second case correctly, the previous incomplete - subscription must be removed before we can proceed with creating the new subscription. + /* At this point, we have either: + 1) A new subscription to a plan is being created. + 2) An attempt at buying a previously subscribed to plan but which has not been + paid for. + Both are OK. But in order to handle the second case correctly, the previous incomplete + subscription must be removed before we can proceed with creating the new subscription. - (In the second case there will be a "SUBSCRIBES_TO_PLAN" link between the customer - object and the plan object, but no "PURCHASED" link to the plans "product" object.) + (In the second case there will be a "SUBSCRIBES_TO_PLAN" link between the customer + object and the plan object, but no "PURCHASED" link to the plans "product" object.) - The motivation for supporting the second case, is that it allows the subscriber to - reattempt to buy a plan using a different payment source. + The motivation for supporting the second case, is that it allows the subscriber to + reattempt to buy a plan using a different payment source. - Remove existing incomplete subscription if any. */ - get(Plan forCustomer (Customer withId customerId)) - .map { - if (it.any { x -> x.id == planId }) { - removeSubscription(customerId, planId, invoiceNow = true) - } + Remove existing incomplete subscription if any. */ + get(Plan forCustomer (Customer withId customerId)) + .map { + if (it.any { x -> x.id == planId }) { + removeSubscription(customerId, planId, invoiceNow = true) } + } - /* Lookup in payment backend will fail if no value found for 'stripePlanId'. */ - val planStripeId = plan.stripePlanId ?: SystemError(type = planEntity.name, id = plan.id, - message = "No reference to Stripe plan found in ${plan.id}") - .left() - .bind() + /* Lookup in payment backend will fail if no value found for 'stripePlanId'. */ + val planStripeId = plan.stripePlanId ?: SystemError(type = planEntity.name, id = plan.id, + message = "No reference to Stripe plan found in ${plan.id}") + .left() + .bind() - val subscriptionDetailsInfo = paymentProcessor.createSubscription( - planId = planStripeId, - customerId = profileInfo.id, - trialEnd = trialEnd, - taxRegionId = taxRegionId) - .mapLeft { - NotCreatedError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", - error = it) - }.linkReversalActionToTransaction(transaction) { - paymentProcessor.cancelSubscription(it.id) - }.bind() + val subscriptionDetailsInfo = paymentProcessor.createSubscription( + planId = planStripeId, + customerId = profileInfo.id, + trialEnd = trialEnd, + taxRegionId = taxRegionId) + .mapLeft { + NotCreatedError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", + error = it) + }.linkReversalActionToTransaction(transaction) { + paymentProcessor.cancelSubscription(it.id) + }.bind() - /* Dispatch according to the charge result. */ - when (subscriptionDetailsInfo.status) { - PaymentStatus.PAYMENT_SUCCEEDED -> { - } - PaymentStatus.REQUIRES_PAYMENT_METHOD -> { - NotCreatedError(type = "Customer subscription to Plan", - id = "$customerId -> ${plan.id}", - error = InvalidRequestError("Payment method failed") - ).left().bind() - } - PaymentStatus.REQUIRES_ACTION, - PaymentStatus.TRIAL_START -> { - /* No action required. Charge for the subscription will eventually - be reported as a Stripe event. */ - logger.info( - "Pending payment for subscription $planId for customer $customerId (${subscriptionDetailsInfo.status.name})") - } + /* Dispatch according to the charge result. */ + when (subscriptionDetailsInfo.status) { + PaymentStatus.PAYMENT_SUCCEEDED -> { } + PaymentStatus.REQUIRES_PAYMENT_METHOD -> { + NotCreatedError(type = "Customer subscription to Plan", + id = "$customerId -> ${plan.id}", + error = InvalidRequestError("Payment method failed") + ).left().bind() + } + PaymentStatus.REQUIRES_ACTION, + PaymentStatus.TRIAL_START -> { + /* No action required. Charge for the subscription will eventually + be reported as a Stripe event. */ + logger.info( + "Pending payment for subscription $planId for customer $customerId (${subscriptionDetailsInfo.status.name})") + } + } - /* Store information from payment backend for later use. */ - fact { - (Customer withId customerId) subscribesTo (Plan withId planId) using - PlanSubscription( - subscriptionId = subscriptionDetailsInfo.id, - created = subscriptionDetailsInfo.created, - trialEnd = subscriptionDetailsInfo.trialEnd) - }.bind() + /* Store information from payment backend for later use. */ + fact { + (Customer withId customerId) subscribesTo (Plan withId planId) using + PlanSubscription( + subscriptionId = subscriptionDetailsInfo.id, + created = subscriptionDetailsInfo.created, + trialEnd = subscriptionDetailsInfo.trialEnd) + }.bind() - subscriptionDetailsInfo - }.fix() - }.unsafeRunSync() + subscriptionDetailsInfo + } } private fun oneTimePurchase( @@ -1538,75 +1531,72 @@ object Neo4jStoreSingleton : GraphStore { price: Price, productLabel: String, taxRegionId: String?, - transaction: PrimeTransaction): Either = IO { + transaction: PrimeTransaction): Either = Either.fx { - Either.monad().binding { + /* A source must be associated with a payment profile with the payment vendor. + Create the profile if it don't exists. */ + fetchOrCreatePaymentProfile(customer) + .bind() - /* A source must be associated with a payment profile with the payment vendor. - Create the profile if it don't exists. */ - fetchOrCreatePaymentProfile(customer) - .bind() + var addedSourceId: String? = null - var addedSourceId: String? = null - - /* Add source if set and if it has not already been added to the payment profile. */ - if (sourceId != null) { - val sourceDetails = paymentProcessor.getSavedSources(customer.id) - .mapLeft { - org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for user", - internalError = it) - }.bind() - addedSourceId = sourceId - - if (!sourceDetails.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { - addedSourceId = paymentProcessor.addSource(customer.id, sourceId) - /* For the success case, saved source is removed after the invoice has been - paid if 'saveCard == false'. Make sure same happens even for failure - case by linking reversal action to transaction */ - .finallyDo(transaction) { - removePaymentSource(saveCard, customer.id, it.id) - }.bind().id - } - } - - val invoice = paymentProcessor.createInvoice( - customerId = customer.id, - amount = price.amount, - currency = price.currency, - description = productLabel, - taxRegionId = taxRegionId, - sourceId = addedSourceId) + /* Add source if set and if it has not already been added to the payment profile. */ + if (sourceId != null) { + val sourceDetails = paymentProcessor.getSavedSources(customer.id) .mapLeft { - logger.error("Failed to create invoice for customer ${customer.id}, source $addedSourceId, sku $sku") - it - }.linkReversalActionToTransaction(transaction) { - paymentProcessor.removeInvoice(it.id) - logger.warn(NOTIFY_OPS_MARKER, """ - Failed to pay invoice for customer ${customer.id}, invoice-id: ${it.id}. - Verify that the invoice has been deleted or voided in Stripe dashboard. - """.trimIndent()) + org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for user", + internalError = it) }.bind() + addedSourceId = sourceId + + if (!sourceDetails.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { + addedSourceId = paymentProcessor.addSource(customer.id, sourceId) + /* For the success case, saved source is removed after the invoice has been + paid if 'saveCard == false'. Make sure same happens even for failure + case by linking reversal action to transaction */ + .finallyDo(transaction) { + removePaymentSource(saveCard, customer.id, it.id) + }.bind().id + } + } - /* Force immediate payment of the invoice. */ - val invoicePaymentInfo = paymentProcessor.payInvoice(invoice.id) - .mapLeft { - logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") - /* Adds failed purchase to customer history. */ - AuditLog.warn(customerId = customer.id, - message = "Failed to complete purchase of product $sku for ${formatMoney(price)} " + - "status: ${it.code} decline reason: ${it.declineCode}") - it - }.linkReversalActionToTransaction(transaction) { - paymentProcessor.refundCharge(it.chargeId) - logger.warn(NOTIFY_OPS_MARKER, """ - Refunded customer ${customer.id} for invoice: ${it.id}. - Verify that the invoice has been refunded in Stripe dashboard. - """.trimIndent()) - }.bind() + val invoice = paymentProcessor.createInvoice( + customerId = customer.id, + amount = price.amount, + currency = price.currency, + description = productLabel, + taxRegionId = taxRegionId, + sourceId = addedSourceId) + .mapLeft { + logger.error("Failed to create invoice for customer ${customer.id}, source $addedSourceId, sku $sku") + it + }.linkReversalActionToTransaction(transaction) { + paymentProcessor.removeInvoice(it.id) + logger.warn(NOTIFY_OPS_MARKER, """ + Failed to pay invoice for customer ${customer.id}, invoice-id: ${it.id}. + Verify that the invoice has been deleted or voided in Stripe dashboard. + """.trimIndent()) + }.bind() - invoicePaymentInfo - }.fix() - }.unsafeRunSync() + /* Force immediate payment of the invoice. */ + val invoicePaymentInfo = paymentProcessor.payInvoice(invoice.id) + .mapLeft { + logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") + /* Adds failed purchase to customer history. */ + AuditLog.warn(customerId = customer.id, + message = "Failed to complete purchase of product $sku for ${formatMoney(price)} " + + "status: ${it.code} decline reason: ${it.declineCode}") + it + }.linkReversalActionToTransaction(transaction) { + paymentProcessor.refundCharge(it.chargeId) + logger.warn(NOTIFY_OPS_MARKER, """ + Refunded customer ${customer.id} for invoice: ${it.id}. + Verify that the invoice has been refunded in Stripe dashboard. + """.trimIndent()) + }.bind() + + invoicePaymentInfo + } private fun removePaymentSource(saveCard: Boolean, paymentCustomerId: String, sourceId: String) { // In case we fail to remove saved source, we log it at error level. @@ -1624,33 +1614,30 @@ object Neo4jStoreSingleton : GraphStore { customerId: String, sku: String, bytes: Long, - transaction: PrimeTransaction): Either = IO { + transaction: PrimeTransaction): Either = Either.fx { - Either.monad().binding { - - if (bytes == 0L) { - logger.error("Product with 0 bytes: sku = {}", sku) - } else { - /* Update balance with bought data. */ - /* TODO: Add rollback in case of errors later on. */ - write("""MATCH (cr:${customerEntity.name} { id:'$customerId' })-[:${customerToBundleRelation.name}]->(bundle:${bundleEntity.name}) - SET bundle.balance = toString(toInteger(bundle.balance) + $bytes) - """.trimIndent(), transaction) { - Either.cond( - test = it.summary().counters().containsUpdates(), - ifTrue = {}, - ifFalse = { - logger.error("Failed to update balance for customer: {}", customerId) - NotUpdatedError( - type = "Balance of Customer", - id = customerId) - }) - }.bind() - AuditLog.info(customerId = customerId, message = "Added $bytes bytes to data bundle") - } - Unit - }.fix() - }.unsafeRunSync() + if (bytes == 0L) { + logger.error("Product with 0 bytes: sku = {}", sku) + } else { + /* Update balance with bought data. */ + /* TODO: Add rollback in case of errors later on. */ + write("""MATCH (cr:${customerEntity.name} { id:'$customerId' })-[:${customerToBundleRelation.name}]->(bundle:${bundleEntity.name}) + SET bundle.balance = toString(toInteger(bundle.balance) + $bytes) + """.trimIndent(), transaction) { + Either.cond( + test = it.summary().counters().containsUpdates(), + ifTrue = {}, + ifFalse = { + logger.error("Failed to update balance for customer: {}", customerId) + NotUpdatedError( + type = "Balance of Customer", + id = customerId) + }) + }.bind() + AuditLog.info(customerId = customerId, message = "Added $bytes bytes to data bundle") + } + Unit + } // // Purchase Records @@ -1675,25 +1662,23 @@ object Neo4jStoreSingleton : GraphStore { val invoiceId = purchaseRecord.properties["invoiceId"] - return IO { - Either.monad().binding { + return Either.fx { - if (invoiceId != null - && getPurchaseRecordUsingInvoiceId(customerId, invoiceId).isRight()) { - /* Avoid charging for the same invoice twice if invoice information is present. */ + if (invoiceId != null + && getPurchaseRecordUsingInvoiceId(customerId, invoiceId).isRight()) { + /* Avoid charging for the same invoice twice if invoice information is present. */ - ValidationError(type = purchaseRecordEntity.name, - id = purchaseRecord.id, - message = "A purchase record for ${purchaseRecord.product} for customer $customerId already exists") - .left() - .bind() - } - create { purchaseRecord }.bind() - fact { (PurchaseRecord withId purchaseRecord.id) forPurchaseBy (Customer withId customerId) }.bind() - fact { (PurchaseRecord withId purchaseRecord.id) forPurchaseOf (Product withSku purchaseRecord.product.sku) }.bind() - purchaseRecord.id - }.fix() - }.unsafeRunSync() + ValidationError(type = purchaseRecordEntity.name, + id = purchaseRecord.id, + message = "A purchase record for ${purchaseRecord.product} for customer $customerId already exists") + .left() + .bind() + } + create { purchaseRecord }.bind() + fact { (PurchaseRecord withId purchaseRecord.id) forPurchaseBy (Customer withId customerId) }.bind() + fact { (PurchaseRecord withId purchaseRecord.id) forPurchaseOf (Product withSku purchaseRecord.product.sku) }.bind() + purchaseRecord.id + } } /* As Stripes invoice-id is used as the 'id' of a purchase record, this method @@ -1948,63 +1933,61 @@ object Neo4jStoreSingleton : GraphStore { identity: ModelIdentity, version: MyInfoApiVersion, authorisationCode: String): Either { - return IO { - Either.monad().binding { - - val customer = getCustomer(identity = identity).bind() + return Either.fx { - // set MY_INFO KYC Status to Pending - setKycStatus( - customer = customer, - regionCode = "sg", - kycType = MY_INFO, - kycStatus = KycStatus.PENDING).bind() - - val myInfoData = try { - when (version) { - V3 -> myInfoKycV3Service - }.getPersonData(authorisationCode) - } catch (e: Exception) { - logger.error("Failed to fetched MyInfo $version using authCode = $authorisationCode", e) - null - } ?: SystemError( - type = "MyInfo Auth Code", - id = authorisationCode, - message = "Failed to fetched MyInfo $version").left().bind() - - val personData = myInfoData.personData ?: SystemError( - type = "MyInfo Auth Code", - id = authorisationCode, - message = "Failed to fetched MyInfo $version").left().bind() - - val kycStatus = if(myInfoData.birthDate.isLessThan18yrsAgo()) { - AuditLog.warn(customerId = customer.id, message = "Customer age is less than 18yrs.") - REJECTED - } else { - // store data only if approved - secureArchiveService.archiveEncrypted( - customerId = customer.id, - fileName = "myInfoData", - regionCodes = listOf("sg"), - dataMap = mapOf( - "uinFin" to myInfoData.uinFin.toByteArray(), - "personData" to personData.toByteArray() - ) - ).bind() - KycStatus.APPROVED - } + val customer = getCustomer(identity = identity).bind() + + // set MY_INFO KYC Status to Pending + setKycStatus( + customer = customer, + regionCode = "sg", + kycType = MY_INFO, + kycStatus = KycStatus.PENDING).bind() + + val myInfoData = try { + when (version) { + V3 -> myInfoKycV3Service + }.getPersonData(authorisationCode) + } catch (e: Exception) { + logger.error("Failed to fetched MyInfo $version using authCode = $authorisationCode", e) + null + } ?: SystemError( + type = "MyInfo Auth Code", + id = authorisationCode, + message = "Failed to fetched MyInfo $version").left().bind() + + val personData = myInfoData.personData ?: SystemError( + type = "MyInfo Auth Code", + id = authorisationCode, + message = "Failed to fetched MyInfo $version").left().bind() + + val kycStatus = if(myInfoData.birthDate.isLessThan18yrsAgo()) { + AuditLog.warn(customerId = customer.id, message = "Customer age is less than 18yrs.") + REJECTED + } else { + // store data only if approved + secureArchiveService.archiveEncrypted( + customerId = customer.id, + fileName = "myInfoData", + regionCodes = listOf("sg"), + dataMap = mapOf( + "uinFin" to myInfoData.uinFin.toByteArray(), + "personData" to personData.toByteArray() + ) + ).bind() + KycStatus.APPROVED + } - // set MY_INFO KYC Status to Approved - setKycStatus( - customer = customer, - regionCode = "sg", - kycType = MY_INFO, - kycStatus = kycStatus, - kycExpiryDate = myInfoData.passExpiryDate?.toString()).bind() + // set MY_INFO KYC Status to Approved + setKycStatus( + customer = customer, + regionCode = "sg", + kycType = MY_INFO, + kycStatus = kycStatus, + kycExpiryDate = myInfoData.passExpiryDate?.toString()).bind() - personData - }.fix() - }.unsafeRunSync() + personData + } } // @@ -2017,41 +2000,39 @@ object Neo4jStoreSingleton : GraphStore { identity: ModelIdentity, nricFinId: String): Either { - return IO { - Either.monad().binding { + return Either.fx { - logger.info("checkNricFinIdUsingDave for $nricFinId") + logger.info("checkNricFinIdUsingDave for $nricFinId") - val customer = getCustomer(identity = identity).bind() + val customer = getCustomer(identity = identity).bind() - // set NRIC_FIN KYC Status to Pending - setKycStatus( - customer = customer, - regionCode = "sg", - kycType = NRIC_FIN, - kycStatus = KycStatus.PENDING).bind() + // set NRIC_FIN KYC Status to Pending + setKycStatus( + customer = customer, + regionCode = "sg", + kycType = NRIC_FIN, + kycStatus = KycStatus.PENDING).bind() - if (daveKycService.validate(nricFinId)) { - logger.info("checkNricFinIdUsingDave validated $nricFinId") - } else { - logger.info("checkNricFinIdUsingDave failed to validate $nricFinId") - ValidationError(type = "NRIC/FIN ID", id = nricFinId, message = "Invalid NRIC/FIN ID").left().bind() - } - - secureArchiveService.archiveEncrypted( - customerId = customer.id, - fileName = "nricFin", - regionCodes = listOf("sg"), - dataMap = mapOf("nricFinId" to nricFinId.toByteArray()) - ).bind() + if (daveKycService.validate(nricFinId)) { + logger.info("checkNricFinIdUsingDave validated $nricFinId") + } else { + logger.info("checkNricFinIdUsingDave failed to validate $nricFinId") + ValidationError(type = "NRIC/FIN ID", id = nricFinId, message = "Invalid NRIC/FIN ID").left().bind() + } - // set NRIC_FIN KYC Status to Approved - setKycStatus( - customer = customer, - regionCode = "sg", - kycType = NRIC_FIN).bind() - }.fix() - }.unsafeRunSync() + secureArchiveService.archiveEncrypted( + customerId = customer.id, + fileName = "nricFin", + regionCodes = listOf("sg"), + dataMap = mapOf("nricFinId" to nricFinId.toByteArray()) + ).bind() + + // set NRIC_FIN KYC Status to Approved + setKycStatus( + customer = customer, + regionCode = "sg", + kycType = NRIC_FIN).bind() + } } // @@ -2062,33 +2043,31 @@ object Neo4jStoreSingleton : GraphStore { address: String, regionCode: String): Either { - return IO { - Either.monad().binding { + return Either.fx { - val customer = getCustomer(identity = identity).bind() + val customer = getCustomer(identity = identity).bind() - // set ADDRESS KYC Status to Pending - setKycStatus( - customer = customer, - regionCode = regionCode, - kycType = ADDRESS, - kycStatus = KycStatus.PENDING).bind() + // set ADDRESS KYC Status to Pending + setKycStatus( + customer = customer, + regionCode = regionCode, + kycType = ADDRESS, + kycStatus = KycStatus.PENDING).bind() - secureArchiveService.archiveEncrypted( - customerId = customer.id, - fileName = "address", - regionCodes = listOf(regionCode), - dataMap = mapOf( - "address" to address.toByteArray()) - ).bind() - - // set ADDRESS KYC Status to Approved - setKycStatus( - customer = customer, - regionCode = regionCode, - kycType = ADDRESS).bind() - }.fix() - }.unsafeRunSync() + secureArchiveService.archiveEncrypted( + customerId = customer.id, + fileName = "address", + regionCodes = listOf(regionCode), + dataMap = mapOf( + "address" to address.toByteArray()) + ).bind() + + // set ADDRESS KYC Status to Approved + setKycStatus( + customer = customer, + regionCode = regionCode, + kycType = ADDRESS).bind() + } } // @@ -2123,118 +2102,116 @@ object Neo4jStoreSingleton : GraphStore { kycIdType: String? = null, transaction: Transaction): Either { - return IO { - Either.monad().binding { + return Either.fx { - // get combinations of KYC needed for this region to be Approved - val approvedKycTypeSetList = getApprovedKycTypeSetList(regionCode) + // get combinations of KYC needed for this region to be Approved + val approvedKycTypeSetList = getApprovedKycTypeSetList(regionCode) - // fetch existing values from DB - val existingCustomerRegion = customerRegionRelationStore.get( - fromId = customer.id, - toId = regionCode, - transaction = transaction) - .flatMapLeft { storeError -> - if(storeError is NotFoundError && storeError.type == customerRegionRelation.name) { - // default value if absent in DB - CustomerRegion( - status = PENDING, - kycStatusMap = getKycStatusMapForRegion(regionCode), - initiatedOn = utcTimeNow() - ).right() - } else { - storeError.left() - } - }.bind() + // fetch existing values from DB + val existingCustomerRegion = customerRegionRelationStore.get( + fromId = customer.id, + toId = regionCode, + transaction = transaction) + .flatMapLeft { storeError -> + if(storeError is NotFoundError && storeError.type == customerRegionRelation.name) { + // default value if absent in DB + CustomerRegion( + status = PENDING, + kycStatusMap = getKycStatusMapForRegion(regionCode), + initiatedOn = utcTimeNow() + ).right() + } else { + storeError.left() + } + }.bind() - // using existing and received KYC status, compute new KYC status - val existingKycStatusMap = existingCustomerRegion.kycStatusMap - val existingKycStatus = existingKycStatusMap[kycType] - val newKycStatus = when (existingKycStatus) { - // APPROVED is end state. No more state change. - KycStatus.APPROVED -> KycStatus.APPROVED - // REJECTED and PENDING to 'any' is allowed - else -> kycStatus - } + // using existing and received KYC status, compute new KYC status + val existingKycStatusMap = existingCustomerRegion.kycStatusMap + val existingKycStatus = existingKycStatusMap[kycType] + val newKycStatus = when (existingKycStatus) { + // APPROVED is end state. No more state change. + KycStatus.APPROVED -> KycStatus.APPROVED + // REJECTED and PENDING to 'any' is allowed + else -> kycStatus + } - // if new status is different from existing status - if (existingKycStatus != newKycStatus) { - if (kycStatus == newKycStatus) { - AuditLog.info(customerId = customer.id, message = "Setting $kycType status from $existingKycStatus to $newKycStatus") - } else { - AuditLog.info(customerId = customer.id, message = "Setting $kycType status from $existingKycStatus to $newKycStatus instead of $kycStatus") - } - if (newKycStatus == KycStatus.APPROVED) { - onKycApprovedAction.apply( - customer = customer, - regionCode = regionCode, - kycType = kycType, - kycExpiryDate = kycExpiryDate, - kycIdType = kycIdType, - allowedRegionsService = allowedRegionsService, - transaction = PrimeTransaction(transaction) - ).bind() - } + // if new status is different from existing status + if (existingKycStatus != newKycStatus) { + if (kycStatus == newKycStatus) { + AuditLog.info(customerId = customer.id, message = "Setting $kycType status from $existingKycStatus to $newKycStatus") } else { - AuditLog.info(customerId = customer.id, message = "Ignoring setting $kycType status to $kycStatus since it is already $existingKycStatus") + AuditLog.info(customerId = customer.id, message = "Setting $kycType status from $existingKycStatus to $newKycStatus instead of $kycStatus") + } + if (newKycStatus == KycStatus.APPROVED) { + onKycApprovedAction.apply( + customer = customer, + regionCode = regionCode, + kycType = kycType, + kycExpiryDate = kycExpiryDate, + kycIdType = kycIdType, + allowedRegionsService = allowedRegionsService, + transaction = PrimeTransaction(transaction) + ).bind() } + } else { + AuditLog.info(customerId = customer.id, message = "Ignoring setting $kycType status to $kycStatus since it is already $existingKycStatus") + } - // update KYC status map with new value. This map will then be stored in DB. - val newKycStatusMap = existingKycStatusMap.copy(key = kycType, value = newKycStatus) + // update KYC status map with new value. This map will then be stored in DB. + val newKycStatusMap = existingKycStatusMap.copy(key = kycType, value = newKycStatus) - // check if Region is Approved. - val isRegionApproved = approvedKycTypeSetList.any { kycTypeSet -> - // Region is approved if the set of Approved KYCs is a superset of any one of the set configured in the list - approvedKycTypeSetList. - newKycStatusMap.filter { it.value == KycStatus.APPROVED }.keys.containsAll(kycTypeSet) - } + // check if Region is Approved. + val isRegionApproved = approvedKycTypeSetList.any { kycTypeSet -> + // Region is approved if the set of Approved KYCs is a superset of any one of the set configured in the list - approvedKycTypeSetList. + newKycStatusMap.filter { it.value == KycStatus.APPROVED }.keys.containsAll(kycTypeSet) + } - // if the Region status is Approved, but the existing status was not Approved, then it has been approved now. - val isRegionApprovedNow = existingCustomerRegion.status != APPROVED && isRegionApproved + // if the Region status is Approved, but the existing status was not Approved, then it has been approved now. + val isRegionApprovedNow = existingCustomerRegion.status != APPROVED && isRegionApproved - // Save Region status as APPROVED, if it is approved. Do not change Region status otherwise. - val newRegionStatus = if (isRegionApproved) { - APPROVED - } else { - existingCustomerRegion.status - } + // Save Region status as APPROVED, if it is approved. Do not change Region status otherwise. + val newRegionStatus = if (isRegionApproved) { + APPROVED + } else { + existingCustomerRegion.status + } - // timestamp for region approval - val regionApprovedOn = if (isRegionApprovedNow) { + // timestamp for region approval + val regionApprovedOn = if (isRegionApprovedNow) { - AuditLog.info(customerId = customer.id, message = "Approved for region - $regionCode") + AuditLog.info(customerId = customer.id, message = "Approved for region - $regionCode") - onRegionApprovedAction.apply( - customer = customer, - regionCode = regionCode, - transaction = PrimeTransaction(transaction) - ).bind() + onRegionApprovedAction.apply( + customer = customer, + regionCode = regionCode, + transaction = PrimeTransaction(transaction) + ).bind() - utcTimeNow() - } else { - existingCustomerRegion.approvedOn - } + utcTimeNow() + } else { + existingCustomerRegion.approvedOn + } - // Save KYC expiry date if it is not null. - val newKycExpiryDateMap = kycExpiryDate - ?.let { existingCustomerRegion.kycExpiryDateMap.copy(key = kycType, value = it) } - ?: existingCustomerRegion.kycExpiryDateMap - - customerRegionRelationStore - .createOrUpdate( - fromId = customer.id, - relation = CustomerRegion( - status = newRegionStatus, - kycStatusMap = newKycStatusMap, - kycExpiryDateMap = newKycExpiryDateMap, - initiatedOn = existingCustomerRegion.initiatedOn, - approvedOn = regionApprovedOn - ), - toId = regionCode, - transaction = transaction) - .bind() + // Save KYC expiry date if it is not null. + val newKycExpiryDateMap = kycExpiryDate + ?.let { existingCustomerRegion.kycExpiryDateMap.copy(key = kycType, value = it) } + ?: existingCustomerRegion.kycExpiryDateMap + + customerRegionRelationStore + .createOrUpdate( + fromId = customer.id, + relation = CustomerRegion( + status = newRegionStatus, + kycStatusMap = newKycStatusMap, + kycExpiryDateMap = newKycExpiryDateMap, + initiatedOn = existingCustomerRegion.initiatedOn, + approvedOn = regionApprovedOn + ), + toId = regionCode, + transaction = transaction) + .bind() - }.fix() - }.unsafeRunSync() + } } private fun getKycStatusMapForRegion(regionCode: String): Map { @@ -2423,101 +2400,93 @@ object Neo4jStoreSingleton : GraphStore { plan: Plan, stripeProductName: String, planProduct: Product): Either = writeTransaction { - IO { - Either.monad().binding { + Either.fx { - get(Product withSku plan.id) - .map { - AlreadyExistsError(type = productEntity.name, id = "Failed to find product associated with plan ${plan.id}") - .left() - .bind() - } + if (get(Product withSku plan.id).isRight()) { + AlreadyExistsError(type = productEntity.name, id = "Failed to find product associated with plan ${plan.id}") + .left() + .bind() + } - get(Plan withId plan.id) - .map { - AlreadyExistsError(type = planEntity.name, id = "Failed to find plan ${plan.id}") - .left() - .bind() - } + if (get(Plan withId plan.id).isRight()) { + AlreadyExistsError(type = planEntity.name, id = "Failed to find plan ${plan.id}") + .left() + .bind() + } - val productInfo = paymentProcessor.createProduct(stripeProductName) - .mapLeft { - NotCreatedError(type = planEntity.name, id = "Failed to create plan ${plan.id}", - error = it) - }.linkReversalActionToTransaction(transaction) { - paymentProcessor.removeProduct(it.id) - }.bind() - val planInfo = paymentProcessor.createPlan( - productInfo.id, - planProduct.price.amount, - planProduct.price.currency, - PaymentProcessor.Interval.valueOf(plan.interval.toUpperCase()), plan.intervalCount) - .mapLeft { - NotCreatedError(type = planEntity.name, id = "Failed to create plan ${plan.id}", - error = it) - }.linkReversalActionToTransaction(transaction) { - paymentProcessor.removePlan(it.id) - }.bind() + val productInfo = paymentProcessor.createProduct(stripeProductName) + .mapLeft { + NotCreatedError(type = planEntity.name, id = "Failed to create plan ${plan.id}", + error = it) + }.linkReversalActionToTransaction(transaction) { + paymentProcessor.removeProduct(it.id) + }.bind() + val planInfo = paymentProcessor.createPlan( + productInfo.id, + planProduct.price.amount, + planProduct.price.currency, + PaymentProcessor.Interval.valueOf(plan.interval.toUpperCase()), plan.intervalCount) + .mapLeft { + NotCreatedError(type = planEntity.name, id = "Failed to create plan ${plan.id}", + error = it) + }.linkReversalActionToTransaction(transaction) { + paymentProcessor.removePlan(it.id) + }.bind() - /* The associated product to the plan. Note that: - sku - name of the plan - property value 'productClass' is set to "plan" - TODO: Update to new backend model. */ - val product = planProduct.copy( - payment = planProduct.payment + mapOf( - "type" to SUBSCRIPTION.name) - ) + /* The associated product to the plan. Note that: + sku - name of the plan + property value 'productClass' is set to "plan" + TODO: Update to new backend model. */ + val product = planProduct.copy( + payment = planProduct.payment + mapOf( + "type" to SUBSCRIPTION.name) + ) - /* Propagates errors from lower layer if any. */ - create { product }.bind() - create { - plan.copy( - stripePlanId = planInfo.id, - stripeProductId = productInfo.id) - }.bind() - get(Plan withId plan.id) - .bind() - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + /* Propagates errors from lower layer if any. */ + create { product }.bind() + create { + plan.copy( + stripePlanId = planInfo.id, + stripeProductId = productInfo.id) + }.bind() + get(Plan withId plan.id) + .bind() + }.ifFailedThenRollback(transaction) } override fun deletePlan(planId: String): Either = writeTransaction { - IO { - Either.monad().binding { - val plan = get(Plan withId planId) - .bind() - /* The name of the product is the same as the name of the corresponding plan. */ - get(Product withSku planId) - .bind() + Either.fx { + val plan = get(Plan withId planId) + .bind() + /* The name of the product is the same as the name of the corresponding plan. */ + get(Product withSku planId) + .bind() - /* Not removing the product due to purchase references. */ + /* Not removing the product due to purchase references. */ - /* Removing the plan will remove the plan itself and all relations going to it. */ - delete(Plan withId plan.id) - .bind() + /* Removing the plan will remove the plan itself and all relations going to it. */ + delete(Plan withId plan.id) + .bind() - /* Lookup in payment backend will fail if no value found for 'planId'. */ - plan.stripePlanId?.let { stripePlanId -> - paymentProcessor.removePlan(stripePlanId) - .mapLeft { - NotDeletedError(type = planEntity.name, id = "Failed to delete ${plan.id}", - error = it) - }.bind() - } + /* Lookup in payment backend will fail if no value found for 'planId'. */ + plan.stripePlanId?.let { stripePlanId -> + paymentProcessor.removePlan(stripePlanId) + .mapLeft { + NotDeletedError(type = planEntity.name, id = "Failed to delete ${plan.id}", + error = it) + }.bind() + } - /* Lookup in payment backend will fail if no value found for 'productId'. */ - plan.stripeProductId?.let { stripeProductId -> - paymentProcessor.removeProduct(stripeProductId) - .mapLeft { - NotDeletedError(type = planEntity.name, id = "Failed to delete ${plan.id}", - error = it) - }.bind() - } - plan - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + /* Lookup in payment backend will fail if no value found for 'productId'. */ + plan.stripeProductId?.let { stripeProductId -> + paymentProcessor.removeProduct(stripeProductId) + .mapLeft { + NotDeletedError(type = planEntity.name, id = "Failed to delete ${plan.id}", + error = it) + }.bind() + } + plan + }.ifFailedThenRollback(transaction) } override fun subscribeToPlan( @@ -2525,25 +2494,22 @@ object Neo4jStoreSingleton : GraphStore { planId: String, trialEnd: Long): Either = writeTransaction { - IO { - Either.monad().binding { + Either.fx { - val customer = getCustomer(identity = identity) - .bind() + val customer = getCustomer(identity = identity) + .bind() - val product = getProduct(identity, planId) - .bind() + val product = getProduct(identity, planId) + .bind() - subscribeToPlan( - customerId = customer.id, - planId = planId, - taxRegionId = product.paymentTaxRegionId) - .bind() + subscribeToPlan( + customerId = customer.id, + planId = planId, + taxRegionId = product.paymentTaxRegionId) + .bind() - Unit - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + Unit + }.ifFailedThenRollback(transaction) } override fun unsubscribeFromPlan(identity: ModelIdentity, planId: String, invoiceNow: Boolean): Either = readTransaction { @@ -2554,29 +2520,27 @@ object Neo4jStoreSingleton : GraphStore { } private fun removeSubscription(customerId: String, planId: String, invoiceNow: Boolean): Either = writeTransaction { - IO { - Either.monad().binding { - val plan = get(Plan withId planId) - .bind() - val planSubscription = get((Customer withId customerId) subscribesTo (Plan withId planId)) - .bind() - .single() - paymentProcessor.cancelSubscription(planSubscription.subscriptionId, invoiceNow) - .mapLeft { - NotDeletedError(type = planEntity.name, id = "$customerId -> ${plan.id}", - error = it) - }.flatMap { - Unit.right() - }.bind() + Either.fx { + val plan = get(Plan withId planId) + .bind() - unlink((Customer withId customerId) subscribesTo (Plan withId planId)) - .flatMap { - Either.right(plan) - }.bind() - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + val planSubscription = get((Customer withId customerId) subscribesTo (Plan withId planId)) + .bind() + .single() + paymentProcessor.cancelSubscription(planSubscription.subscriptionId, invoiceNow) + .mapLeft { + NotDeletedError(type = planEntity.name, id = "$customerId -> ${plan.id}", + error = it) + }.flatMap { + Unit.right() + }.bind() + + unlink((Customer withId customerId) subscribesTo (Plan withId planId)) + .flatMap { + Either.right(plan) + }.bind() + }.ifFailedThenRollback(transaction) } override fun purchasedSubscription( @@ -2586,36 +2550,33 @@ object Neo4jStoreSingleton : GraphStore { sku: String, amount: Long, currency: String): Either = writeTransaction { - IO { - Either.monad().binding { - val product = get(Product withSku sku).bind() - val plan = get(Plan withId sku).bind() - val purchaseRecord = PurchaseRecord( - id = chargeId, - product = product, - timestamp = Instant.now().toEpochMilli(), - properties = mapOf("invoiceId" to invoiceId) - ) + Either.fx { + val product = get(Product withSku sku).bind() + val plan = get(Plan withId sku).bind() + val purchaseRecord = PurchaseRecord( + id = chargeId, + product = product, + timestamp = Instant.now().toEpochMilli(), + properties = mapOf("invoiceId" to invoiceId) + ) - /* Will exit if an existing purchase record matches on 'invoiceId'. */ - createPurchaseRecord(customerId, purchaseRecord) - .bind() + /* Will exit if an existing purchase record matches on 'invoiceId'. */ + createPurchaseRecord(customerId, purchaseRecord) + .bind() - // FIXME Moving customer to new segments should be done only based on productClass. - /* Offer products to the newly signed up subscriber. */ - product.segmentIds.forEach { segmentId -> - assignCustomerToSegment( - customerId = customerId, - segmentId = segmentId, - transaction = transaction) - .bind() - } - logger.info("Customer $customerId completed payment of invoice $invoiceId for subscription to plan ${plan.id}") + // FIXME Moving customer to new segments should be done only based on productClass. + /* Offer products to the newly signed up subscriber. */ + product.segmentIds.forEach { segmentId -> + assignCustomerToSegment( + customerId = customerId, + segmentId = segmentId, + transaction = transaction) + .bind() + } + logger.info("Customer $customerId completed payment of invoice $invoiceId for subscription to plan ${plan.id}") - plan - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + plan + }.ifFailedThenRollback(transaction) } // @@ -2638,87 +2599,86 @@ object Neo4jStoreSingleton : GraphStore { } override fun checkPaymentTransactions(start: Long, end: Long): Either>> = readTransaction { - IO { - Either.monad().binding { - /* To account for time differences for when a payment transaction is stored - to payment backend and to the purchase record DB, the search for - corresponding payment and purchase records are done with a wider time range - that what is specificed with the start and end timestamps. + Either.fx>> { - When all common records has been removed using, the remaining records, if any, - are then againg checked for records that lies excactly within the start and - end timestamps. */ + /* To account for time differences for when a payment transaction is stored + to payment backend and to the purchase record DB, the search for + corresponding payment and purchase records are done with a wider time range + that what is specificed with the start and end timestamps. - val padding = 600000L /* 10 min in milliseconds. */ + When all common records has been removed using, the remaining records, if any, + are then againg checked for records that lies excactly within the start and + end timestamps. */ - val startPadded = if (start < padding) start else start - padding - val endPadded = end + padding + val padding = 600000L /* 10 min in milliseconds. */ - val purchaseRecords = getPurchaseTransactions(startPadded, endPadded) - .mapLeft { - org.ostelco.prime.paymentprocessor.core.NotFoundError("Error when fetching purchase records", - internalError = it) - }.bind() - val paymentRecords = getPaymentTransactions(startPadded, endPadded) - .bind() + val startPadded = if (start < padding) start else start - padding + val endPadded = end + padding - /* TODO: For handling amounts and currencies consider to use - JSR-354 Currency and Money API. */ - - purchaseRecords.map { - mapOf("type" to "purchaseRecord", - "chargeId" to it.id, - "amount" to it.product.price.amount, - "currency" to it.product.price.currency.toLowerCase(), - "refunded" to (it.refund != null), - "created" to it.timestamp) - }.plus( - paymentRecords.map { - mapOf("type" to "paymentRecord", - "chargeId" to it.id, - "amount" to it.amount, - "currency" to it.currency, /* (Stripe) Always lower case. */ - "refunded" to it.refunded, - "created" to it.created) - } - ).groupBy { - it["chargeId"].hashCode() + it["amount"].hashCode() + - it["currency"].hashCode() + it["refunded"].hashCode() - }.map { - /* Report if payment backend and/or purchase record store have - duplicates or more of the same transaction. */ - if (it.value.size > 2) - logger.error(NOTIFY_OPS_MARKER, - "${it.value.size} duplicates found for payment transaction/purchase record ${it.value.first()["chargeId"]}") - it - }.filter { - it.value.size == 1 - }.map { - it.value.first() - }.filter { - /* Filter down to records that lies excactly within the start - and end timestamps. */ - val ts = it["created"] as? Long ?: 0L - - if (ts == 0L) { - logger.error(NOTIFY_OPS_MARKER, if (it["type"] == "purchaseRecord") - "Purchase record ${it["chargeId"]} has 'created' timestamp set to 0" - else - "Payment transaction record ${it["chargeId"]} has 'created' timestamp set to 0") - true - } else { - ts >= start && ts <= end + val purchaseRecords = getPurchaseTransactions(startPadded, endPadded) + .mapLeft { + org.ostelco.prime.paymentprocessor.core.NotFoundError("Error when fetching purchase records", + internalError = it) + }.bind() + val paymentRecords = getPaymentTransactions(startPadded, endPadded) + .bind() + + /* TODO: For handling amounts and currencies consider to use + JSR-354 Currency and Money API. */ + + purchaseRecords.map { + mapOf("type" to "purchaseRecord", + "chargeId" to it.id, + "amount" to it.product.price.amount, + "currency" to it.product.price.currency.toLowerCase(), + "refunded" to (it.refund != null), + "created" to it.timestamp) + }.plus( + paymentRecords.map { + mapOf("type" to "paymentRecord", + "chargeId" to it.id, + "amount" to it.amount, + "currency" to it.currency, /* (Stripe) Always lower case. */ + "refunded" to it.refunded, + "created" to it.created) } - }.map { + ).groupBy { + it["chargeId"].hashCode() + it["amount"].hashCode() + + it["currency"].hashCode() + it["refunded"].hashCode() + }.map { + /* Report if payment backend and/or purchase record store have + duplicates or more of the same transaction. */ + if (it.value.size > 2) + logger.error(NOTIFY_OPS_MARKER, + "${it.value.size} duplicates found for payment transaction/purchase record ${it.value.first()["chargeId"]}") + it + }.filter { + it.value.size == 1 + }.map { + it.value.first() + }.filter { + /* Filter down to records that lies excactly within the start + and end timestamps. */ + val ts = it["created"] as? Long ?: 0L + + if (ts == 0L) { logger.error(NOTIFY_OPS_MARKER, if (it["type"] == "purchaseRecord") - "Found no matching payment transaction record for purchase record ${it["chargeId"]}" + "Purchase record ${it["chargeId"]} has 'created' timestamp set to 0" else - "Found no matching purchase record for payment transaction record ${it["chargeId"]}") - it + "Payment transaction record ${it["chargeId"]} has 'created' timestamp set to 0") + true + } else { + ts >= start && ts <= end } - }.fix() - }.unsafeRunSync() + }.map { + logger.error(NOTIFY_OPS_MARKER, if (it["type"] == "purchaseRecord") + "Found no matching payment transaction record for purchase record ${it["chargeId"]}" + else + "Found no matching purchase record for payment transaction record ${it["chargeId"]}") + it + } + } } // @@ -2743,49 +2703,47 @@ object Neo4jStoreSingleton : GraphStore { identity: ModelIdentity, purchaseRecordId: String, reason: String): Either = writeTransaction { - IO { - Either.monad().binding { - val (_, customerAnalyticsId) = getCustomerAndAnalyticsId(identity = identity) - .mapLeft { - logger.error("Failed to find customer with identity - $identity") - NotFoundPaymentError("Failed to find customer with identity - $identity", - internalError = it) - }.bind() - val purchaseRecord = get(PurchaseRecord withId purchaseRecordId) - // If we can't find the record, return not-found - .mapLeft { - org.ostelco.prime.paymentprocessor.core.NotFoundError("Purchase Record unavailable", - internalError = it) - }.bind() - checkPurchaseRecordForRefund(purchaseRecord) - .bind() - // TODO: (kmm) Move this last. - val refundId = paymentProcessor.refundCharge( - purchaseRecord.id, - purchaseRecord.product.price.amount) - .bind() - val refund = RefundRecord(refundId, reason, Instant.now().toEpochMilli()) - val changedPurchaseRecord = purchaseRecord.copy( - refund = refund - ) - update { changedPurchaseRecord } - .mapLeft { - logger.error("Failed to update purchase record, for refund $refund.id, chargeId $purchaseRecordId, payment has been refunded in Stripe") - UpdatePurchaseError("Failed to update purchase record for refund ${refund.id}", - internalError = it) - }.bind() + Either.fx { + val (_, customerAnalyticsId) = getCustomerAndAnalyticsId(identity = identity) + .mapLeft { + logger.error("Failed to find customer with identity - $identity") + NotFoundPaymentError("Failed to find customer with identity - $identity", + internalError = it) + }.bind() + val purchaseRecord = get(PurchaseRecord withId purchaseRecordId) + // If we can't find the record, return not-found + .mapLeft { + org.ostelco.prime.paymentprocessor.core.NotFoundError("Purchase Record unavailable", + internalError = it) + }.bind() + checkPurchaseRecordForRefund(purchaseRecord) + .bind() - analyticsReporter.reportRefund( - customerAnalyticsId = customerAnalyticsId, - purchaseId = purchaseRecord.id, - reason = reason - ) + // TODO: (kmm) Move this last. + val refundId = paymentProcessor.refundCharge( + purchaseRecord.id, + purchaseRecord.product.price.amount) + .bind() + val refund = RefundRecord(refundId, reason, Instant.now().toEpochMilli()) + val changedPurchaseRecord = purchaseRecord.copy( + refund = refund + ) + update { changedPurchaseRecord } + .mapLeft { + logger.error("Failed to update purchase record, for refund $refund.id, chargeId $purchaseRecordId, payment has been refunded in Stripe") + UpdatePurchaseError("Failed to update purchase record for refund ${refund.id}", + internalError = it) + }.bind() - ProductInfo(purchaseRecord.product.sku) - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + analyticsReporter.reportRefund( + customerAnalyticsId = customerAnalyticsId, + purchaseId = purchaseRecord.id, + reason = reason + ) + + ProductInfo(purchaseRecord.product.sku) + }.ifFailedThenRollback(transaction) } // @@ -2845,18 +2803,15 @@ object Neo4jStoreSingleton : GraphStore { } private fun WriteTransaction.createOffer(offer: ModelOffer): Either { - return IO { - Either.monad().binding { - create { Offer(id = offer.id) }.bind() - for (segmentId in offer.segments) { - fact { (Offer withId offer.id) isOfferedTo (Segment withId segmentId) }.bind() - } - for (sku in offer.products) { - fact { (Offer withId offer.id) containsProduct (Product withSku sku) }.bind() - } - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + return Either.fx { + create { Offer(id = offer.id) }.bind() + for (segmentId in offer.segments) { + fact { (Offer withId offer.id) isOfferedTo (Segment withId segmentId) }.bind() + } + for (sku in offer.products) { + fact { (Offer withId offer.id) containsProduct (Product withSku sku) }.bind() + } + }.ifFailedThenRollback(transaction) } // @@ -2895,16 +2850,13 @@ object Neo4jStoreSingleton : GraphStore { products = productIds, segments = segmentIds) - IO { - Either.monad().binding { + Either.fx { - products.forEach { product -> create { product }.bind() } - segments.forEach { segment -> create { Segment(id = segment.id) }.bind() } - createOffer(actualOffer).bind() + products.forEach { product -> create { product }.bind() } + segments.forEach { segment -> create { Segment(id = segment.id) }.bind() } + createOffer(actualOffer).bind() - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + }.ifFailedThenRollback(transaction) } /** @@ -2912,12 +2864,9 @@ object Neo4jStoreSingleton : GraphStore { */ override fun atomicCreateSegments(createSegments: Collection): Either = writeTransaction { - IO { - Either.monad().binding { - createSegments.forEach { segment -> createSegment(segment).bind() } - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + Either.fx { + createSegments.forEach { segment -> createSegment(segment).bind() } + }.ifFailedThenRollback(transaction) } /** @@ -2925,12 +2874,9 @@ object Neo4jStoreSingleton : GraphStore { */ override fun atomicUpdateSegments(updateSegments: Collection): Either = writeTransaction { - IO { - Either.monad().binding { - updateSegments.forEach { segment -> updateSegment(segment).bind() } - }.fix() - }.unsafeRunSync() - .ifFailedThenRollback(transaction) + Either.fx { + updateSegments.forEach { segment -> updateSegment(segment).bind() } + }.ifFailedThenRollback(transaction) } override fun atomicAddToSegments(addToSegments: Collection): Either { diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt index 0fb287c2f..488ea6c8e 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt @@ -1,10 +1,8 @@ package org.ostelco.prime.storage.graph import arrow.core.Either -import arrow.core.fix +import arrow.core.extensions.fx import arrow.core.right -import arrow.effects.IO -import arrow.instances.either.monad.monad import com.palantir.docker.compose.DockerComposeRule import com.palantir.docker.compose.connection.waiting.HealthChecks import kotlinx.coroutines.runBlocking @@ -1200,74 +1198,68 @@ class Neo4jStoreTest { create { Segment(id = "country-sg") } }.mapLeft { fail(it.message) } - IO { - Either.monad().binding { - - Neo4jStoreSingleton.addCustomer( - identity = IDENTITY, - customer = CUSTOMER).bind() - - Neo4jStoreSingleton.approveRegionForCustomer( - customerId = CUSTOMER.id, - regionCode = "sg") - .bind() - - Neo4jStoreSingleton.addSubscription( - identity = IDENTITY, - regionCode = "sg", - iccId = ICC_ID, - alias = ALIAS, - msisdn = MSISDN) - .mapLeft { fail(it.message) } - .bind() - - // test - Neo4jStoreSingleton.removeCustomer(identity = IDENTITY).bind() - }.fix() - }.unsafeRunSync() - .mapLeft { fail(it.message) } - - // asserts - readTransaction { - IO { - Either.monad().binding { - - val exCustomer = get(ExCustomer withId CUSTOMER.id).bind() - assertEquals( - expected = ExCustomer(id = CUSTOMER.id, createdOn = exCustomer.createdOn, terminationDate = "%d-%02d-%02d".format(LocalDate.now().year, LocalDate.now().monthValue, LocalDate.now().dayOfMonth)), - actual = exCustomer, - message = "ExCustomer does not match") - - val simProfiles = get(org.ostelco.prime.storage.graph.model.SimProfile forExCustomer (ExCustomer withId CUSTOMER.id)).bind() - assertEquals(expected = 1, actual = simProfiles.size, message = "No SIM profiles found for ExCustomer") - - val simProfile = simProfiles[0] - assertEquals( - expected = simProfile.iccId, - actual = ICC_ID, - message = "ICC ID of simProfile for ExCustomer do not match") + Either.fx { - val subscriptions = get(Subscription wasSubscribedBy (ExCustomer withId CUSTOMER.id)).bind() - assertEquals(expected = 1, actual = subscriptions.size, message = "No subscriptions found for ExCustomer") + Neo4jStoreSingleton.addCustomer( + identity = IDENTITY, + customer = CUSTOMER).bind() - val subscription = subscriptions[0] - assertEquals( - expected = subscription.msisdn, - actual = MSISDN, - message = "MSISDN of subscription for ExCustomer do not match") + Neo4jStoreSingleton.approveRegionForCustomer( + customerId = CUSTOMER.id, + regionCode = "sg") + .bind() - val regions = get(Region linkedToExCustomer (ExCustomer withId CUSTOMER.id)).bind() - assertEquals(expected = 1, actual = regions.size, message = "No regions found for ExCustomer") + Neo4jStoreSingleton.addSubscription( + identity = IDENTITY, + regionCode = "sg", + iccId = ICC_ID, + alias = ALIAS, + msisdn = MSISDN) + .mapLeft { fail(it.message) } + .bind() - val region = regions[0] - assertEquals( - expected = Region("sg", "Singapore"), - actual = region, - message = "Region for ExCustomer do not match") + // test + Neo4jStoreSingleton.removeCustomer(identity = IDENTITY).bind() + }.mapLeft { fail(it.message) } - }.fix() - }.unsafeRunSync() - .mapLeft { fail(it.message) } + // asserts + readTransaction { + Either.fx { + + val exCustomer = get(ExCustomer withId CUSTOMER.id).bind() + assertEquals( + expected = ExCustomer(id = CUSTOMER.id, createdOn = exCustomer.createdOn, terminationDate = "%d-%02d-%02d".format(LocalDate.now().year, LocalDate.now().monthValue, LocalDate.now().dayOfMonth)), + actual = exCustomer, + message = "ExCustomer does not match") + + val simProfiles = get(org.ostelco.prime.storage.graph.model.SimProfile forExCustomer (ExCustomer withId CUSTOMER.id)).bind() + assertEquals(expected = 1, actual = simProfiles.size, message = "No SIM profiles found for ExCustomer") + + val simProfile = simProfiles[0] + assertEquals( + expected = simProfile.iccId, + actual = ICC_ID, + message = "ICC ID of simProfile for ExCustomer do not match") + + val subscriptions = get(Subscription wasSubscribedBy (ExCustomer withId CUSTOMER.id)).bind() + assertEquals(expected = 1, actual = subscriptions.size, message = "No subscriptions found for ExCustomer") + + val subscription = subscriptions[0] + assertEquals( + expected = subscription.msisdn, + actual = MSISDN, + message = "MSISDN of subscription for ExCustomer do not match") + + val regions = get(Region linkedToExCustomer (ExCustomer withId CUSTOMER.id)).bind() + assertEquals(expected = 1, actual = regions.size, message = "No regions found for ExCustomer") + + val region = regions[0] + assertEquals( + expected = Region("sg", "Singapore"), + actual = region, + message = "Region for ExCustomer do not match") + + }.mapLeft { fail(it.message) } } } From 7b8b968157da1e96e7810da70921fe633b0ee1c2 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 15:08:04 +0100 Subject: [PATCH 46/56] In Neo4jStore, removed bind() for assignments in Arrow Fx --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 119 ++++++++---------- 1 file changed, 54 insertions(+), 65 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index c24857657..4613b3483 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -439,7 +439,7 @@ object Neo4jStoreSingleton : GraphStore { override fun removeCustomer(identity: ModelIdentity): Either = writeTransaction { Either.fx { // get customer id - val customer = getCustomer(identity).bind() + val (customer) = getCustomer(identity) val customerId = customer.id // create ex-customer with same id create { ExCustomer(id = customerId, terminationDate = LocalDate.now().toString(), createdOn = customer.createdOn) }.bind() @@ -449,7 +449,7 @@ object Neo4jStoreSingleton : GraphStore { fact { (ExCustomer withId customerId) subscribedTo (Subscription withMsisdn subscription.msisdn) }.bind() } // get all SIM profiles and link them to ex-customer. - val simProfiles = get(SimProfile forCustomer (Customer withId customerId)).bind() + val (simProfiles) = get(SimProfile forCustomer (Customer withId customerId)) val simProfileRegions = mutableSetOf() for (simProfile in simProfiles) { fact { (ExCustomer withId customerId) had (SimProfile withId simProfile.id) }.bind() @@ -457,7 +457,7 @@ object Neo4jStoreSingleton : GraphStore { simProfileRegions.addAll(get(Region linkedToSimProfile (SimProfile withId simProfile.id)).bind()) } // get Regions linked to Customer - val regions = get(Region linkedToCustomer (Customer withId customerId)).bind() + val (regions) = get(Region linkedToCustomer (Customer withId customerId)) // TODO vihang: clear eKYC data for Regions without any SimProfile // val regionsWithoutSimProfile = regions - simProfileRegions // // Link regions with SIM profiles to ExCustomer @@ -684,7 +684,7 @@ object Neo4jStoreSingleton : GraphStore { logger.warn("Found {} SIM Profiles with iccId {}", simProfiles.size, iccId) } simProfiles.forEach { simProfile -> - val customers = get(Customer withSimProfile (SimProfile withId simProfile.id)).bind() + val (customers) = get(Customer withSimProfile (SimProfile withId simProfile.id)) customers.forEach { customer -> AuditLog.info(customerId = customer.id, message = "Sim Profile (iccId = $iccId) is $status") } @@ -700,7 +700,7 @@ object Neo4jStoreSingleton : GraphStore { if (timestampField != null) { update(SimProfile withId simProfile.id, set = timestampField to utcTimeNow()).bind() } - val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() + val (subscriptions) = get(Subscription under (SimProfile withId simProfile.id)) subscriptions.forEach { subscription -> logger.info("Notify status {} for subscription.analyticsId {}", status, subscription.analyticsId) analyticsReporter.reportSubscriptionStatusUpdate(subscription.analyticsId, status) @@ -735,10 +735,10 @@ object Neo4jStoreSingleton : GraphStore { profileType: String?, alias: String): Either = writeTransaction { Either.fx { - val customerId = getCustomerId(identity = identity).bind() - val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() + val (customerId) = getCustomerId(identity = identity) + val (bundles) = get(Bundle forCustomer (Customer withId customerId)) validateBundleList(bundles, customerId).bind() - val customer = get(Customer withId customerId).bind() + val (customer) = get(Customer withId customerId) val status = customerRegionRelationStore.get( fromId = customerId, toId = regionCode.toLowerCase(), @@ -749,10 +749,9 @@ object Neo4jStoreSingleton : GraphStore { status = status, customerId = customerId, regionCode = regionCode.toLowerCase()).bind() - val region = get(Region withCode regionCode.toLowerCase()).bind() - val simEntry = simManager.allocateNextEsimProfile(hlr = hssNameLookup.getHssName(region.id.toLowerCase()), phoneType = profileType) + val (region) = get(Region withCode regionCode.toLowerCase()) + val (simEntry) = simManager.allocateNextEsimProfile(hlr = hssNameLookup.getHssName(region.id.toLowerCase()), phoneType = profileType) .mapLeft { NotFoundError("eSIM profile", id = "Loltel") } - .bind() val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId, alias = alias, requestedOn = utcTimeNow()) create { simProfile }.bind() fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() @@ -823,9 +822,8 @@ object Neo4jStoreSingleton : GraphStore { val simProfiles = readTransaction { Either.fx> { - val customerId = getCustomerId(identity = identity).bind() - val simProfiles = get(SimProfile forCustomer (Customer withId customerId)) - .bind() + val (customerId) = getCustomerId(identity = identity) + val (simProfiles) = get(SimProfile forCustomer (Customer withId customerId)) if (regionCode == null) { simProfiles.forEach { simProfile -> val region = get(Region linkedToSimProfile (SimProfile withId simProfile.id)) @@ -892,7 +890,7 @@ object Neo4jStoreSingleton : GraphStore { val simProfileEither = writeTransaction { Either.fx { - val customerId = getCustomerId(identity = identity).bind() + val (customerId) = getCustomerId(identity = identity) val simProfile = get(SimProfile forCustomer (Customer withId customerId)) .bind() .firstOrNull { simProfile -> simProfile.iccId == iccId } @@ -905,7 +903,7 @@ object Neo4jStoreSingleton : GraphStore { } return Either.fx { - val simProfile = simProfileEither.bind() + val (simProfile) = simProfileEither val simEntry = simManager.getSimProfile( hlr = hssNameLookup.getHssName(regionCode), iccId = iccId) @@ -941,7 +939,7 @@ object Neo4jStoreSingleton : GraphStore { val simProfileEither = writeTransaction { Either.fx { - val customerId = getCustomerId(identity = identity).bind() + val (customerId) = getCustomerId(identity = identity) val simProfile = get(SimProfile forCustomer (Customer withId customerId)) .bind() .firstOrNull { simProfile -> simProfile.iccId == iccId } @@ -956,7 +954,7 @@ object Neo4jStoreSingleton : GraphStore { } return Either.fx { - val simProfile = simProfileEither.bind() + val (simProfile) = simProfileEither val simEntry = simManager.getSimProfile( hlr = hssNameLookup.getHssName(regionCode), iccId = iccId) @@ -992,7 +990,7 @@ object Neo4jStoreSingleton : GraphStore { val infoEither = readTransaction { Either.fx> { - val customer = getCustomer(identity = identity).bind() + val (customer) = getCustomer(identity = identity) val simProfile = get(SimProfile forCustomer (Customer withId customer.id)) .bind() .firstOrNull { simProfile -> simProfile.iccId == iccId } @@ -1004,13 +1002,12 @@ object Neo4jStoreSingleton : GraphStore { return Either.fx { val (customer, simProfile) = infoEither.bind() - val simEntry = simManager.getSimProfile( + val (simEntry) = simManager.getSimProfile( hlr = hssNameLookup.getHssName(regionCode), iccId = iccId) .mapLeft { NotFoundError(type = simProfileEntity.name, id = simProfile.iccId) } - .bind() emailNotifier.sendESimQrCodeEmail( email = customer.contactEmail, @@ -1037,9 +1034,9 @@ object Neo4jStoreSingleton : GraphStore { override fun deleteSimProfileWithSubscription(regionCode: String, iccId: String): Either = writeTransaction { Either.fx { - val simProfiles = get(SimProfile linkedToRegion (Region withCode regionCode)).bind() + val (simProfiles) = get(SimProfile linkedToRegion (Region withCode regionCode)) simProfiles.forEach { simProfile -> - val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() + val (subscriptions) = get(Subscription under (SimProfile withId simProfile.id)) subscriptions.forEach { subscription -> delete(Subscription withMsisdn subscription.msisdn).bind() } @@ -1061,7 +1058,7 @@ object Neo4jStoreSingleton : GraphStore { msisdn: String): Either = writeTransaction { Either.fx { - val customerId = getCustomerId(identity = identity).bind() + val (customerId) = getCustomerId(identity = identity) val simProfile = SimProfile( id = UUID.randomUUID().toString(), @@ -1075,7 +1072,7 @@ object Neo4jStoreSingleton : GraphStore { fact { (Subscription withMsisdn msisdn) isUnder (SimProfile withId simProfile.id) }.bind() fact { (Customer withId customerId) subscribesTo (Subscription withMsisdn msisdn) }.bind() - val bundles = get(Bundle forCustomer (Customer withId customerId)).bind() + val (bundles) = get(Bundle forCustomer (Customer withId customerId)) validateBundleList(bundles, customerId).bind() bundles.forEach { bundle -> fact { (Subscription withMsisdn msisdn) consumesFrom (Bundle withId bundle.id) using SubscriptionToBundle() }.bind() @@ -1087,7 +1084,7 @@ object Neo4jStoreSingleton : GraphStore { override fun getSubscriptions(identity: ModelIdentity, regionCode: String?): Either> = readTransaction { Either.fx> { - val customerId = getCustomerId(identity = identity).bind() + val (customerId) = getCustomerId(identity = identity) if (regionCode == null) { get(Subscription subscribedBy (Customer withId customerId)).bind() } else { @@ -1238,19 +1235,18 @@ object Neo4jStoreSingleton : GraphStore { Either.fx { - val customer = getCustomer(identity = identity) + val (customer) = getCustomer(identity = identity) .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError( "Failed to get customer data for customer with identity - $identity", internalError = it) - }.bind() + } - val product = getProduct(identity, sku) + val (product) = getProduct(identity, sku) .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Product $sku is unavailable", internalError = it) } - .bind() if (product.price.amount > 0) { val (chargeId, invoiceId) = when (product.paymentType) { @@ -1413,11 +1409,11 @@ object Neo4jStoreSingleton : GraphStore { } if (sourceId != null) { - val sourceDetails = paymentProcessor.getSavedSources(customer.id) + val (sourceDetails) = paymentProcessor.getSavedSources(customer.id) .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Failed to fetch sources for customer: ${customer.id}", internalError = it) - }.bind() + } if (!sourceDetails.any { sourceDetailsInfo -> sourceDetailsInfo.id == sourceId }) { paymentProcessor.addSource(customer.id, sourceId) .bind().id @@ -1444,13 +1440,12 @@ object Neo4jStoreSingleton : GraphStore { trialEnd: Long = 0L): Either { return Either.fx { - val plan = get(Plan withId planId) - .bind() - val profileInfo = paymentProcessor.getPaymentProfile(customerId) + val (plan) = get(Plan withId planId) + val (profileInfo) = paymentProcessor.getPaymentProfile(customerId) .mapLeft { NotFoundError(type = planEntity.name, id = "Failed to subscribe $customerId to ${plan.id}", error = it) - }.bind() + } /* At this point, we have either: 1) A new subscription to a plan is being created. @@ -1579,7 +1574,7 @@ object Neo4jStoreSingleton : GraphStore { }.bind() /* Force immediate payment of the invoice. */ - val invoicePaymentInfo = paymentProcessor.payInvoice(invoice.id) + val (invoicePaymentInfo) = paymentProcessor.payInvoice(invoice.id) .mapLeft { logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") /* Adds failed purchase to customer history. */ @@ -1593,7 +1588,7 @@ object Neo4jStoreSingleton : GraphStore { Refunded customer ${customer.id} for invoice: ${it.id}. Verify that the invoice has been refunded in Stripe dashboard. """.trimIndent()) - }.bind() + } invoicePaymentInfo } @@ -1935,7 +1930,7 @@ object Neo4jStoreSingleton : GraphStore { authorisationCode: String): Either { return Either.fx { - val customer = getCustomer(identity = identity).bind() + val (customer) = getCustomer(identity = identity) // set MY_INFO KYC Status to Pending setKycStatus( @@ -2004,7 +1999,7 @@ object Neo4jStoreSingleton : GraphStore { logger.info("checkNricFinIdUsingDave for $nricFinId") - val customer = getCustomer(identity = identity).bind() + val (customer) = getCustomer(identity = identity) // set NRIC_FIN KYC Status to Pending setKycStatus( @@ -2045,7 +2040,7 @@ object Neo4jStoreSingleton : GraphStore { return Either.fx { - val customer = getCustomer(identity = identity).bind() + val (customer) = getCustomer(identity = identity) // set ADDRESS KYC Status to Pending setKycStatus( @@ -2108,7 +2103,7 @@ object Neo4jStoreSingleton : GraphStore { val approvedKycTypeSetList = getApprovedKycTypeSetList(regionCode) // fetch existing values from DB - val existingCustomerRegion = customerRegionRelationStore.get( + val (existingCustomerRegion) = customerRegionRelationStore.get( fromId = customer.id, toId = regionCode, transaction = transaction) @@ -2123,7 +2118,7 @@ object Neo4jStoreSingleton : GraphStore { } else { storeError.left() } - }.bind() + } // using existing and received KYC status, compute new KYC status val existingKycStatusMap = existingCustomerRegion.kycStatusMap @@ -2414,14 +2409,14 @@ object Neo4jStoreSingleton : GraphStore { .bind() } - val productInfo = paymentProcessor.createProduct(stripeProductName) + val (productInfo) = paymentProcessor.createProduct(stripeProductName) .mapLeft { NotCreatedError(type = planEntity.name, id = "Failed to create plan ${plan.id}", error = it) }.linkReversalActionToTransaction(transaction) { paymentProcessor.removeProduct(it.id) - }.bind() - val planInfo = paymentProcessor.createPlan( + } + val (planInfo) = paymentProcessor.createPlan( productInfo.id, planProduct.price.amount, planProduct.price.currency, @@ -2431,7 +2426,7 @@ object Neo4jStoreSingleton : GraphStore { error = it) }.linkReversalActionToTransaction(transaction) { paymentProcessor.removePlan(it.id) - }.bind() + } /* The associated product to the plan. Note that: sku - name of the plan @@ -2456,8 +2451,7 @@ object Neo4jStoreSingleton : GraphStore { override fun deletePlan(planId: String): Either = writeTransaction { Either.fx { - val plan = get(Plan withId planId) - .bind() + val (plan) = get(Plan withId planId) /* The name of the product is the same as the name of the corresponding plan. */ get(Product withSku planId) .bind() @@ -2496,11 +2490,9 @@ object Neo4jStoreSingleton : GraphStore { Either.fx { - val customer = getCustomer(identity = identity) - .bind() + val (customer) = getCustomer(identity = identity) - val product = getProduct(identity, planId) - .bind() + val (product) = getProduct(identity, planId) subscribeToPlan( customerId = customer.id, @@ -2522,8 +2514,7 @@ object Neo4jStoreSingleton : GraphStore { private fun removeSubscription(customerId: String, planId: String, invoiceNow: Boolean): Either = writeTransaction { Either.fx { - val plan = get(Plan withId planId) - .bind() + val (plan) = get(Plan withId planId) val planSubscription = get((Customer withId customerId) subscribesTo (Plan withId planId)) .bind() @@ -2551,8 +2542,8 @@ object Neo4jStoreSingleton : GraphStore { amount: Long, currency: String): Either = writeTransaction { Either.fx { - val product = get(Product withSku sku).bind() - val plan = get(Plan withId sku).bind() + val (product) = get(Product withSku sku) + val (plan) = get(Plan withId sku) val purchaseRecord = PurchaseRecord( id = chargeId, product = product, @@ -2616,13 +2607,12 @@ object Neo4jStoreSingleton : GraphStore { val startPadded = if (start < padding) start else start - padding val endPadded = end + padding - val purchaseRecords = getPurchaseTransactions(startPadded, endPadded) + val (purchaseRecords) = getPurchaseTransactions(startPadded, endPadded) .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Error when fetching purchase records", internalError = it) - }.bind() - val paymentRecords = getPaymentTransactions(startPadded, endPadded) - .bind() + } + val (paymentRecords) = getPaymentTransactions(startPadded, endPadded) /* TODO: For handling amounts and currencies consider to use JSR-354 Currency and Money API. */ @@ -2711,20 +2701,19 @@ object Neo4jStoreSingleton : GraphStore { NotFoundPaymentError("Failed to find customer with identity - $identity", internalError = it) }.bind() - val purchaseRecord = get(PurchaseRecord withId purchaseRecordId) + val (purchaseRecord) = get(PurchaseRecord withId purchaseRecordId) // If we can't find the record, return not-found .mapLeft { org.ostelco.prime.paymentprocessor.core.NotFoundError("Purchase Record unavailable", internalError = it) - }.bind() + } checkPurchaseRecordForRefund(purchaseRecord) .bind() // TODO: (kmm) Move this last. - val refundId = paymentProcessor.refundCharge( + val (refundId) = paymentProcessor.refundCharge( purchaseRecord.id, purchaseRecord.product.price.amount) - .bind() val refund = RefundRecord(refundId, reason, Instant.now().toEpochMilli()) val changedPurchaseRecord = purchaseRecord.copy( refund = refund From 41904a7d34f4e50c3e706688b6f6eb8db2bbfe35 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Sat, 30 Nov 2019 15:22:31 +0100 Subject: [PATCH 47/56] Updated neo4j-store kts files with Arrow Fx --- .../main/resources/OnNewCustomerAction.kts | 42 +++++++++---------- .../main/resources/OnRegionApprovedAction.kts | 32 +++++++------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/neo4j-store/src/main/resources/OnNewCustomerAction.kts b/neo4j-store/src/main/resources/OnNewCustomerAction.kts index 07bd21947..0fc6d47fb 100644 --- a/neo4j-store/src/main/resources/OnNewCustomerAction.kts +++ b/neo4j-store/src/main/resources/OnNewCustomerAction.kts @@ -1,7 +1,5 @@ import arrow.core.Either -import arrow.core.fix -import arrow.effects.IO -import arrow.instances.either.monad.monad +import arrow.core.extensions.fx import org.ostelco.prime.dsl.WriteTransaction import org.ostelco.prime.dsl.withSku import org.ostelco.prime.model.Customer @@ -23,25 +21,23 @@ object : OnNewCustomerAction { val welcomePackProductSku = "2GB_FREE_ON_JOINING" - return IO { - Either.monad().binding { - WriteTransaction(transaction).apply { - val product = get(Product withSku welcomePackProductSku).bind() - createPurchaseRecord( - customer.id, - PurchaseRecord( - id = UUID.randomUUID().toString(), - product = product, - timestamp = Instant.now().toEpochMilli() - ) - ).bind() - applyProduct( - customerId = customer.id, - product = product - ).bind() - } - Unit - }.fix() - }.unsafeRunSync() + return Either.fx { + WriteTransaction(transaction).apply { + val (product) = get(Product withSku welcomePackProductSku) + createPurchaseRecord( + customer.id, + PurchaseRecord( + id = UUID.randomUUID().toString(), + product = product, + timestamp = Instant.now().toEpochMilli() + ) + ).bind() + applyProduct( + customerId = customer.id, + product = product + ).bind() + } + Unit + } } } \ No newline at end of file diff --git a/neo4j-store/src/main/resources/OnRegionApprovedAction.kts b/neo4j-store/src/main/resources/OnRegionApprovedAction.kts index afac60c99..b4af1119e 100644 --- a/neo4j-store/src/main/resources/OnRegionApprovedAction.kts +++ b/neo4j-store/src/main/resources/OnRegionApprovedAction.kts @@ -1,8 +1,5 @@ import arrow.core.Either -import arrow.core.fix -import arrow.core.getOrElse -import arrow.effects.IO -import arrow.instances.either.monad.monad +import arrow.core.extensions.fx import org.ostelco.prime.auditlog.AuditLog import org.ostelco.prime.dsl.WriteTransaction import org.ostelco.prime.dsl.withId @@ -19,19 +16,18 @@ object : OnRegionApprovedAction { regionCode: String, transaction: PrimeTransaction ): Either { - return IO { - Either.monad().binding { - WriteTransaction(transaction).apply { - val segmentId = get(Segment withId "plan-country-${regionCode.toLowerCase()}") - .getOrElse { - get(Segment withId "country-${regionCode.toLowerCase()}").bind() - } - .id - fact { (Customer withId customer.id) belongsToSegment (Segment withId segmentId) }.bind() - AuditLog.info(customer.id, "Added customer to segment - $segmentId") - } - Unit - }.fix() - }.unsafeRunSync() + return Either.fx { + WriteTransaction(transaction).apply { + val segmentId = get(Segment withId "plan-country-${regionCode.toLowerCase()}") + .fold( + { get(Segment withId "country-${regionCode.toLowerCase()}").bind() }, + { it } + ) + .id + fact { (Customer withId customer.id) belongsToSegment (Segment withId segmentId) }.bind() + AuditLog.info(customer.id, "Added customer to segment - $segmentId") + } + Unit + } } } \ No newline at end of file From c6b0893f0ee38dd2c38b6235c92328fdc6124fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 2 Dec 2019 10:39:13 +0100 Subject: [PATCH 48/56] Don't log irrelevant logs --- .../inventory/SimInventoryCallbackService.kt | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt index 0bdedce07..c3a9f3033 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/inventory/SimInventoryCallbackService.kt @@ -92,14 +92,30 @@ class SimInventoryCallbackService(val dao: SimInventoryDAO) : SmDpPlusCallbackSe } } } else { - /* XXX Update to handle other cases explicitly + review of logging. */ - logger.warn("download-progress-info: Received message with notificationPointStatus {} for ICCID {}" + - "(notificationPointId: {}, profileType: {}, resultData: {})", - notificationPointStatus, - numericIccId, - notificationPointId, - profileType, - resultData) + // Log non-successful operations differently. Confirmation failures and eligibility retry checks + // are not something that should show up in the logs and take attention from ops personnel, so we're + // just logging it at info level. Everything else shouldn't fail and if it does it should be researched + // by ops. + when (notificationPointId) { + 1, 2 -> { + logger.info("download-progress-info: Received message with notificationPointStatus {} for ICCID {}" + + "(notificationPointId: {}, profileType: {}, resultData: {})", + notificationPointStatus, + numericIccId, + notificationPointId, + profileType, + resultData) + } + else -> { + logger.warn("download-progress-info: Received message with notificationPointStatus {} for ICCID {}" + + "(notificationPointId: {}, profileType: {}, resultData: {})", + notificationPointStatus, + numericIccId, + notificationPointId, + profileType, + resultData) + } + } } } From 9ff333702396c14c3888eecca34a7aad417d7fec Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 15 Nov 2019 20:06:28 +0100 Subject: [PATCH 49/56] Updated Neo4j Load Test --- .../prime/storage/graph/Neo4jLoadTest.kt | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt index a2870f169..85239098e 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt @@ -17,7 +17,10 @@ import org.ostelco.prime.model.Customer import org.ostelco.prime.model.Identity import org.ostelco.prime.model.Price import org.ostelco.prime.model.Product +import org.ostelco.prime.model.ProductClass.SIMPLE_DATA import org.ostelco.prime.model.ProductProperties.NO_OF_BYTES +import org.ostelco.prime.model.ProductProperties.PRODUCT_CLASS +import org.ostelco.prime.model.Region import org.ostelco.prime.storage.graph.model.Segment import java.util.* import java.util.concurrent.CountDownLatch @@ -45,15 +48,26 @@ class Neo4jLoadTest { create { Product(sku = "2GB_FREE_ON_JOINING", price = Price(0, ""), - properties = mapOf(NO_OF_BYTES.s to "2_147_483_648")) + properties = mapOf( + PRODUCT_CLASS.s to SIMPLE_DATA.name, + NO_OF_BYTES.s to "${2.GiB()}" + ) + ) } create { Product(sku = "1GB_FREE_ON_REFERRED", price = Price(0, ""), - properties = mapOf(NO_OF_BYTES.s to "1_000_000_000")) + properties = mapOf( + PRODUCT_CLASS.s to SIMPLE_DATA.name, + NO_OF_BYTES.s to "1_000_000_000" + ) + ) } create { - Segment(id = "country-${COUNTRY.toLowerCase()}") + Segment(id = "country-$COUNTRY_CODE}") + } + create { + Region(id = COUNTRY_CODE, name = "Norway") } } } @@ -93,12 +107,11 @@ class Neo4jLoadTest { repeat(COUNT) { i -> launch { Neo4jStoreSingleton.consume(msisdn = "${i % USERS}", usedBytes = USED, requestedBytes = REQUESTED) { storeResult -> - storeResult.fold( + storeResult.bimap( { fail(it.message) }, { - // println("Balance = %,d, Granted = %,d".format(it.second, it.first)) cdl.countDown() - assert(true) + // println("Balance = %,d, Granted = %,d".format(it.balance, it.granted)) }) } } @@ -120,7 +133,7 @@ class Neo4jLoadTest { .fold( { fail(it.message) }, { - assertEquals(expected = 100_000_000 - COUNT / USERS * USED - REQUESTED, + assertEquals(expected = 2.GiB() - COUNT / USERS * USED - REQUESTED, actual = it.single().balance, message = "Balance does not match") } @@ -136,7 +149,7 @@ class Neo4jLoadTest { const val REQUESTED = 100L const val NAME = "Test User" - const val COUNTRY = "NO" + const val COUNTRY_CODE = "no" @ClassRule @JvmField @@ -196,4 +209,6 @@ class Neo4jLoadTest { Neo4jClient.stop() } } -} \ No newline at end of file +} + +fun Int.GiB() = this.toLong() * 1024 * 1024 * 1024 From 4dfa5907ca513af37b6c03bc71901cfe0fbd2b64 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Fri, 15 Nov 2019 20:07:48 +0100 Subject: [PATCH 50/56] Removed CountDownLatch from Neo4j Load Test --- .../kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt index 85239098e..a5ca70e46 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jLoadTest.kt @@ -23,7 +23,6 @@ import org.ostelco.prime.model.ProductProperties.PRODUCT_CLASS import org.ostelco.prime.model.Region import org.ostelco.prime.storage.graph.model.Segment import java.util.* -import java.util.concurrent.CountDownLatch import kotlin.system.measureTimeMillis import kotlin.test.BeforeTest import kotlin.test.Ignore @@ -101,8 +100,6 @@ class Neo4jLoadTest { val durationInMillis = measureTimeMillis { - val cdl = CountDownLatch(COUNT) - runBlocking(Dispatchers.Default) { repeat(COUNT) { i -> launch { @@ -110,7 +107,6 @@ class Neo4jLoadTest { storeResult.bimap( { fail(it.message) }, { - cdl.countDown() // println("Balance = %,d, Granted = %,d".format(it.balance, it.granted)) }) } @@ -120,8 +116,6 @@ class Neo4jLoadTest { // Wait for all the responses to be returned println("Waiting for all responses to be returned") } - - cdl.await() } // Print load test results From 3548c39c6fa09f6d4ba5feda6d663c8dc499f23c Mon Sep 17 00:00:00 2001 From: mpeterss Date: Mon, 2 Dec 2019 14:53:19 +0100 Subject: [PATCH 51/56] Add smdp-ples emulator to enable seagull test again --- docker-compose.seagull.yaml | 30 ++++++++++++++++++- .../ccr-cca.client.multiple-cc-units.init.xml | 10 ++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docker-compose.seagull.yaml b/docker-compose.seagull.yaml index 9b16816ac..fb9071535 100644 --- a/docker-compose.seagull.yaml +++ b/docker-compose.seagull.yaml @@ -23,13 +23,21 @@ services: - DATASTORE_EMULATOR_HOST=localhost:9090 - DATASTORE_PROJECT_ID=${GCP_PROJECT_ID} - MANDRILL_API_KEY= + - LOCAL_TESTING=true + - DB_USER=postgres_user + - DB_PASSWORD=postgres_password + - DB_URL=postgres:5432/sim-inventory + - SMDPLUS_ES2PLUS_ENDPOINT=http://smdp-plus-emulator:8080 - ACCEPTANCE_TESTING=true ports: - "9090:8080" - "8082:8082" + - "8081:8081" depends_on: + - "ext-auth-provider" - "datastore-emulator" - "pubsub-emulator" + - "smdp-plus-emulator" - "neo4j" command: ["/bin/bash", "./wait.sh"] tmpfs: @@ -60,7 +68,8 @@ services: - PUBSUB_CCA_SUBSCRIPTION_ID=ocsgw-cca-sub - PUBSUB_ACTIVATE_SUBSCRIPTION_ID=ocsgw-activate-sub - DIAMETER_CONFIG_FILE=server-jdiameter-config.xml - - OCS_DATASOURCE_TYPE=Local + - OCS_DATASOURCE_TYPE=Proxy + - OCS_SECONDARY_DATASOURCE_TYPE=PubSub - CONFIG_FOLDER=/config/ - "JAVA_OPTS=-Xms512m -Xmx1024m -server" volumes: @@ -97,6 +106,25 @@ services: - DATASTORE_DATASET=${GCP_PROJECT_ID} command: ["gcloud", "beta", "emulators", "datastore", "start", "--host-port=0.0.0.0:8081"] + ext-auth-provider: + container_name: ext-auth-provider + build: ext-auth-provider + + smdp-plus-emulator: + container_name: smdp-plus-emulator + build: sim-administration/sm-dp-plus-emulator + ports: + - "18080:8080" + - "18081:8081" + + postgres: + container_name: postgres + build: + context: sim-administration/postgres + dockerfile: Dockerfile + tmpfs: "//var/lib/postgresql/data" + ports: + - "55432:5432" networks: net: diff --git a/seagull/scenario/ccr-cca.client.multiple-cc-units.init.xml b/seagull/scenario/ccr-cca.client.multiple-cc-units.init.xml index a76254580..5ce6aa1e0 100644 --- a/seagull/scenario/ccr-cca.client.multiple-cc-units.init.xml +++ b/seagull/scenario/ccr-cca.client.multiple-cc-units.init.xml @@ -87,7 +87,15 @@ - + + + + + + + + + From 4f3b40413fe5bc9e485cc220ca9e0d2c3cff8e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Cederl=C3=B6f?= Date: Tue, 3 Dec 2019 10:24:16 +0100 Subject: [PATCH 52/56] Fix typo --- docker-compose.seagull.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.seagull.yaml b/docker-compose.seagull.yaml index fb9071535..2d801d942 100644 --- a/docker-compose.seagull.yaml +++ b/docker-compose.seagull.yaml @@ -122,7 +122,7 @@ services: build: context: sim-administration/postgres dockerfile: Dockerfile - tmpfs: "//var/lib/postgresql/data" + tmpfs: "/var/lib/postgresql/data" ports: - "55432:5432" From 334946b31ba3b94f6751f91f365c52e5875b539d Mon Sep 17 00:00:00 2001 From: mpeterss Date: Wed, 4 Dec 2019 10:45:56 +0100 Subject: [PATCH 53/56] More information in ocs gw logs Added latency information Added request number to each log --- .../kotlin/org/ostelco/diameter/CreditControlContext.kt | 6 ++++-- .../ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java | 1 + .../ocsgw/datasource/protobuf/ProtobufDataSource.kt | 4 ++-- .../ostelco/ocsgw/datasource/protobuf/PubSubDataSource.kt | 7 +++++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/diameter-stack/src/main/kotlin/org/ostelco/diameter/CreditControlContext.kt b/diameter-stack/src/main/kotlin/org/ostelco/diameter/CreditControlContext.kt index 4251576a3..d5447d5f0 100644 --- a/diameter-stack/src/main/kotlin/org/ostelco/diameter/CreditControlContext.kt +++ b/diameter-stack/src/main/kotlin/org/ostelco/diameter/CreditControlContext.kt @@ -35,7 +35,9 @@ class CreditControlContext( // Set to true, when answer to not to be sent to P-GW. This logic is used by ProxyDatasource. var skipAnswer: Boolean = false - var requestTime = System.currentTimeMillis() + private val contextCreatedTime = System.currentTimeMillis() + + var sentToOcsTime: Long = 0L val creditControlRequest: CreditControlRequest = AvpParser().parse( CreditControlRequest::class, @@ -136,6 +138,6 @@ class CreditControlContext( } fun logLatency() { - logger.info("Time from request to answer {} ms. SessionId [{}]", System.currentTimeMillis() - requestTime, this.sessionId) + logger.info("Time from request to answer {} ms. OCS roundtrip {} ms. SessionId [{}] request number [{}]", System.currentTimeMillis() - contextCreatedTime, System.currentTimeMillis() - sentToOcsTime, this.sessionId, this.creditControlRequest.ccRequestNumber?.unsigned32) } } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java index 901951eb1..5e9b2aa0e 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/GrpcDataSource.java @@ -270,6 +270,7 @@ public void handleRequest(final CreditControlContext context) { CreditControlRequestInfo creditControlRequestInfo = protobufDataSource.handleRequest(context, null); if (creditControlRequestInfo != null) { + context.setSentToOcsTime(System.currentTimeMillis()); producer.queueEvent(creditControlRequestInfo); } } diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt index 7038cc4f8..0b7c3f0b4 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/ProtobufDataSource.kt @@ -41,11 +41,11 @@ class ProtobufDataSource { fun handleCcrAnswer(answer: CreditControlAnswerInfo) { try { - logger.info("[<<] CreditControlAnswer for msisdn {} requestId {}", answer.msisdn, answer.requestId) + logger.info("[<<] CreditControlAnswer for msisdn {} requestId {} request number [{}]", answer.msisdn, answer.requestId, answer.requestNumber) val ccrContext = ccrMap.remove(answer.requestId + "-" + answer.requestNumber) if (ccrContext != null) { ccrContext.logLatency() - logger.debug("Found Context for answer msisdn {} requestId [{}] request number {}", ccrContext.creditControlRequest.msisdn, ccrContext.sessionId, ccrContext.creditControlRequest.ccRequestNumber?.integer32) + logger.debug("Found Context for answer msisdn {} requestId [{}] request number [{}]", ccrContext.creditControlRequest.msisdn, ccrContext.sessionId, ccrContext.creditControlRequest.ccRequestNumber?.integer32) removeFromSessionMap(ccrContext) updateBlockedList(answer, ccrContext.creditControlRequest) if (!ccrContext.skipAnswer) { diff --git a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/PubSubDataSource.kt b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/PubSubDataSource.kt index ec5c5eafb..f9a21ec12 100644 --- a/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/PubSubDataSource.kt +++ b/ocsgw/src/main/java/org/ostelco/ocsgw/datasource/protobuf/PubSubDataSource.kt @@ -69,6 +69,7 @@ class PubSubDataSource( val creditControlRequestInfo = protobufDataSource.handleRequest(context, ccaTopicId) if (creditControlRequestInfo != null) { + context.sentToOcsTime = System.currentTimeMillis() sendRequest(creditControlRequestInfo) } } @@ -81,7 +82,7 @@ class PubSubDataSource( setupPubSubSubscriber(projectId, ccaSubscriptionId) { message, consumer -> val ccaInfo = CreditControlAnswerInfo.parseFrom(message) if (ccaInfo.resultCode != ResultCode.UNKNOWN) { - logger.info("Pubsub received CreditControlAnswer for msisdn {} sessionId [{}]", ccaInfo.msisdn, ccaInfo.requestId) + logger.info("Pubsub received CreditControlAnswer for msisdn {} sessionId [{}] request number [{}]", ccaInfo.msisdn, ccaInfo.requestId, ccaInfo.requestNumber) protobufDataSource.handleCcrAnswer(ccaInfo) } consumer.ack() @@ -131,7 +132,9 @@ class PubSubDataSource( override fun onSuccess(messageId: String) { // Once published, returns server-assigned message ids (unique within the topic) - // logger.debug("Submitted message with request-id: {} successfully", messageId) + if (creditControlRequestInfo.type != CreditControlRequestType.NONE) { + logger.debug("Submitted CCR on pubsub for session[{}] request number [{}] successfully", creditControlRequestInfo.requestId, creditControlRequestInfo.requestNumber) + } } }, singleThreadScheduledExecutor) } From f5ead5af74099cef17e81aadf559477a4bbc96cf Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Tue, 3 Dec 2019 14:57:16 +0100 Subject: [PATCH 54/56] Fix and update error messages for Stripe 'card' errors Now logs card failures as 'info' messages. --- .../customer/endpoint/store/SubscriberDAOImpl.kt | 13 +++++++++---- .../org/ostelco/prime/storage/graph/Neo4jStore.kt | 6 +++--- .../prime/paymentprocessor/subscribers/Reporter.kt | 2 +- .../kotlin/org/ostelco/prime/apierror/ApiError.kt | 2 +- .../org/ostelco/prime/apierror/ApiErrorCodes.kt | 1 + 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt index d140d0537..6bd693eb0 100644 --- a/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt +++ b/customer-endpoint/src/main/kotlin/org/ostelco/prime/customer/endpoint/store/SubscriberDAOImpl.kt @@ -2,6 +2,7 @@ package org.ostelco.prime.customer.endpoint.store import arrow.core.Either import arrow.core.flatMap +import arrow.core.left import org.ostelco.prime.activation.Activation import org.ostelco.prime.apierror.ApiError import org.ostelco.prime.apierror.ApiErrorCode @@ -29,6 +30,7 @@ import org.ostelco.prime.model.Subscription import org.ostelco.prime.model.withSimProfileStatusAsInstalled import org.ostelco.prime.module.getResource import org.ostelco.prime.paymentprocessor.PaymentProcessor +import org.ostelco.prime.paymentprocessor.core.CardError import org.ostelco.prime.paymentprocessor.core.PlanAlredyPurchasedError import org.ostelco.prime.paymentprocessor.core.ProductInfo import org.ostelco.prime.paymentprocessor.core.SourceDetailsInfo @@ -304,12 +306,15 @@ class SubscriberDAOImpl : SubscriberDAO { saveCard).fold( { paymentError -> when (paymentError) { - is PlanAlredyPurchasedError -> Either.left(mapPaymentErrorToApiError("Already subscribed to plan. ", + is CardError -> mapPaymentErrorToApiError("Payment source rejected. ", + ApiErrorCode.PAYMENT_SOURCE_REJECTED, + paymentError).left() + is PlanAlredyPurchasedError -> mapPaymentErrorToApiError("Already subscribed to plan. ", ApiErrorCode.ALREADY_SUBSCRIBED_TO_PLAN, - paymentError)) - else -> Either.left(mapPaymentErrorToApiError("Failed to purchase product. ", + paymentError).left() + else -> mapPaymentErrorToApiError("Failed to purchase product. ", ApiErrorCode.FAILED_TO_PURCHASE_PRODUCT, - paymentError)) + paymentError).left() } // if no error, check if this was a topup of empty account, in that case send activate }, { productInfo -> diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 4613b3483..e8aa8845f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1567,8 +1567,8 @@ object Neo4jStoreSingleton : GraphStore { it }.linkReversalActionToTransaction(transaction) { paymentProcessor.removeInvoice(it.id) - logger.warn(NOTIFY_OPS_MARKER, """ - Failed to pay invoice for customer ${customer.id}, invoice-id: ${it.id}. + logger.info(NOTIFY_OPS_MARKER, """ + Payment of invoice ${it.id} failed for customer ${customer.id}. Verify that the invoice has been deleted or voided in Stripe dashboard. """.trimIndent()) }.bind() @@ -1576,7 +1576,7 @@ object Neo4jStoreSingleton : GraphStore { /* Force immediate payment of the invoice. */ val (invoicePaymentInfo) = paymentProcessor.payInvoice(invoice.id) .mapLeft { - logger.warn("Payment of invoice ${invoice.id} failed for customer ${customer.id}.") + logger.info("Charging for invoice ${invoice.id} for product $sku failed for customer ${customer.id}.") /* Adds failed purchase to customer history. */ AuditLog.warn(customerId = customer.id, message = "Failed to complete purchase of product $sku for ${formatMoney(price)} " + diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt index 4f8a86dde..247a78550 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/subscribers/Reporter.kt @@ -205,7 +205,7 @@ object Reporter { event) ) event.type == "payment_intent.payment_failed" -> logger.warn( - format("Failed to create payment ${intent.id} for ${currency(intent.amount, intent.currency)}", + format("Creating payment ${intent.id} of ${currency(intent.amount, intent.currency)} failed", event) ) event.type == "payment_intent.succeeded" -> logger.debug( diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt index 708681923..7e6900655 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt @@ -34,7 +34,7 @@ object ApiErrorMapper { /* Log level depends on the type of payment error. */ fun mapPaymentErrorToApiError(description: String, errorCode: ApiErrorCode, paymentError: PaymentError): ApiError { if (paymentError is org.ostelco.prime.paymentprocessor.core.CardError) { - logger.warn("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) + logger.info("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) } else { logger.error("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt index 42835ebb1..4739d6da8 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt @@ -13,6 +13,7 @@ enum class ApiErrorCode { FAILED_TO_SET_DEFAULT_PAYMENT_SOURCE, FAILED_TO_FETCH_PAYMENT_SOURCES_LIST, FAILED_TO_REMOVE_PAYMENT_SOURCE, + PAYMENT_SOURCE_REJECTED, // payment monitor FAILED_TO_CHECK_API_VERSION, From 9a6c3edddcb7d386150308df8c28be4ddae65ae9 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Wed, 4 Dec 2019 14:06:13 +0100 Subject: [PATCH 55/56] Remove 'invoice intents' events from list of subscribed to Stripe events Will reduce the log "noise" a bit as payment failures will not cause a failed "invoice intent" Stripe event to be sent. --- payment-processor/script/update-webhook.sh | 3 --- prime/infra/stripe-monitor-task.yaml | 3 --- 2 files changed, 6 deletions(-) diff --git a/payment-processor/script/update-webhook.sh b/payment-processor/script/update-webhook.sh index 6d026b6ca..ff4a1222e 100755 --- a/payment-processor/script/update-webhook.sh +++ b/payment-processor/script/update-webhook.sh @@ -35,9 +35,6 @@ update_event_list() { -d enabled_events[]="invoice.upcoming" \ -d enabled_events[]="invoice.updated" \ -d enabled_events[]="invoice.voided" \ - -d enabled_events[]="payment_intent.created" \ - -d enabled_events[]="payment_intent.payment_failed" \ - -d enabled_events[]="payment_intent.succeeded" \ -d enabled_events[]="payout.failed" \ -d enabled_events[]="payout.paid" } diff --git a/prime/infra/stripe-monitor-task.yaml b/prime/infra/stripe-monitor-task.yaml index 7cf058489..c5a66b77c 100644 --- a/prime/infra/stripe-monitor-task.yaml +++ b/prime/infra/stripe-monitor-task.yaml @@ -39,9 +39,6 @@ spec: "invoice.upcoming", "invoice.updated", "invoice.voided", - "payment_intent.created", - "payment_intent.payment_failed", - "payment_intent.succeeded", "payout.failed", "payout.paid"]' - name: INTERVAL From 2c74ecc8a4365067151a99cb53e08a18cbc7513e Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Wed, 4 Dec 2019 14:53:23 +0100 Subject: [PATCH 56/56] Set timestamps rows to current time on schema update --- sim-administration/postgres/add-timestamps.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sim-administration/postgres/add-timestamps.sql b/sim-administration/postgres/add-timestamps.sql index 9fd05a6cc..a9b86be2d 100644 --- a/sim-administration/postgres/add-timestamps.sql +++ b/sim-administration/postgres/add-timestamps.sql @@ -9,6 +9,7 @@ -- alter table sim_entries add column tsHlrState timestamp; alter table sim_entries alter column tsHlrState set default now(); +update sim_entries set tsHlrState = now(); create or replace function hlrState_changed() returns trigger as $$ begin @@ -28,6 +29,7 @@ execute procedure hlrState_changed(); -- alter table sim_entries add column tsSmdpPlusState timestamp; alter table sim_entries alter column tsSmdpPlusState set default now(); +update sim_entries set tsSmdpPlusState = now(); create or replace function smdpPlusState_changed() returns trigger as $$ begin @@ -47,6 +49,7 @@ execute procedure smdpPlusState_changed(); -- alter table sim_entries add column tsProvisionState timestamp; alter table sim_entries alter column tsProvisionState set default now(); +update sim_entries set tsProvisionState = now(); create or replace function provisionState_changed() returns trigger as $$ begin