Merge pull request #78 from ostelco/develop

Merge Develop to Master

.travis.yml

@@ -6,7 +6,9 @@ cache:
 
 before_install:
   - sudo apt-get install jq
-  - wget -O ~/codacy-coverage-reporter-assembly-latest.jar $(curl https://api.github.com/repos/codacy/codacy-coverage-reporter/releases/latest | jq -r .assets[0].browser_download_url)
+  # The curl command is not always working. Kept original command in comment incase codacy updates
+  #- wget -O ~/codacy-coverage-reporter-assembly-latest.jar $(curl https://api.github.com/repos/codacy/codacy-coverage-reporter/releases/latest | jq -r .assets[0].browser_download_url)
+  - wget -O ~/codacy-coverage-reporter-assembly-latest.jar https://github.com/codacy/codacy-coverage-reporter/releases/download/4.0.0/codacy-coverage-reporter-4.0.0-assembly.jar
 
 after_success:
   - java -cp ~/codacy-coverage-reporter-assembly-latest.jar com.codacy.CodacyCoverageReporter -l Java -r prime/build/reports/jacoco/test/jacocoTestReport.xml

analytics/build.gradle

@@ -9,7 +9,7 @@ plugins {
 dependencies {
   implementation project(':ocs-api')
   implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
-  implementation 'com.google.cloud:google-cloud-pubsub:0.43.0-beta'
+  implementation 'com.google.cloud:google-cloud-pubsub:0.44.0-beta'
   implementation 'com.google.cloud.dataflow:google-cloud-dataflow-java-sdk-all:2.4.0'
   runtime 'org.apache.beam:beam-runners-google-cloud-dataflow-java:2.4.0'
   implementation 'ch.qos.logback:logback-classic:1.2.3'

auth-server/build.gradle

@@ -9,7 +9,7 @@ plugins {
 dependencies {
   implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlinVersion"
   implementation "io.dropwizard:dropwizard-core:$dropwizardVersion"
-  implementation 'com.google.firebase:firebase-admin:5.9.0'
+  implementation 'com.google.firebase:firebase-admin:5.10.0'
   testImplementation "io.dropwizard:dropwizard-testing:$dropwizardVersion"
   testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlinVersion"
   testRuntime 'org.hamcrest:hamcrest-all:1.3'

docker-compose.override.yaml

@@ -25,17 +25,17 @@ services:
       - "ocsgw"
     command: ["./wait_for_ocsgw.sh"]
 
-  pseudonym-server:
-    container_name: pseudonym-server
-    build: pseudonym-server
-    ports:
-      - "8090:8080"
-    environment:
-      - DATASTORE_EMULATOR_HOST=localhost:9090
-      - DATASTORE_PROJECT_ID=pantel-2decb
-      - PUBSUB_EMULATOR_HOST=localhost:9080
-      - PUBSUB_PROJECT_ID=pantel-2decb
-    command: ["/bin/bash", "./wait_for_emulators.sh"]
+#  pseudonym-server:
+#    container_name: pseudonym-server
+#    build: pseudonym-server
+#    ports:
+#      - "8090:8080"
+#    environment:
+#      - DATASTORE_EMULATOR_HOST=localhost:9090
+#      - DATASTORE_PROJECT_ID=pantel-2decb
+#      - PUBSUB_EMULATOR_HOST=localhost:9080
+#      - PUBSUB_PROJECT_ID=pantel-2decb
+#    command: ["/bin/bash", "./wait_for_emulators.sh"]
 
   pubsub-emulator:
     container_name: pubsub-emulator

exporter/Dockerfile

@@ -0,0 +1,24 @@
+FROM ubuntu:17.10
+
+MAINTAINER CSI "[email protected]"
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		curl=7.55.1-1ubuntu2.4 \
+		uuid-runtime=2.30.1-0ubuntu4.1 \
+		lsb-release=9.20160110ubuntu5 \
+		ca-certificates=20170717 \
+    && echo "deb http://packages.cloud.google.com/apt cloud-sdk-artful main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list \
+    && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+    && apt-get update && apt-get install -y --no-install-recommends \
+        google-cloud-sdk=198.0.0-0 \
+	&& rm -rf /var/lib/apt/lists/*
+
+COPY script/idle.sh /idle.sh
+COPY script/export_data.sh /export_data.sh
+COPY script/delete_export_data.sh /delete_export_data.sh
+
+RUN chmod +x /idle.sh
+RUN chmod +x /export_data.sh
+RUN chmod +x /delete_export_data.sh
+
+CMD ["/idle.sh"]

exporter/README.md

@@ -0,0 +1,50 @@
+# Exporter
+
+This contains a set of scripts to generate the data for analayis. The export script
+`export_data.sh` creates a new big query table with a new uuid which maps the pseudonyms to
+msisdn. This table is then used to join the hourly data consumption to create a new
+pseudonymised version of hourly consumption. The output doesn't contain msisdns. The output
+table is then exported as a csv file in google cloud storage. There is also a script
+`delete_export_data.sh` to delete these tables & files.
+
+Currently this is deployed as a pod. The commands can be run by logging into the container.
+
+Tables created:
+1) exported_pseudonyms.<exportId> : The pseudonyms table, which can be used to reverse lookup msisdns
+2) exported_data_consumption.<exportId> : The output hourly consumption table, used to export data to csv
+
+How to deploy/use this in kubernetes cluster
+
+```
+#PROJECT_ID=pantel-2decb
+export PROJECT_ID="$(gcloud config get-value project -q)"
+
+# Create cluster
+gcloud container clusters create private-cluster --scopes=default,bigquery,datastore,pubsub,sql,storage-rw --num-nodes=3
+
+# Get authentication credentials for the cluster
+gcloud container clusters get-credentials private-cluster
+
+# Build the Docker image (In the folder with Dockerfile)
+docker build -t gcr.io/${PROJECT_ID}/exporter:v1 .
+
+# Push to the registry
+gcloud docker -- push gcr.io/${PROJECT_ID}/exporter:v1
+
+# Apply the deployment
+kubectl apply -f ./exporter.yaml
+
+# Details of the deployment
+kubectl describe deployment exporter
+kubectl get pods
+
+# Login to the pod
+kubectl exec -it <exporter pod name> -- /bin/bash
+
+# Run exporter from the above shell
+/export_data.sh
+
+# Delete deployment
+kubectl delete  deployment exporter
+
+```

exporter/exporter.yaml

@@ -0,0 +1,20 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: exporter
+  labels:
+    app: exporter
+    tier: backend
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: exporter
+        tier: backend
+    spec:
+      containers:
+      - name: exporter
+        image: gcr.io/pantel-2decb/exporter:v2.1
+        ports:
+        - containerPort: 8080

exporter/script/delete_export_data.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#set -x
+
+exportId=$1
+if [ -z "$1" ]; then
+    echo "Specify the id of the export operation you want to delete"
+    exit
+fi
+exportId=${exportId//-}
+exportId=${exportId,,}
+projectId=pantel-2decb
+
+pseudonymsTable=exported_pseudonyms.$exportId
+dataConsumptionTable=exported_data_consumption.$exportId
+csvfile=$projectId-dataconsumption-export/$exportId.csv
+
+echo "Cleaning all data for export $exportId"
+echo "Deleting Table $pseudonymsTable"
+bq rm -f -t $pseudonymsTable
+echo "Deleting Table $dataConsumptionTable"
+bq rm -f -t $dataConsumptionTable
+echo "Deleting csv gs://$csvfile"
+gsutil rm gs://$csvfile
+
+echo "Finished cleanup for the export $exportId"

exporter/script/export_data.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+#set -x
+
+exportId=$1
+if [ -z "$1" ]; then
+    exportId=$(uuidgen)
+fi
+exportId=${exportId//-}
+exportId=${exportId,,}
+projectId=pantel-2decb
+
+pseudonymsTable=$projectId.exported_pseudonyms.$exportId
+hourlyConsumptionTable=$projectId.data_consumption.hourly_consumption
+dataConsumptionTable=exported_data_consumption.$exportId
+csvfile=$projectId-dataconsumption-export/$exportId.csv
+
+# Generate the pseudonym tables for this export
+echo "Starting export job for $exportId"
+pseudonymHost="pseudonym-server-service.default.svc.cluster.local"
+startUrl="http://$pseudonymHost/pseudonym/export/$exportId"
+httpStatus=$(curl -sL -w "%{http_code}" $startUrl -o /dev/null)
+
+if [[ $httpStatus != 200 ]]; then
+  echo "Failed to start the pseudonym table creation: $httpStatus"
+  exit
+fi
+
+# Wait for the table creation to finish
+echo "Waiting to finish table export for $exportId"
+sleep 30 &
+queryUrl="http://$pseudonymHost/pseudonym/exportstatus/$exportId"
+httpStatus=$(curl -sL -w "%{http_code}" $queryUrl -o /dev/null)
+if [[ $httpStatus != 200 ]]; then
+  echo "Failed to query the table creation: $httpStatus"
+  exit
+fi
+
+jsonResult=RUNNING
+while [[ $jsonResult = RUNNING || $jsonResult = INITIAL ]]; do
+  jsonResult=$(curl -X GET $queryUrl  2> /dev/null | sed -n -e 's/.*"status"://p'| cut -d \" -f 2)
+done
+if [[ $jsonResult != FINISHED ]]; then
+  echo "Table creation failed $(curl -X GET $queryUrl  2> /dev/null)"
+  exit
+fi
+echo "Created Table $pseudonymsTable"
+
+
+echo "Creating table $dataConsumptionTable"
+# SQL for joining pseudonym & hourly consumption tables.
+read -r -d '' sqlForJoin << EOM
+SELECT
+   hc.bytes, ps.msisdnid, hc.timestamp
+FROM
+   \`$hourlyConsumptionTable\` as hc
+JOIN
+  \`$pseudonymsTable\` as ps
+ON  ps.msisdn = hc.msisdn
+EOM
+# Run the query using bq & dump results to the new table
+bq --location=EU --format=none query --destination_table $dataConsumptionTable --replace --use_legacy_sql=false $sqlForJoin
+echo "Created table $dataConsumptionTable"
+
+echo "Exporting data to csv $csvfile"
+bq --location=EU extract --destination_format=CSV $dataConsumptionTable gs://$csvfile
+echo "Exported data to gs://$csvfile"

exporter/script/idle.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+echo "This is a idle script (infinite loop) to keep container running."
+echo "Please replace this script."
+
+cleanup ()
+{
+  kill -s SIGTERM $!
+  exit 0
+}
+
+trap cleanup SIGINT SIGTERM
+
+while [ 1 ]
+do
+  sleep 60 &
+  wait $!
+done

ocsgw/src/main/java/org/ostelco/ocsgw/OcsServer.java

@@ -60,15 +60,19 @@ public void sendReAuthRequest(final SessionContext sessionContext) {
             if (ccaSession != null && ccaSession.isValid()) {
                 // ToDo: Not sure why there are multiple sessions for one session Id.
                 for (Session session : ccaSession.getSessions()) {
-                    Request request = session.createRequest(258,
-                            ApplicationId.createByAuthAppId(4L),
-                            sessionContext.getOriginRealm(),
-                            sessionContext.getOriginHost()
-                    );
-                    AvpSet avps = request.getAvps();
-                    avps.addAvp(Avp.RE_AUTH_REQUEST_TYPE, ReAuthRequestType.AUTHORIZE_ONLY.ordinal(), true, false);
-                    ReAuthRequest reAuthRequest = new ReAuthRequestImpl(request);
-                    ccaSession.sendReAuthRequest(reAuthRequest);
+                    if (session.isValid()) {
+                        Request request = session.createRequest(258,
+                                ApplicationId.createByAuthAppId(4L),
+                                sessionContext.getOriginRealm(),
+                                sessionContext.getOriginHost()
+                        );
+                        AvpSet avps = request.getAvps();
+                        avps.addAvp(Avp.RE_AUTH_REQUEST_TYPE, ReAuthRequestType.AUTHORIZE_ONLY.ordinal(), true, false);
+                        ReAuthRequest reAuthRequest = new ReAuthRequestImpl(request);
+                        ccaSession.sendReAuthRequest(reAuthRequest);
+                    } else {
+                        LOG.info("Invalid session");
+                    }
                 }
             } else {
                 LOG.info("No session with ID {}", sessionContext.getSessionId());

ostelco-lib/build.gradle

@@ -7,7 +7,7 @@ plugins {
 dependencies {
     implementation "io.dropwizard:dropwizard-core:$dropwizardVersion"
     implementation "io.dropwizard:dropwizard-auth:$dropwizardVersion"
-    implementation 'com.google.firebase:firebase-admin:5.9.0'
+    implementation 'com.google.firebase:firebase-admin:5.10.0'
     implementation 'com.lmax:disruptor:3.4.2'
     implementation 'com.google.guava:guava:24.1-jre'
 

prime/build.gradle

@@ -4,6 +4,7 @@ plugins {
   id "com.github.johnrengelman.shadow" version "2.0.3"
   id "org.jetbrains.kotlin.jvm" version "1.2.31"
   id "idea"
+  id "project-report"
 }
 
 sourceSets {
@@ -24,8 +25,8 @@ dependencies {
   implementation project(':ocs-api')
   implementation project(':ostelco-lib')
   implementation "io.dropwizard:dropwizard-core:$dropwizardVersion"
-  implementation 'com.google.firebase:firebase-admin:5.9.0'
-  implementation 'com.google.cloud:google-cloud-pubsub:0.43.0-beta'
+  implementation 'com.google.firebase:firebase-admin:5.10.0'
+  implementation 'com.google.cloud:google-cloud-pubsub:0.44.0-beta'
   implementation 'com.lmax:disruptor:3.4.2'
   implementation 'com.google.guava:guava:24.1-jre'
 

prime/src/integration-tests/kotlin/org/ostelco/prime/firebase/FbPurchaseEventRoundtripTest.kt

@@ -96,8 +96,7 @@ class FbPurchaseEventRoundtripTest {
             }
         })
 
-        val req = PurchaseRequestImpl(DATA_TOPUP_3GB, EventProcessorTest.PAYMENT_TOKEN)
-        req.setMsisdn(EPHERMERAL_MSISDN)
+        val req = PurchaseRequestImpl(DATA_TOPUP_3GB, EventProcessorTest.PAYMENT_TOKEN, EPHERMERAL_MSISDN)
 
         Assert.assertNotEquals(null, storage!!.getSubscriberFromMsisdn(EPHERMERAL_MSISDN))
 

prime/src/integration-tests/kotlin/org/ostelco/prime/firebase/FbStorageTest.kt

@@ -115,7 +115,7 @@ class FbStorageTest {
                     }
                 })
 
-        val cr = PurchaseRequestImpl(DATA_TOPUP_3GB, PAYMENT_TOKEN)
+        val cr = PurchaseRequestImpl(DATA_TOPUP_3GB, PAYMENT_TOKEN, EPHERMERAL_MSISDN)
         val id = fbStorage!!.injectPurchaseRequest(cr)
         val id2 = fbStorage!!.injectPurchaseRequest(cr)
         prids!!.add(id)

prime/src/main/kotlin/org/ostelco/prime/firebase/AbstractChildEventListener.kt

@@ -10,19 +10,19 @@ import com.google.firebase.database.DatabaseError
  */
 abstract class AbstractChildEventListener : ChildEventListener {
 
-    override fun onChildAdded(dataSnapshot: DataSnapshot, prevChildKey: String) {
+    override fun onChildAdded(dataSnapshot: DataSnapshot, prevChildKey: String?) {
         // Intended to be overridden in by subclass. Default is to do nothing.
     }
 
-    override fun onChildChanged(snapshot: DataSnapshot, previousChildName: String) {
+    override fun onChildChanged(snapshot: DataSnapshot, previousChildName: String?) {
         // Intended to be overridden in by subclass. Default is to do nothing.
     }
 
     override fun onChildRemoved(snapshot: DataSnapshot) {
         // Intended to be overridden in by subclass. Default is to do nothing.
     }
 
-    override fun onChildMoved(snapshot: DataSnapshot, previousChildName: String) {
+    override fun onChildMoved(snapshot: DataSnapshot, previousChildName: String?) {
         // Intended to be overridden in by subclass. Default is to do nothing.
     }