From e950e1d1e2db87a2b6e2259bb9915766b90f8d1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 24 Oct 2019 23:33:31 +0200 Subject: [PATCH 001/309] Adding a few comments about what we want to do. --- .../sim-batch-management/sim-batch-db.go | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 sim-administration/sim-batch-management/sim-batch-db.go diff --git a/sim-administration/sim-batch-management/sim-batch-db.go b/sim-administration/sim-batch-management/sim-batch-db.go new file mode 100644 index 000000000..4d6fa0822 --- /dev/null +++ b/sim-administration/sim-batch-management/sim-batch-db.go @@ -0,0 +1,82 @@ +package main + +// A simple "finger exercise" to get to know sqlite in golang. Purely +// exploratory, no specific goal in sight. +// NOTE: It will asap be rewritten to manage a persistent store of +// batch parameters, and also be able to facilitate a workflow +// sheparding the orders from inception to deployment. +// Todo: +// * Introduce common model package for existing structs. +// * Extend model with metedata for persistence. +// * Write DAO based on this file, but make it properly +// unit tested using in-memory database. +// * Figure out how to keep config somewhere safe without +// having to type too much. + +import ( + "database/sql" + "fmt" + "github.com/jmoiron/sqlx" + "strconv" + + _ "github.com/mattn/go-sqlite3" +) + +// Sqlx https://jmoiron.github.io/sqlx/ +// Person represents a person. +type Person struct { + ID int `db:"id" json:"id"` + Firstname string `db:"firstname" json:"firstname"` + Lastname string `db:"lastname" json:"lastname"` +} + +func main() { + + // Get a reference to the database, create a table if it + // doesn't exist already. + + var db *sqlx.DB + + // exactly the same as the built-in + db, err := sqlx.Open("sqlite3", "./nraboy.db") + + database, err := sql.Open("sqlite3", "./nraboy.db") + if err != nil { + fmt.Errorf("open sql", err) + } + + // + statement, _ := database.Prepare("CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT)") + _, err = statement.Exec() + if err != nil { + fmt.Errorf("Failed to create table: ", err) + } + + // Insert a row. + statement, _ = database.Prepare("INSERT INTO people (firstname, lastname) VALUES (?, ?)") + _, err = statement.Exec("Nic", "Raboy") + if err != nil { + fmt.Errorf("Failed to insert row: ", err) + } + + // Query all the rows. + rows, _ := database.Query("SELECT id, firstname, lastname FROM people") + var id int + var firstname string + var lastname string + for rows.Next() { + rows.Scan(&id, &firstname, &lastname) + if err != nil { + fmt.Errorf("Failed to scan row: ", err) + } + fmt.Println(strconv.Itoa(id) + ": " + firstname + " " + lastname) + } + + fmt.Print("Foo->") + rowz, err := db.Queryx("SELECT * FROM people") + for rowz.Next() { + var p Person + err = rowz.StructScan(&p) + fmt.Println("The p = ", p) + } +} From f42394e24f6f5e008c227890f7f056cd2e346ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 25 Oct 2019 15:57:28 +0200 Subject: [PATCH 002/309] Make go model package, and refactor rest of program to compile using it --- go.mod | 2 + go.sum | 7 ++ .../sim-batch-management/model/model.go | 54 +++++++++++ .../outfile_to_hss_input_converter_lib.go | 70 +++++-------- .../sim-batch-management/sim-batch-db.go | 2 +- .../upload-sim-batch-lib-test.go | 14 +-- .../uploadtoprime/upload-sim-batch-lib.go | 97 ++++++++----------- 7 files changed, 136 insertions(+), 110 deletions(-) create mode 100644 sim-administration/sim-batch-management/model/model.go diff --git a/go.mod b/go.mod index 8ea2428ce..eb89dbb49 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.13 require ( github.com/google/go-cmp v0.3.1 // indirect + github.com/jmoiron/sqlx v1.2.0 + github.com/mattn/go-sqlite3 v1.11.0 github.com/pkg/errors v0.8.1 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 802d8ce53..62b2b7ef3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,12 @@ +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go new file mode 100644 index 000000000..6a92bbcf6 --- /dev/null +++ b/sim-administration/sim-batch-management/model/model.go @@ -0,0 +1,54 @@ +package model + +// TODO: Put all records used to manage workflows in this +// package, then build a DAO interface a "store" package + +type OutputBatch struct { + ProfileType string + Url string + Length int + FirstMsisdn int + MsisdnIncrement int + FirstIccid int + IccidIncrement int + FirstImsi int + ImsiIncrement int +} + +type InputBatch struct { + Customer string + ProfileType string + OrderDate string + BatchNo string + Quantity int + FirstIccid int + FirstImsi int +} + + +type OutputFileRecord struct { + Filename string + InputVariables map[string]string + HeaderDescription map[string]string + Entries []SimEntry + // TODO: As it is today, the noOfEntries is just the number of Entries, + // but I may want to change that to be the declared number of Entries, + // and then later, dynamically, read in the individual Entries + // in a channel that is just piped to the goroutine that writes + // them to file, and fails if the number of declared Entries + // differs from the actual number of Entries. .... but that is + // for another day. + NoOfEntries int + OutputFileName string +} + + +type SimEntry struct { + RawIccid string + IccidWithChecksum string + IccidWithoutChecksum string + Imsi string + Ki string + OutputFileName string +} + diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go index 6e53238ff..ea585148c 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "log" "os" "regexp" @@ -12,26 +13,6 @@ import ( "strings" ) -/// -/// Data structures -/// - -type OutputFileRecord struct { - Filename string - inputVariables map[string]string - headerDescription map[string]string - Entries []SimEntry - // TODO: As it is today, the noOfEntries is just the number of Entries, - // but I may want to change that to be the declared number of Entries, - // and then later, dynamically, read in the individual Entries - // in a channel that is just piped to the goroutine that writes - // them to file, and fails if the number of declared Entries - // differs from the actual number of Entries. .... but that is - // for another day. - noOfEntries int - OutputFileName string -} - const ( INITIAL = "initial" HEADER_DESCRIPTION = "header_description" @@ -40,21 +21,6 @@ const ( UNKNOWN_HEADER = "unknown" ) -type SimEntry struct { - rawIccid string - iccidWithChecksum string - iccidWithoutChecksum string - imsi string - ki string - outputFileName string -} - -type ParserState struct { - currentState string - inputVariables map[string]string - headerDescription map[string]string - entries []SimEntry -} /// /// Functions @@ -87,7 +53,15 @@ func ParseLineIntoKeyValueMap(line string, theMap map[string]string) { theMap[key] = value } -func ReadOutputFile(filename string) OutputFileRecord { + +type ParserState struct { + currentState string + inputVariables map[string]string + headerDescription map[string]string + entries []model.SimEntry +} + +func ReadOutputFile(filename string) model.OutputFileRecord { _, err := os.Stat(filename) @@ -164,12 +138,12 @@ func ReadOutputFile(filename string) OutputFileRecord { var iccidWithoutChecksum = loltelutils.TrimSuffix(iccidWithChecksum, 1) // TODO: Enable this!! checkICCIDSyntax(iccidWithChecksum) - entry := SimEntry{ - rawIccid: rawIccid, - iccidWithChecksum: iccidWithChecksum, - iccidWithoutChecksum: iccidWithoutChecksum, - imsi: imsi, - ki: ki} + entry := model.SimEntry{ + RawIccid: rawIccid, + IccidWithChecksum: iccidWithChecksum, + IccidWithoutChecksum: iccidWithoutChecksum, + Imsi: imsi, + Ki: ki} state.entries = append(state.entries, entry) case UNKNOWN_HEADER: @@ -197,12 +171,12 @@ func ReadOutputFile(filename string) OutputFileRecord { countedNoOfEntries) } - result := OutputFileRecord{ + result := model.OutputFileRecord{ Filename: filename, - inputVariables: state.inputVariables, - headerDescription: state.headerDescription, + InputVariables: state.inputVariables, + HeaderDescription: state.headerDescription, Entries: state.entries, - noOfEntries: declaredNoOfEntities, + NoOfEntries: declaredNoOfEntities, OutputFileName: getOutputFileName(state), } @@ -274,7 +248,7 @@ func fileExists(filename string) bool { } // TODO: Consider rewriting using https://golang.org/pkg/encoding/csv/ -func WriteHssCsvFile(filename string, entries []SimEntry) error { +func WriteHssCsvFile(filename string, entries []model.SimEntry) error { if fileExists(filename) { log.Fatal("Output file already exists. '", filename, "'.") @@ -292,7 +266,7 @@ func WriteHssCsvFile(filename string, entries []SimEntry) error { max := 0 for i, entry := range entries { - s := fmt.Sprintf("%s, %s, %s\n", entry.iccidWithChecksum, entry.imsi, entry.ki) + s := fmt.Sprintf("%s, %s, %s\n", entry.IccidWithChecksum, entry.Imsi, entry.Ki) _, err = f.WriteString(s) if err != nil { log.Fatal("Couldn't write to hss csv file '", filename, "': ", err) diff --git a/sim-administration/sim-batch-management/sim-batch-db.go b/sim-administration/sim-batch-management/sim-batch-db.go index 4d6fa0822..1b8ac630c 100644 --- a/sim-administration/sim-batch-management/sim-batch-db.go +++ b/sim-administration/sim-batch-management/sim-batch-db.go @@ -30,7 +30,7 @@ type Person struct { Lastname string `db:"lastname" json:"lastname"` } -func main() { +func main_not() { // Get a reference to the database, create a table if it // doesn't exist already. diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go index e1d249b45..96bbc1967 100644 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go @@ -9,13 +9,13 @@ import ( func TestParseInputFileGeneratorCommmandline(t *testing.T) { parsedBatch := ParseInputFileGeneratorCommmandline() - assert.Equal(t, "Footel", parsedBatch.customer) - assert.Equal(t, "BAR_FOOTEL_STD", parsedBatch.profileType) - assert.Equal(t, "20191007", parsedBatch.orderDate) - assert.Equal(t, "2019100701", parsedBatch.batchNo) - assert.Equal(t, 10, parsedBatch.quantity) - assert.Equal(t, 894700000000002214, parsedBatch.firstIccid) - assert.Equal(t, 242017100012213, parsedBatch.firstImsi) + assert.Equal(t, "Footel", parsedBatch.Customer) + assert.Equal(t, "BAR_FOOTEL_STD", parsedBatch.ProfileType) + assert.Equal(t, "20191007", parsedBatch.OrderDate) + assert.Equal(t, "2019100701", parsedBatch.BatchNo) + assert.Equal(t, 10, parsedBatch.Quantity) + assert.Equal(t, 894700000000002214, parsedBatch.FirstIccid) + assert.Equal(t, 242017100012213, parsedBatch.FirstImsi) } // TODO: Make a test that checks that the correct number of things are made, diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index bdb069639..40cc3f93c 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -14,6 +14,7 @@ import ( "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "log" "net/url" "regexp" @@ -66,23 +67,23 @@ func LuhnChecksum(number int) int { return generateControlDigit(strconv.Itoa(number)) } -func GenerateCsvPayload(batch OutputBatch) string { +func GenerateCsvPayload(batch model.OutputBatch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") - var iccidWithoutLuhnChecksum = batch.firstIccid + var iccidWithoutLuhnChecksum = batch.FirstIccid - var imsi = batch.firstImsi - var msisdn = batch.firstMsisdn - for i := 0; i < batch.length; i++ { + var imsi = batch.FirstImsi + var msisdn = batch.FirstMsisdn + for i := 0; i < batch.Length; i++ { iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, LuhnChecksum(iccidWithoutLuhnChecksum)) - line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.profileType) + line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.ProfileType) sb.WriteString(line) - iccidWithoutLuhnChecksum += batch.iccidIncrement - imsi += batch.imsiIncrement - msisdn += batch.msisdnIncrement + iccidWithoutLuhnChecksum += batch.IccidIncrement + imsi += batch.ImsiIncrement + msisdn += batch.MsisdnIncrement } return sb.String() @@ -146,23 +147,13 @@ func checkProfileType(name string, potentialProfileName string) { } } -type OutputBatch struct { - profileType string - Url string - length int - firstMsisdn int - msisdnIncrement int - firstIccid int - iccidIncrement int - firstImsi int - imsiIncrement int -} + func IccidWithoutLuhnChecksum(s string) string { return loltelutils.TrimSuffix(s, 1) } -func ParseUploadFileGeneratorCommmandline() OutputBatch { +func ParseUploadFileGeneratorCommmandline() model.OutputBatch { // // Set up command line parsing @@ -179,8 +170,8 @@ func ParseUploadFileGeneratorCommmandline() OutputBatch { lastMsisdn := flag.String("last-msisdn", "Not a valid MSISDN", "Last MSISDN in batch") profileType := flag.String("profile-type", "Not a valid sim profile type", "SIM profile type") batchLengthString := flag.String( - "batch-quantity", - "Not a valid batch-quantity, must be an integer", + "batch-Quantity", + "Not a valid batch-Quantity, must be an integer", "Number of sim cards in batch") // XXX Legal values are Loltel and M1 at this time, how to configure that @@ -221,11 +212,11 @@ func ParseUploadFileGeneratorCommmandline() OutputBatch { batchLength, err := strconv.Atoi(*batchLengthString) if err != nil { - log.Fatalf("Not a valid batch quantity string '%s'.\n", *batchLengthString) + log.Fatalf("Not a valid batch Quantity string '%s'.\n", *batchLengthString) } if batchLength <= 0 { - log.Fatalf("OutputBatch quantity must be positive, but was '%d'", batchLength) + log.Fatalf("OutputBatch Quantity must be positive, but was '%d'", batchLength) } uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", @@ -242,7 +233,7 @@ func ParseUploadFileGeneratorCommmandline() OutputBatch { log.Println("firstmsisdn = ", *firstMsisdn) log.Println("lastmsisdn = ", *lastMsisdn) - log.Println("msisdnIncrement = ", msisdnIncrement) + log.Println("MsisdnIncrement = ", msisdnIncrement) var firstMsisdnInt, _ = strconv.Atoi(*firstMsisdn) var lastMsisdnInt, _ = strconv.Atoi(*lastMsisdn) @@ -275,16 +266,16 @@ func ParseUploadFileGeneratorCommmandline() OutputBatch { } // Return a correctly parsed batch - return OutputBatch{ - profileType: *profileType, + return model.OutputBatch{ + ProfileType: *profileType, Url: uploadUrl, - length: loltelutils.Abs(iccidlen), - firstIccid: firstIccidInt, - iccidIncrement: loltelutils.Sign(iccidlen), - firstImsi: firstImsiInt, - imsiIncrement: loltelutils.Sign(imsiLen), - firstMsisdn: firstMsisdnInt, - msisdnIncrement: msisdnIncrement, + Length: loltelutils.Abs(iccidlen), + FirstIccid: firstIccidInt, + IccidIncrement: loltelutils.Sign(iccidlen), + FirstImsi: firstImsiInt, + ImsiIncrement: loltelutils.Sign(imsiLen), + FirstMsisdn: firstMsisdnInt, + MsisdnIncrement: msisdnIncrement, } } @@ -292,37 +283,35 @@ func ParseUploadFileGeneratorCommmandline() OutputBatch { /// Input batch management /// -type InputBatch struct { - customer string - profileType string - orderDate string - batchNo string - quantity int - firstIccid int - firstImsi int -} +func ParseInputFileGeneratorCommmandline() model.InputBatch { -func ParseInputFileGeneratorCommmandline() InputBatch { // TODO: This function should be rewritten to parse a string array and send it to flags. // we need to up our Go-Fu before we can make flag.Parse(arguments) work - return InputBatch{customer: "Footel", profileType: "BAR_FOOTEL_STD", orderDate: "20191007", batchNo: "2019100701", quantity: 10, firstIccid: 894700000000002214, firstImsi: 242017100012213} + return model.InputBatch{ + Customer: "Footel", + ProfileType: "BAR_FOOTEL_STD", + OrderDate: "20191007", + BatchNo: "2019100701", + Quantity: 10, + FirstIccid: 894700000000002214, + FirstImsi: 242017100012213} } -func GenerateInputFile(batch InputBatch) string { +func GenerateInputFile(batch model.InputBatch) string { result := "*HEADER DESCRIPTION\n" + "***************************************\n" + - fmt.Sprintf("Customer :%s\n", batch.customer) + - fmt.Sprintf("ProfileType : %s\n", batch.profileType) + - fmt.Sprintf("Order Date : %s\n", batch.orderDate) + - fmt.Sprintf("Batch No : %s\n", batch.batchNo) + - fmt.Sprintf("Quantity : %d\n", batch.quantity) + + fmt.Sprintf("Customer :%s\n", batch.Customer) + + fmt.Sprintf("ProfileType : %s\n", batch.ProfileType) + + fmt.Sprintf("Order Date : %s\n", batch.OrderDate) + + fmt.Sprintf("Batch No : %s\n", batch.BatchNo) + + fmt.Sprintf("Quantity : %d\n", batch.Quantity) + "***************************************\n" + "*INPUT VARIABLES\n" + "***************************************\n" + "var_In:\n" + - fmt.Sprintf(" ICCID: %d\n", batch.firstIccid) + - fmt.Sprintf("IMSI: %d\n", batch.firstImsi) + + fmt.Sprintf(" ICCID: %d\n", batch.FirstIccid) + + fmt.Sprintf("IMSI: %d\n", batch.FirstImsi) + "***************************************\n" + "*OUTPUT VARIABLES\n" + "***************************************\n" + From 18439bde60b83b3db4a2c85801b853e27d6d11ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 28 Oct 2019 15:06:57 +0100 Subject: [PATCH 003/309] Ignore the wrapper shellscript --- sim-administration/sim-batch-management/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/sim-administration/sim-batch-management/.gitignore b/sim-administration/sim-batch-management/.gitignore index c97f963b3..64488d3a0 100644 --- a/sim-administration/sim-batch-management/.gitignore +++ b/sim-administration/sim-batch-management/.gitignore @@ -1 +1,2 @@ *.sh +es2pluswrapper.sh From e41e86d22dbfda5eff556ff20061cd25c67fd7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 28 Oct 2019 15:18:09 +0100 Subject: [PATCH 004/309] Hole in the wall now established, moving on. --- .../sim-batch-management/es2plus_smoketest.go | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 sim-administration/sim-batch-management/es2plus_smoketest.go diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go new file mode 100755 index 000000000..05639065e --- /dev/null +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -0,0 +1,54 @@ +//usr/bin/env go run "$0" "$@"; exit "$?" +package main + + +import ( +"crypto/tls" +"crypto/x509" +"fmt" +"io" +"log" +"flag" +) + +// ES2PLUS_ENDPOINT="https://mconnect-es2-005.staging.oberthur.net:1034"f + +func main() { + + certFilePath := flag.String("cert", "", "Certificate pem file.") + keyFilePath := flag.String("key", "", "Certificate key file.") + hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") + + flag.Parse() + + cert, err := tls.LoadX509KeyPair( + *certFilePath, + *keyFilePath) + if err != nil { + log.Fatalf("server: loadkeys: %s", err) + } + config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} + conn, err := tls.Dial("tcp", *hostport, &config) + if err != nil { + log.Fatalf("client: dial: %s", err) + } + defer conn.Close() + log.Println("client: connected to: ", conn.RemoteAddr()) + state := conn.ConnectionState() + for _, v := range state.PeerCertificates { + fmt.Println("Client: Server public key is:") + fmt.Println(x509.MarshalPKIXPublicKey(v.PublicKey)) + } + log.Println("client: handshake: ", state.HandshakeComplete) + log.Println("client: mutual: ", state.NegotiatedProtocolIsMutual) + message := "Hello\n" + n, err := io.WriteString(conn, message) + if err != nil { + log.Fatalf("client: write: %s", err) + } + log.Printf("client: wrote %q (%d bytes)", message, n) + reply := make([]byte, 256) + n, err = conn.Read(reply) + log.Printf("client: read %q (%d bytes)", string(reply[:n]), n) + log.Print("client: exiting") +} \ No newline at end of file From 94b99c986bb6f544d3c023ee9c907141ecca4421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 08:59:10 +0100 Subject: [PATCH 005/309] Whitespace --- .../sim-batch-management/es2plus_smoketest.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 05639065e..6b160aaef 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -1,14 +1,13 @@ //usr/bin/env go run "$0" "$@"; exit "$?" package main - import ( -"crypto/tls" -"crypto/x509" -"fmt" -"io" -"log" -"flag" + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "io" + "log" ) // ES2PLUS_ENDPOINT="https://mconnect-es2-005.staging.oberthur.net:1034"f @@ -51,4 +50,4 @@ func main() { n, err = conn.Read(reply) log.Printf("client: read %q (%d bytes)", string(reply[:n]), n) log.Print("client: exiting") -} \ No newline at end of file +} From 8488be10e3f366ec767e639b89c86eb501a34684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 09:59:13 +0100 Subject: [PATCH 006/309] Building basic es2+ payload --- .../sim-batch-management/es2plus_smoketest.go | 51 +++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 6b160aaef..3992cf073 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -7,27 +7,70 @@ import ( "flag" "fmt" "io" + "encoding/json" "log" ) -// ES2PLUS_ENDPOINT="https://mconnect-es2-005.staging.oberthur.net:1034"f +// +// Our new ES2+ library +// + +type ES2PlusHeader struct { + FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` + FunctionCallIdentifier string `json:"functionCallIdentifier"` +} + +type ES2PlusGetProfileStatusRequest struct { + Header ES2PlusHeader `json:"header"` + IccidList []ES2PlusIccid `json:"iccidList"` +} + + +type ES2PlusIccid struct { + Iccid string `json:"iccid"` +} + + +func NewStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { + return ES2PlusGetProfileStatusRequest { + Header: ES2PlusHeader{ FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier:functionRequesterIdentifier }, + IccidList: [] ES2PlusIccid {ES2PlusIccid{Iccid: iccid}}, + } +} func main() { + certFilePath := flag.String("cert", "", "Certificate pem file.") keyFilePath := flag.String("key", "", "Certificate key file.") hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") + requesterId := flag.String("requesterid", "", "ES2+ requester ID.") + + + fmt.Println("certFilePath = ", *certFilePath) + fmt.Println("keyFilePath = ", *keyFilePath) + fmt.Println("hostport = ", *hostport) + fmt.Println("requesterId = ", *requesterId) flag.Parse() + foo := NewStatusRequest("8947000000000000038", *requesterId, "banana") + fooB, _ := json.Marshal(&foo) + fmt.Println(string(fooB)) + + + // funcName(*certFilePath, *keyFilePath, *hostport) +} + +func funcName(certFilePath string, keyFilePath string, hostport string) { cert, err := tls.LoadX509KeyPair( - *certFilePath, - *keyFilePath) + certFilePath, + keyFilePath) if err != nil { log.Fatalf("server: loadkeys: %s", err) } config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} - conn, err := tls.Dial("tcp", *hostport, &config) + conn, err := tls.Dial("tcp", hostport, &config) if err != nil { log.Fatalf("client: dial: %s", err) } From 6ad5428cc68af0469cc5f1dccbf77cbf0e58085c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 10:30:03 +0100 Subject: [PATCH 007/309] Now generating correct JSON (as far as I can tell) --- .../sim-batch-management/es2plus_smoketest.go | 62 ++++++++++++++++--- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 3992cf073..1f57cfafd 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -4,11 +4,14 @@ package main import ( "crypto/tls" "crypto/x509" + "encoding/json" "flag" "fmt" + "bytes" + "strings" "io" - "encoding/json" "log" + "net/http" ) // @@ -40,28 +43,64 @@ func NewStatusRequest(iccid string, functionRequesterIdentifier string, function func main() { - certFilePath := flag.String("cert", "", "Certificate pem file.") keyFilePath := flag.String("key", "", "Certificate key file.") hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") requesterId := flag.String("requesterid", "", "ES2+ requester ID.") - - fmt.Println("certFilePath = ", *certFilePath) - fmt.Println("keyFilePath = ", *keyFilePath) - fmt.Println("hostport = ", *hostport) - fmt.Println("requesterId = ", *requesterId) + fmt.Println("certFilePath = '", *certFilePath, "'") + fmt.Println("keyFilePath = '", *keyFilePath, "'") + fmt.Println("hostport = '", *hostport, "'") + fmt.Println("requesterId = '", *requesterId, "'") flag.Parse() - foo := NewStatusRequest("8947000000000000038", *requesterId, "banana") - fooB, _ := json.Marshal(&foo) - fmt.Println(string(fooB)) + // Generate a "hole in the wall" getProfileStatus request, to be generalized later. + payload := NewStatusRequest("8947000000000000038", *requesterId, "banana") + jsonStrB, _ := json.Marshal(&payload) + fmt.Println(string(jsonStrB)) + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/getProfileStatus", hostport) + + // TODO: Consider also https://github.com/parnurzeal/gorequest + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + + req.Header.Set("X-Admin-Protocol:", "gsma/rsp/v2.0.0") + req.Header.Set("Content-Type", "application/json") + + fmt.Print(" --> ", formatRequest(req)) // funcName(*certFilePath, *keyFilePath, *hostport) } +// formatRequest generates ascii representation of a request +func formatRequest(r *http.Request) string { + // Create return string + var request []string + // Add the request string + url := fmt.Sprintf("%v %v %v", r.Method, r.URL, r.Proto) + request = append(request, url) + // Add the host + request = append(request, fmt.Sprintf("Host: %v", r.Host)) + // Loop through headers + for name, headers := range r.Header { + name = strings.ToLower(name) + for _, h := range headers { + request = append(request, fmt.Sprintf("%v: %v", name, h)) + } + } + + // If this is a POST, add post data + if r.Method == "POST" { + r.ParseForm() + request = append(request, "\n") + request = append(request, r.Form.Encode()) + } + // Return the request as a string + return strings.Join(request, "\n") +} + + func funcName(certFilePath string, keyFilePath string, hostport string) { cert, err := tls.LoadX509KeyPair( certFilePath, @@ -81,6 +120,9 @@ func funcName(certFilePath string, keyFilePath string, hostport string) { fmt.Println("Client: Server public key is:") fmt.Println(x509.MarshalPKIXPublicKey(v.PublicKey)) } + + + log.Println("client: handshake: ", state.HandshakeComplete) log.Println("client: mutual: ", state.NegotiatedProtocolIsMutual) message := "Hello\n" From f292ea1c794cea3982de3cbae27f93e8cfbac33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 10:45:48 +0100 Subject: [PATCH 008/309] First http post got through, result looks legit --- .../sim-batch-management/es2plus_smoketest.go | 124 ++++++++++-------- 1 file changed, 71 insertions(+), 53 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 1f57cfafd..a104777a4 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -3,15 +3,13 @@ package main import ( "crypto/tls" - "crypto/x509" - "encoding/json" "flag" - "fmt" + "encoding/json" "bytes" - "strings" - "io" + "fmt" "log" "net/http" + "strings" ) // @@ -19,25 +17,23 @@ import ( // type ES2PlusHeader struct { - FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` - FunctionCallIdentifier string `json:"functionCallIdentifier"` + FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` + FunctionCallIdentifier string `json:"functionCallIdentifier"` } type ES2PlusGetProfileStatusRequest struct { - Header ES2PlusHeader `json:"header"` - IccidList []ES2PlusIccid `json:"iccidList"` + Header ES2PlusHeader `json:"header"` + IccidList []ES2PlusIccid `json:"iccidList"` } - type ES2PlusIccid struct { - Iccid string `json:"iccid"` + Iccid string `json:"iccid"` } - func NewStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { - return ES2PlusGetProfileStatusRequest { - Header: ES2PlusHeader{ FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier:functionRequesterIdentifier }, - IccidList: [] ES2PlusIccid {ES2PlusIccid{Iccid: iccid}}, + return ES2PlusGetProfileStatusRequest{ + Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, + IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } } @@ -55,22 +51,7 @@ func main() { flag.Parse() - // Generate a "hole in the wall" getProfileStatus request, to be generalized later. - payload := NewStatusRequest("8947000000000000038", *requesterId, "banana") - jsonStrB, _ := json.Marshal(&payload) - fmt.Println(string(jsonStrB)) - - url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/getProfileStatus", hostport) - - // TODO: Consider also https://github.com/parnurzeal/gorequest - req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) - - req.Header.Set("X-Admin-Protocol:", "gsma/rsp/v2.0.0") - req.Header.Set("Content-Type", "application/json") - - fmt.Print(" --> ", formatRequest(req)) - - // funcName(*certFilePath, *keyFilePath, *hostport) + funcName(*certFilePath, *keyFilePath, *hostport, *requesterId) } // formatRequest generates ascii representation of a request @@ -100,39 +81,76 @@ func formatRequest(r *http.Request) string { return strings.Join(request, "\n") } - -func funcName(certFilePath string, keyFilePath string, hostport string) { +func funcName(certFilePath string, keyFilePath string, hostport string, requesterId string) { cert, err := tls.LoadX509KeyPair( certFilePath, keyFilePath) if err != nil { log.Fatalf("server: loadkeys: %s", err) } + config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} - conn, err := tls.Dial("tcp", hostport, &config) - if err != nil { - log.Fatalf("client: dial: %s", err) - } - defer conn.Close() - log.Println("client: connected to: ", conn.RemoteAddr()) - state := conn.ConnectionState() - for _, v := range state.PeerCertificates { - fmt.Println("Client: Server public key is:") - fmt.Println(x509.MarshalPKIXPublicKey(v.PublicKey)) + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &config, + }, } + // Generate a "hole in the wall" getProfileStatus request, to be generalized later. + payload := NewStatusRequest("8947000000000000038", requesterId, "banana") + jsonStrB, _ := json.Marshal(&payload) + fmt.Println(string(jsonStrB)) + + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/getProfileStatus", hostport) + + // TODO: Consider also https://github.com/parnurzeal/gorequest + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + + req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") + req.Header.Set("Content-Type", "application/json") - log.Println("client: handshake: ", state.HandshakeComplete) - log.Println("client: mutual: ", state.NegotiatedProtocolIsMutual) - message := "Hello\n" - n, err := io.WriteString(conn, message) + fmt.Print(" request --> ", formatRequest(req)) + + + + resp, err := client.Do(req) if err != nil { - log.Fatalf("client: write: %s", err) + panic(err) } - log.Printf("client: wrote %q (%d bytes)", message, n) - reply := make([]byte, 256) - n, err = conn.Read(reply) - log.Printf("client: read %q (%d bytes)", string(reply[:n]), n) - log.Print("client: exiting") + + + fmt.Print(" Response --> ", resp) + + + + /* + + conn, err := tls.Dial("tcp", hostport, &config) + + if err != nil { + log.Fatalf("client: dial: %s", err) + } + defer conn.Close() + log.Println("client: connected to: ", conn.RemoteAddr()) + state := conn.ConnectionState() + for _, v := range state.PeerCertificates { + fmt.Println("Client: Server public key is:") + fmt.Println(x509.MarshalPKIXPublicKey(v.PublicKey)) + } + + log.Println("client: handshake: ", state.HandshakeComplete) + log.Println("client: mutual: ", state.NegotiatedProtocolIsMutual) + message := "Hello\n" + n, err := io.WriteString(conn, message) + if err != nil { + log.Fatalf("client: write: %s", err) + } + log.Printf("client: wrote %q (%d bytes)", message, n) + reply := make([]byte, 256) + n, err = conn.Read(reply) + log.Printf("client: read %q (%d bytes)", string(reply[:n]), n) + log.Print("client: exiting") + */ } From 11ae0842872df3aa73979f7cb42becdff7162770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 11:30:37 +0100 Subject: [PATCH 009/309] Ho ho ho, now we have a minimally working ES2 stack (that can handle only status queries, but hey, everybody has to start somewhere --- .../ostelco/sim/es2plus/Es2PlusEntities.kt | 1 + .../sim-batch-management/es2plus_smoketest.go | 90 ++++++++++++------- 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt index b707f62cf..f1c23090d 100644 --- a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt +++ b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt @@ -131,6 +131,7 @@ data class Es2ProfileStatusResponse( @JsonInclude(JsonInclude.Include.NON_NULL) data class ProfileStatus( @JsonProperty("status_last_update_timestamp") val lastUpdateTimestamp:String? = null, + // TODO: Should the next line be deleted? @JsonProperty("profileStatusList") val profileStatusList: List? = listOf(), @JsonProperty("acToken") val acToken: String? = null, @JsonProperty("state") val state: String? = null, diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index a104777a4..462d5375d 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -2,14 +2,15 @@ package main import ( + "bytes" "crypto/tls" - "flag" "encoding/json" - "bytes" + "flag" "fmt" "log" "net/http" "strings" + "io/ioutil" ) // @@ -30,6 +31,44 @@ type ES2PlusIccid struct { Iccid string `json:"iccid"` } +type FunctionExecutionStatus struct { + FunctionExecutionStatusType string `json:"status"` // Should be an enumeration type + StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` +} + +type ES2PlusStatusCodeData struct { + SubjectCode string `json:"subjectCode"` + ReasonCode string `json:"reasonCode"` + SubjectIdentifier string `json:"subjectIdentifier"` + Message string `json:"message"` +} + + +type ES2PlusResponseHeader struct { + FunctionExecutionStatus string `json:"functionExecutionStatus"` +} + +type ES2ProfileStatusResponse struct { + Header ES2PlusResponseHeader `json:"header"` + ProfileStatusList []ProfileStatus `json:"profileStatusList"` + CompletionTimestamp string `json:"completionTimestamp"` +} + +type ProfileStatus struct { + StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` + ACToken string `json:"acToken"` + State string `json:"state"` + Eid string `json:"eid"` + Iccid string `json:"iccid"` + LockFlag bool `json:"lockFlag"` +} + + + +// +// Protocol code +// + func NewStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { return ES2PlusGetProfileStatusRequest{ Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, @@ -97,7 +136,6 @@ func funcName(certFilePath string, keyFilePath string, hostport string, requeste }, } - // Generate a "hole in the wall" getProfileStatus request, to be generalized later. payload := NewStatusRequest("8947000000000000038", requesterId, "banana") jsonStrB, _ := json.Marshal(&payload) @@ -111,46 +149,30 @@ func funcName(certFilePath string, keyFilePath string, hostport string, requeste req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") - fmt.Print(" request --> ", formatRequest(req)) - - - resp, err := client.Do(req) if err != nil { panic(err) } - fmt.Print(" Response --> ", resp) - + // TODO Should check response headers here! + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err.Error()) + } - /* + s, err := parseProfileStatusResponse([]byte(body)) - conn, err := tls.Dial("tcp", hostport, &config) + fmt.Println("S ->", s) +} - if err != nil { - log.Fatalf("client: dial: %s", err) - } - defer conn.Close() - log.Println("client: connected to: ", conn.RemoteAddr()) - state := conn.ConnectionState() - for _, v := range state.PeerCertificates { - fmt.Println("Client: Server public key is:") - fmt.Println(x509.MarshalPKIXPublicKey(v.PublicKey)) - } - log.Println("client: handshake: ", state.HandshakeComplete) - log.Println("client: mutual: ", state.NegotiatedProtocolIsMutual) - message := "Hello\n" - n, err := io.WriteString(conn, message) - if err != nil { - log.Fatalf("client: write: %s", err) - } - log.Printf("client: wrote %q (%d bytes)", message, n) - reply := make([]byte, 256) - n, err = conn.Read(reply) - log.Printf("client: read %q (%d bytes)", string(reply[:n]), n) - log.Print("client: exiting") - */ +func parseProfileStatusResponse(body []byte) (*ES2ProfileStatusResponse, error) { + var s = new(ES2ProfileStatusResponse) + err := json.Unmarshal(body, &s) + if(err != nil){ + fmt.Println("whoops:", err) + } + return s, err } From 9e485850391ec5f60733a4cfd815435b5c69c2be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 11:45:23 +0100 Subject: [PATCH 010/309] Refactoring towards a better protocol stack. --- .../sim-batch-management/es2plus_smoketest.go | 71 ++++++++----------- 1 file changed, 31 insertions(+), 40 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 462d5375d..922ec651b 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -2,15 +2,15 @@ package main import ( - "bytes" - "crypto/tls" "encoding/json" + "bytes" "flag" "fmt" - "log" "net/http" - "strings" + "log" "io/ioutil" + "strings" + "crypto/tls" ) // @@ -32,39 +32,36 @@ type ES2PlusIccid struct { } type FunctionExecutionStatus struct { - FunctionExecutionStatusType string `json:"status"` // Should be an enumeration type - StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` + FunctionExecutionStatusType string `json:"status"` // Should be an enumeration type + StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` } type ES2PlusStatusCodeData struct { - SubjectCode string `json:"subjectCode"` - ReasonCode string `json:"reasonCode"` + SubjectCode string `json:"subjectCode"` + ReasonCode string `json:"reasonCode"` SubjectIdentifier string `json:"subjectIdentifier"` - Message string `json:"message"` + Message string `json:"message"` } - type ES2PlusResponseHeader struct { FunctionExecutionStatus string `json:"functionExecutionStatus"` } type ES2ProfileStatusResponse struct { - Header ES2PlusResponseHeader `json:"header"` - ProfileStatusList []ProfileStatus `json:"profileStatusList"` - CompletionTimestamp string `json:"completionTimestamp"` + Header ES2PlusResponseHeader `json:"header"` + ProfileStatusList []ProfileStatus `json:"profileStatusList"` + CompletionTimestamp string `json:"completionTimestamp"` } type ProfileStatus struct { - StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` - ACToken string `json:"acToken"` - State string `json:"state"` - Eid string `json:"eid"` - Iccid string `json:"iccid"` - LockFlag bool `json:"lockFlag"` + StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` + ACToken string `json:"acToken"` + State string `json:"state"` + Eid string `json:"eid"` + Iccid string `json:"iccid"` + LockFlag bool `json:"lockFlag"` } - - // // Protocol code // @@ -83,14 +80,15 @@ func main() { hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") requesterId := flag.String("requesterid", "", "ES2+ requester ID.") - fmt.Println("certFilePath = '", *certFilePath, "'") - fmt.Println("keyFilePath = '", *keyFilePath, "'") - fmt.Println("hostport = '", *hostport, "'") - fmt.Println("requesterId = '", *requesterId, "'") + fmt.Printf("certFilePath = '%s'\n", *certFilePath) + fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) + fmt.Printf("hostport = '%s'\n", *hostport) + fmt.Printf("requesterId = '%s'\n", *requesterId) flag.Parse() - funcName(*certFilePath, *keyFilePath, *hostport, *requesterId) + info, _ := getProfileInfo(*certFilePath, *keyFilePath, *hostport, *requesterId, "8947000000000000038", "Applecart") + fmt.Println("Info -> ", info) } // formatRequest generates ascii representation of a request @@ -120,7 +118,7 @@ func formatRequest(r *http.Request) string { return strings.Join(request, "\n") } -func funcName(certFilePath string, keyFilePath string, hostport string, requesterId string) { +func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { cert, err := tls.LoadX509KeyPair( certFilePath, keyFilePath) @@ -137,7 +135,7 @@ func funcName(certFilePath string, keyFilePath string, hostport string, requeste } // Generate a "hole in the wall" getProfileStatus request, to be generalized later. - payload := NewStatusRequest("8947000000000000038", requesterId, "banana") + payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) jsonStrB, _ := json.Marshal(&payload) fmt.Println(string(jsonStrB)) @@ -154,7 +152,6 @@ func funcName(certFilePath string, keyFilePath string, hostport string, requeste panic(err) } - // TODO Should check response headers here! body, err := ioutil.ReadAll(resp.Body) @@ -162,17 +159,11 @@ func funcName(certFilePath string, keyFilePath string, hostport string, requeste panic(err.Error()) } - s, err := parseProfileStatusResponse([]byte(body)) - - fmt.Println("S ->", s) -} - - -func parseProfileStatusResponse(body []byte) (*ES2ProfileStatusResponse, error) { - var s = new(ES2ProfileStatusResponse) - err := json.Unmarshal(body, &s) - if(err != nil){ + bodyB := []byte(body) + var result = new(ES2ProfileStatusResponse) + err = json.Unmarshal(bodyB, &result) + if (err != nil) { fmt.Println("whoops:", err) } - return s, err + return result, err } From d5f988fa5926169591bd5a99743316c02eebfc8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 11:47:05 +0100 Subject: [PATCH 011/309] Comment --- sim-administration/sim-batch-management/es2plus_smoketest.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 922ec651b..e5e490912 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -153,6 +153,7 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re } // TODO Should check response headers here! + // (in particular X-admin-protocol) body, err := ioutil.ReadAll(resp.Body) if err != nil { From 9abcbe7305e6cbae9110234ec97b06880a58ea1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 12:04:33 +0100 Subject: [PATCH 012/309] Refactoring generation of new client into separate method --- .../sim-batch-management/es2plus_smoketest.go | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index e5e490912..4e035d81f 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -118,21 +118,13 @@ func formatRequest(r *http.Request) string { return strings.Join(request, "\n") } -func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { - cert, err := tls.LoadX509KeyPair( - certFilePath, - keyFilePath) - if err != nil { - log.Fatalf("server: loadkeys: %s", err) - } - config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &config, - }, - } + + + +func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { + client := newClient(certFilePath, keyFilePath) // Generate a "hole in the wall" getProfileStatus request, to be generalized later. payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) @@ -168,3 +160,19 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re } return result, err } + +func newClient(certFilePath string, keyFilePath string) *http.Client { + cert, err := tls.LoadX509KeyPair( + certFilePath, + keyFilePath) + if err != nil { + log.Fatalf("server: loadkeys: %s", err) + } + config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &config, + }, + } + return client +} From e7043ad403f886e606f8dc66fa6a0e22819b8969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 12:11:05 +0100 Subject: [PATCH 013/309] Parametrizing the es2 command --- .../sim-batch-management/es2plus_smoketest.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 4e035d81f..3d595c216 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -120,18 +120,16 @@ func formatRequest(r *http.Request) string { - - - func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { client := newClient(certFilePath, keyFilePath) + es2plusCommand := "getProfileStatus" // Generate a "hole in the wall" getProfileStatus request, to be generalized later. payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) jsonStrB, _ := json.Marshal(&payload) fmt.Println(string(jsonStrB)) - url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/getProfileStatus", hostport) + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) // TODO: Consider also https://github.com/parnurzeal/gorequest req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) From a82f792765d351eef0b2aa1ab5e75c5a77760a9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:20:17 +0100 Subject: [PATCH 014/309] Cleaning up, noting troublespot --- .../sim-batch-management/es2plus_smoketest.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 3d595c216..854a1ec69 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -2,15 +2,15 @@ package main import ( - "encoding/json" "bytes" + "crypto/tls" + "encoding/json" "flag" "fmt" - "net/http" - "log" "io/ioutil" + "log" + "net/http" "strings" - "crypto/tls" ) // @@ -139,7 +139,7 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re resp, err := client.Do(req) if err != nil { - panic(err) + return nil, err } // TODO Should check response headers here! @@ -147,13 +147,15 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re body, err := ioutil.ReadAll(resp.Body) if err != nil { - panic(err.Error()) + return nil, err } bodyB := []byte(body) var result = new(ES2ProfileStatusResponse) err = json.Unmarshal(bodyB, &result) if (err != nil) { + // TODO: This actually fails, so it's not good. Continuing + // just to get progress, but this needs to be fixed. fmt.Println("whoops:", err) } return result, err From b0639def893a28fea0a501fe39c3e97907003e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:30:14 +0100 Subject: [PATCH 015/309] Refactoring to a bit more compact code --- .../sim-batch-management/es2plus_smoketest.go | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 854a1ec69..1ad2f3f50 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -124,41 +124,48 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re client := newClient(certFilePath, keyFilePath) es2plusCommand := "getProfileStatus" + // Generate a "hole in the wall" getProfileStatus request, to be generalized later. payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) - jsonStrB, _ := json.Marshal(&payload) - fmt.Println(string(jsonStrB)) + jsonStrB, err := json.Marshal(&payload) + if err != nil { + return nil, err + } - url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) + response, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) + if err != nil { + return nil, err + } + var result = new(ES2ProfileStatusResponse) + err = json.Unmarshal(response, &result) + if err != nil { + // TODO: This actually fails, so it's not good. Continuing + // just to get progress, but this needs to be fixed. + fmt.Println("whoops:", err) + } + return result, err +} + +func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, client *http.Client) ([]byte, error) { + fmt.Println(string(jsonStrB)) + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) // TODO: Consider also https://github.com/parnurzeal/gorequest req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) - req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) if err != nil { return nil, err } - // TODO Should check response headers here! // (in particular X-admin-protocol) - - body, err := ioutil.ReadAll(resp.Body) + response, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } - - bodyB := []byte(body) - var result = new(ES2ProfileStatusResponse) - err = json.Unmarshal(bodyB, &result) - if (err != nil) { - // TODO: This actually fails, so it's not good. Continuing - // just to get progress, but this needs to be fixed. - fmt.Println("whoops:", err) - } - return result, err + responseBytes := []byte(response) + return responseBytes, nil } func newClient(certFilePath string, keyFilePath string) *http.Client { From 70ac4eca23520c4fc1c55e01f7d170027df7d121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:31:02 +0100 Subject: [PATCH 016/309] Deleting obsolete comments --- sim-administration/sim-batch-management/es2plus_smoketest.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 1ad2f3f50..fe4aa34f7 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -124,8 +124,6 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re client := newClient(certFilePath, keyFilePath) es2plusCommand := "getProfileStatus" - - // Generate a "hole in the wall" getProfileStatus request, to be generalized later. payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) jsonStrB, err := json.Marshal(&payload) if err != nil { @@ -150,7 +148,6 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, client *http.Client) ([]byte, error) { fmt.Println(string(jsonStrB)) url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) - // TODO: Consider also https://github.com/parnurzeal/gorequest req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") From 85bbd68b94e0aefbd25ebf304d35f86cf83117cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:31:41 +0100 Subject: [PATCH 017/309] Give variable more understandable name --- sim-administration/sim-batch-management/es2plus_smoketest.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index fe4aa34f7..68ec006c7 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -130,13 +130,13 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re return nil, err } - response, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) if err != nil { return nil, err } var result = new(ES2ProfileStatusResponse) - err = json.Unmarshal(response, &result) + err = json.Unmarshal(responseBytes, &result) if err != nil { // TODO: This actually fails, so it's not good. Continuing // just to get progress, but this needs to be fixed. From bd4f476deae61661a14d82c1ba6b3fd12019a1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:42:31 +0100 Subject: [PATCH 018/309] Fix bug, parse results properly --- .../sim-batch-management/es2plus_smoketest.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 68ec006c7..cd3236406 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -44,7 +44,7 @@ type ES2PlusStatusCodeData struct { } type ES2PlusResponseHeader struct { - FunctionExecutionStatus string `json:"functionExecutionStatus"` + FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` } type ES2ProfileStatusResponse struct { @@ -138,9 +138,7 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re var result = new(ES2ProfileStatusResponse) err = json.Unmarshal(responseBytes, &result) if err != nil { - // TODO: This actually fails, so it's not good. Continuing - // just to get progress, but this needs to be fixed. - fmt.Println("whoops:", err) + return nil, err } return result, err } From d3677c5d502a562ac5a421c1a511057c6eb426f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:43:35 +0100 Subject: [PATCH 019/309] Move main to the bottom prepare to generate lbirary --- .../sim-batch-management/es2plus_smoketest.go | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index cd3236406..ea948a954 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -73,24 +73,6 @@ func NewStatusRequest(iccid string, functionRequesterIdentifier string, function } } -func main() { - - certFilePath := flag.String("cert", "", "Certificate pem file.") - keyFilePath := flag.String("key", "", "Certificate key file.") - hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") - requesterId := flag.String("requesterid", "", "ES2+ requester ID.") - - fmt.Printf("certFilePath = '%s'\n", *certFilePath) - fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) - fmt.Printf("hostport = '%s'\n", *hostport) - fmt.Printf("requesterId = '%s'\n", *requesterId) - - flag.Parse() - - info, _ := getProfileInfo(*certFilePath, *keyFilePath, *hostport, *requesterId, "8947000000000000038", "Applecart") - fmt.Println("Info -> ", info) -} - // formatRequest generates ascii representation of a request func formatRequest(r *http.Request) string { // Create return string @@ -178,3 +160,27 @@ func newClient(certFilePath string, keyFilePath string) *http.Client { } return client } + + +/// +/// Main. The rest should be put into a library. +/// + + +func main() { + + certFilePath := flag.String("cert", "", "Certificate pem file.") + keyFilePath := flag.String("key", "", "Certificate key file.") + hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") + requesterId := flag.String("requesterid", "", "ES2+ requester ID.") + + fmt.Printf("certFilePath = '%s'\n", *certFilePath) + fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) + fmt.Printf("hostport = '%s'\n", *hostport) + fmt.Printf("requesterId = '%s'\n", *requesterId) + + flag.Parse() + + info, _ := getProfileInfo(*certFilePath, *keyFilePath, *hostport, *requesterId, "8947000000000000038", "Applecart") + fmt.Println("Info -> ", info) +} From 049240fa906b332fc1db2206862d21b66f2d49e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:45:26 +0100 Subject: [PATCH 020/309] Move main to the bottom prepare to generate lbirary --- .../sim-batch-management/es2plus_smoketest.go | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index ea948a954..8be33b6a3 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -13,9 +13,9 @@ import ( "strings" ) -// -// Our new ES2+ library -// +/// +/// Generic headers for invocations and responses +/// type ES2PlusHeader struct { FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` @@ -36,6 +36,16 @@ type FunctionExecutionStatus struct { StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` } + +type ES2PlusResponseHeader struct { + FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` +} + + +// +// Status code invocation. +// + type ES2PlusStatusCodeData struct { SubjectCode string `json:"subjectCode"` ReasonCode string `json:"reasonCode"` @@ -43,10 +53,6 @@ type ES2PlusStatusCodeData struct { Message string `json:"message"` } -type ES2PlusResponseHeader struct { - FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` -} - type ES2ProfileStatusResponse struct { Header ES2PlusResponseHeader `json:"header"` ProfileStatusList []ProfileStatus `json:"profileStatusList"` @@ -62,11 +68,11 @@ type ProfileStatus struct { LockFlag bool `json:"lockFlag"` } -// -// Protocol code -// +/// +/// Protocol code +/// -func NewStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { +func newStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { return ES2PlusGetProfileStatusRequest{ Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, From ed008c244cf584d6a239f42cefc25b1196a52d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 13:46:04 +0100 Subject: [PATCH 021/309] Hiding method that don't need to be public --- sim-administration/sim-batch-management/es2plus_smoketest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 8be33b6a3..fe96d9361 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -112,7 +112,7 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re client := newClient(certFilePath, keyFilePath) es2plusCommand := "getProfileStatus" - payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) + payload := newStatusRequest(iccid, requesterId, functionCallIdentifier) jsonStrB, err := json.Marshal(&payload) if err != nil { return nil, err From d3ea25d084a7050898297b4f8bf5eac25af699b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 14:06:52 +0100 Subject: [PATCH 022/309] Now we have something that is _almost_ generic, that can probably be extended to other commands without too much hassle --- .../sim-batch-management/es2plus_smoketest.go | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index fe96d9361..cfa149fb4 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -131,6 +131,24 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re return result, err } +func marshalUnmarshalGeneriEs2plusCommand(certFilePath string, keyFilePath string, hostport string, es2plusCommand string, payload interface{}, result interface{}) error { + client := newClient(certFilePath, keyFilePath) + + jsonStrB, err := json.Marshal(payload) + if err != nil { + return err + } + + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) + if err != nil { + return err + } + + err = json.Unmarshal(responseBytes, result) + return err +} + + func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, client *http.Client) ([]byte, error) { fmt.Println(string(jsonStrB)) url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) @@ -185,8 +203,17 @@ func main() { fmt.Printf("hostport = '%s'\n", *hostport) fmt.Printf("requesterId = '%s'\n", *requesterId) + flag.Parse() - info, _ := getProfileInfo(*certFilePath, *keyFilePath, *hostport, *requesterId, "8947000000000000038", "Applecart") - fmt.Println("Info -> ", info) + result := new(ES2ProfileStatusResponse) + functionCallIdentifier := "kadkjfad" + iccid := "8965030119040000067" + statusRequest := newStatusRequest(iccid, *requesterId, functionCallIdentifier) + err:= marshalUnmarshalGeneriEs2plusCommand(*certFilePath, *keyFilePath, *hostport, "getProfileStatus", statusRequest, result) + if err != nil { + panic(err) + } + + fmt.Println("result -> ", result) } From 2b29d744729e5c35580862624c7e40340a530738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 14:10:27 +0100 Subject: [PATCH 023/309] Refactoring for a bit more clarity --- .../sim-batch-management/es2plus_smoketest.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index cfa149fb4..4362a9ac2 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -131,8 +131,8 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re return result, err } -func marshalUnmarshalGeneriEs2plusCommand(certFilePath string, keyFilePath string, hostport string, es2plusCommand string, payload interface{}, result interface{}) error { - client := newClient(certFilePath, keyFilePath) +func marshalUnmarshalGeneriEs2plusCommand(client *http.Client, hostport string, es2plusCommand string, payload interface{}, result interface{}) error { + jsonStrB, err := json.Marshal(payload) if err != nil { @@ -210,7 +210,8 @@ func main() { functionCallIdentifier := "kadkjfad" iccid := "8965030119040000067" statusRequest := newStatusRequest(iccid, *requesterId, functionCallIdentifier) - err:= marshalUnmarshalGeneriEs2plusCommand(*certFilePath, *keyFilePath, *hostport, "getProfileStatus", statusRequest, result) + client := newClient(*certFilePath, *keyFilePath) + err:= marshalUnmarshalGeneriEs2plusCommand(client, *hostport, "getProfileStatus", statusRequest, result) if err != nil { panic(err) } From d3c30c6a357ef80aeb7fc508783a7e88ce17ab7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 29 Oct 2019 14:27:49 +0100 Subject: [PATCH 024/309] Now we have a somewhat functioning library --- .../sim-batch-management/es2plus/es2plus.go | 188 ++++++++++++++++++ .../sim-batch-management/es2plus_smoketest.go | 188 +----------------- 2 files changed, 193 insertions(+), 183 deletions(-) create mode 100644 sim-administration/sim-batch-management/es2plus/es2plus.go diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go new file mode 100644 index 000000000..aa66a1764 --- /dev/null +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -0,0 +1,188 @@ +package es2plus + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "strings" +) + +/// +/// Generic headers for invocations and responses +/// + +type ES2PlusHeader struct { + FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` + FunctionCallIdentifier string `json:"functionCallIdentifier"` +} + +type ES2PlusGetProfileStatusRequest struct { + Header ES2PlusHeader `json:"header"` + IccidList []ES2PlusIccid `json:"iccidList"` +} + +type ES2PlusIccid struct { + Iccid string `json:"iccid"` +} + +type FunctionExecutionStatus struct { + FunctionExecutionStatusType string `json:"status"` // Should be an enumeration type + StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` +} + + +type ES2PlusResponseHeader struct { + FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` +} + +// +// Status code invocation. +// + +type ES2PlusStatusCodeData struct { + SubjectCode string `json:"subjectCode"` + ReasonCode string `json:"reasonCode"` + SubjectIdentifier string `json:"subjectIdentifier"` + Message string `json:"message"` +} + +type ES2ProfileStatusResponse struct { + Header ES2PlusResponseHeader `json:"header"` + ProfileStatusList []ProfileStatus `json:"profileStatusList"` + CompletionTimestamp string `json:"completionTimestamp"` +} + +type ProfileStatus struct { + StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` + ACToken string `json:"acToken"` + State string `json:"state"` + Eid string `json:"eid"` + Iccid string `json:"iccid"` + LockFlag bool `json:"lockFlag"` +} + +/// +/// Protocol code +/// + +func NewStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { + return ES2PlusGetProfileStatusRequest{ + Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, + IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + } +} + +// formatRequest generates ascii representation of a request +func formatRequest(r *http.Request) string { + // Create return string + var request []string + // Add the request string + url := fmt.Sprintf("%v %v %v", r.Method, r.URL, r.Proto) + request = append(request, url) + // Add the host + request = append(request, fmt.Sprintf("Host: %v", r.Host)) + // Loop through headers + for name, headers := range r.Header { + name = strings.ToLower(name) + for _, h := range headers { + request = append(request, fmt.Sprintf("%v: %v", name, h)) + } + } + + // If this is a POST, add post data + if r.Method == "POST" { + r.ParseForm() + request = append(request, "\n") + request = append(request, r.Form.Encode()) + } + // Return the request as a string + return strings.Join(request, "\n") +} + + + +func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { + client := NewClient(certFilePath, keyFilePath) + es2plusCommand := "getProfileStatus" + + payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) + jsonStrB, err := json.Marshal(&payload) + if err != nil { + return nil, err + } + + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) + if err != nil { + return nil, err + } + + var result = new(ES2ProfileStatusResponse) + err = json.Unmarshal(responseBytes, &result) + if err != nil { + return nil, err + } + return result, err +} + +func MarshalUnmarshalGeneriEs2plusCommand(client *http.Client, hostport string, es2plusCommand string, payload interface{}, result interface{}) error { + + + jsonStrB, err := json.Marshal(payload) + if err != nil { + return err + } + + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) + if err != nil { + return err + } + + err = json.Unmarshal(responseBytes, result) + return err +} + + +func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, client *http.Client) ([]byte, error) { + fmt.Println(string(jsonStrB)) + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + // TODO Should check response headers here! + // (in particular X-admin-protocol) + response, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + responseBytes := []byte(response) + return responseBytes, nil +} + +// TODO: This is now just a http client, but we should extend the _external_ interface +// generate a es2plus endpoint, that contains the endpoint url, and a generator +// for function invocation IDs. This will also require som reengineering of the +// rest of the API. +func NewClient(certFilePath string, keyFilePath string) *http.Client { + cert, err := tls.LoadX509KeyPair( + certFilePath, + keyFilePath) + if err != nil { + log.Fatalf("server: loadkeys: %s", err) + } + config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &config, + }, + } + return client +} + diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 4362a9ac2..cad0401a2 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -2,189 +2,11 @@ package main import ( - "bytes" - "crypto/tls" - "encoding/json" "flag" "fmt" - "io/ioutil" - "log" - "net/http" - "strings" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" ) -/// -/// Generic headers for invocations and responses -/// - -type ES2PlusHeader struct { - FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` - FunctionCallIdentifier string `json:"functionCallIdentifier"` -} - -type ES2PlusGetProfileStatusRequest struct { - Header ES2PlusHeader `json:"header"` - IccidList []ES2PlusIccid `json:"iccidList"` -} - -type ES2PlusIccid struct { - Iccid string `json:"iccid"` -} - -type FunctionExecutionStatus struct { - FunctionExecutionStatusType string `json:"status"` // Should be an enumeration type - StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` -} - - -type ES2PlusResponseHeader struct { - FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` -} - - -// -// Status code invocation. -// - -type ES2PlusStatusCodeData struct { - SubjectCode string `json:"subjectCode"` - ReasonCode string `json:"reasonCode"` - SubjectIdentifier string `json:"subjectIdentifier"` - Message string `json:"message"` -} - -type ES2ProfileStatusResponse struct { - Header ES2PlusResponseHeader `json:"header"` - ProfileStatusList []ProfileStatus `json:"profileStatusList"` - CompletionTimestamp string `json:"completionTimestamp"` -} - -type ProfileStatus struct { - StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` - ACToken string `json:"acToken"` - State string `json:"state"` - Eid string `json:"eid"` - Iccid string `json:"iccid"` - LockFlag bool `json:"lockFlag"` -} - -/// -/// Protocol code -/// - -func newStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { - return ES2PlusGetProfileStatusRequest{ - Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, - IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, - } -} - -// formatRequest generates ascii representation of a request -func formatRequest(r *http.Request) string { - // Create return string - var request []string - // Add the request string - url := fmt.Sprintf("%v %v %v", r.Method, r.URL, r.Proto) - request = append(request, url) - // Add the host - request = append(request, fmt.Sprintf("Host: %v", r.Host)) - // Loop through headers - for name, headers := range r.Header { - name = strings.ToLower(name) - for _, h := range headers { - request = append(request, fmt.Sprintf("%v: %v", name, h)) - } - } - - // If this is a POST, add post data - if r.Method == "POST" { - r.ParseForm() - request = append(request, "\n") - request = append(request, r.Form.Encode()) - } - // Return the request as a string - return strings.Join(request, "\n") -} - - - -func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { - client := newClient(certFilePath, keyFilePath) - es2plusCommand := "getProfileStatus" - - payload := newStatusRequest(iccid, requesterId, functionCallIdentifier) - jsonStrB, err := json.Marshal(&payload) - if err != nil { - return nil, err - } - - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) - if err != nil { - return nil, err - } - - var result = new(ES2ProfileStatusResponse) - err = json.Unmarshal(responseBytes, &result) - if err != nil { - return nil, err - } - return result, err -} - -func marshalUnmarshalGeneriEs2plusCommand(client *http.Client, hostport string, es2plusCommand string, payload interface{}, result interface{}) error { - - - jsonStrB, err := json.Marshal(payload) - if err != nil { - return err - } - - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) - if err != nil { - return err - } - - err = json.Unmarshal(responseBytes, result) - return err -} - - -func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, client *http.Client) ([]byte, error) { - fmt.Println(string(jsonStrB)) - url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) - req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) - req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") - req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - return nil, err - } - // TODO Should check response headers here! - // (in particular X-admin-protocol) - response, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - responseBytes := []byte(response) - return responseBytes, nil -} - -func newClient(certFilePath string, keyFilePath string) *http.Client { - cert, err := tls.LoadX509KeyPair( - certFilePath, - keyFilePath) - if err != nil { - log.Fatalf("server: loadkeys: %s", err) - } - config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &config, - }, - } - return client -} - /// /// Main. The rest should be put into a library. @@ -206,12 +28,12 @@ func main() { flag.Parse() - result := new(ES2ProfileStatusResponse) + result := new(es2plus.ES2ProfileStatusResponse) functionCallIdentifier := "kadkjfad" iccid := "8965030119040000067" - statusRequest := newStatusRequest(iccid, *requesterId, functionCallIdentifier) - client := newClient(*certFilePath, *keyFilePath) - err:= marshalUnmarshalGeneriEs2plusCommand(client, *hostport, "getProfileStatus", statusRequest, result) + statusRequest := es2plus.NewStatusRequest(iccid, *requesterId, functionCallIdentifier) + client := es2plus.NewClient(*certFilePath, *keyFilePath) + err:= es2plus.MarshalUnmarshalGeneriEs2plusCommand(client, *hostport, "getProfileStatus", statusRequest, result) if err != nil { panic(err) } From fc646402b7ac3cf834857dbbe262ca56b8f99187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 10:24:27 +0100 Subject: [PATCH 025/309] In the middle of refactoring. Passes test, but garbage from pre-refactoring still present --- .../sim-batch-management/es2plus/es2plus.go | 63 ++++++++++++++++--- .../sim-batch-management/es2plus_smoketest.go | 12 ++-- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index aa66a1764..d56d579c6 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -69,13 +69,42 @@ type ProfileStatus struct { /// Protocol code /// -func NewStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { +func Es2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { return ES2PlusGetProfileStatusRequest{ Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } } +func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { + + +// func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { + // client := NewClient(certFilePath, keyFilePath) + es2plusCommand := "getProfileStatus" + + payload := Es2PlusStatusRequest(iccid, client.requesterId, "oo") // Use a goroutine to generate function call identifiers + jsonStrB, err := json.Marshal(&payload) + if err != nil { + return nil, err + } + + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) + + if err != nil { + return nil, err + } + + var result = new(ES2ProfileStatusResponse) + err = json.Unmarshal(responseBytes, &result) + if err != nil { + return nil, err + } + return result, err +} + + + // formatRequest generates ascii representation of a request func formatRequest(r *http.Request) string { // Create return string @@ -104,12 +133,12 @@ func formatRequest(r *http.Request) string { } - +/* func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { client := NewClient(certFilePath, keyFilePath) es2plusCommand := "getProfileStatus" - payload := NewStatusRequest(iccid, requesterId, functionCallIdentifier) + payload := Es2PlusStatusRequest(iccid, requesterId, functionCallIdentifier) jsonStrB, err := json.Marshal(&payload) if err != nil { return nil, err @@ -127,16 +156,16 @@ func getProfileInfo(certFilePath string, keyFilePath string, hostport string, re } return result, err } +*/ -func MarshalUnmarshalGeneriEs2plusCommand(client *http.Client, hostport string, es2plusCommand string, payload interface{}, result interface{}) error { - +func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { jsonStrB, err := json.Marshal(payload) if err != nil { return err } - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) if err != nil { return err } @@ -146,13 +175,13 @@ func MarshalUnmarshalGeneriEs2plusCommand(client *http.Client, hostport string, } -func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, client *http.Client) ([]byte, error) { +func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client) ([]byte, error) { fmt.Println(string(jsonStrB)) url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") - resp, err := client.Do(req) + resp, err := httpClient.Do(req) if err != nil { return nil, err } @@ -166,11 +195,27 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma return responseBytes, nil } + +type Es2PlusClient struct { + httpClient *http.Client + hostport string + requesterId string +} + +func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClient) { + return &Es2PlusClient { + httpClient: newHttpClient(certFilePath, keyFilePath), + hostport: hostport, + requesterId: requesterId, + } +} + + // TODO: This is now just a http client, but we should extend the _external_ interface // generate a es2plus endpoint, that contains the endpoint url, and a generator // for function invocation IDs. This will also require som reengineering of the // rest of the API. -func NewClient(certFilePath string, keyFilePath string) *http.Client { +func newHttpClient(certFilePath string, keyFilePath string) *http.Client { cert, err := tls.LoadX509KeyPair( certFilePath, keyFilePath) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index cad0401a2..7adc082df 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -7,12 +7,10 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" ) - /// /// Main. The rest should be put into a library. /// - func main() { certFilePath := flag.String("cert", "", "Certificate pem file.") @@ -25,18 +23,16 @@ func main() { fmt.Printf("hostport = '%s'\n", *hostport) fmt.Printf("requesterId = '%s'\n", *requesterId) - flag.Parse() - result := new(es2plus.ES2ProfileStatusResponse) - functionCallIdentifier := "kadkjfad" iccid := "8965030119040000067" - statusRequest := es2plus.NewStatusRequest(iccid, *requesterId, functionCallIdentifier) - client := es2plus.NewClient(*certFilePath, *keyFilePath) - err:= es2plus.MarshalUnmarshalGeneriEs2plusCommand(client, *hostport, "getProfileStatus", statusRequest, result) + client := es2plus.Client(*certFilePath, *keyFilePath, *hostport, *requesterId) + + result, err := es2plus.GetStatus(client, iccid) if err != nil { panic(err) } + fmt.Println("result -> ", result) } From 452a2acab2a294b142f2d9442f7e3e06cede7388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 10:32:14 +0100 Subject: [PATCH 026/309] More refacoring --- .../sim-batch-management/es2plus/es2plus.go | 38 ++++--------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index d56d579c6..79b7ea383 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -79,9 +79,12 @@ func Es2PlusStatusRequest(iccid string, functionRequesterIdentifier string, func func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { -// func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { - // client := NewClient(certFilePath, keyFilePath) - es2plusCommand := "getProfileStatus" + var result = new(ES2ProfileStatusResponse) + es2plusCommand := "getProfileStatus" + payload := Es2PlusStatusRequest(iccid, client.requesterId, "oo") + err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, &payload, result) + return result, err + /* payload := Es2PlusStatusRequest(iccid, client.requesterId, "oo") // Use a goroutine to generate function call identifiers jsonStrB, err := json.Marshal(&payload) @@ -89,7 +92,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, return nil, err } - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) if err != nil { return nil, err @@ -100,7 +103,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, if err != nil { return nil, err } - return result, err + return result, err */ } @@ -133,31 +136,6 @@ func formatRequest(r *http.Request) string { } -/* -func getProfileInfo(certFilePath string, keyFilePath string, hostport string, requesterId string, iccid string, functionCallIdentifier string) (*ES2ProfileStatusResponse, error) { - client := NewClient(certFilePath, keyFilePath) - es2plusCommand := "getProfileStatus" - - payload := Es2PlusStatusRequest(iccid, requesterId, functionCallIdentifier) - jsonStrB, err := json.Marshal(&payload) - if err != nil { - return nil, err - } - - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, hostport, es2plusCommand, client) - if err != nil { - return nil, err - } - - var result = new(ES2ProfileStatusResponse) - err = json.Unmarshal(responseBytes, &result) - if err != nil { - return nil, err - } - return result, err -} -*/ - func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { jsonStrB, err := json.Marshal(payload) From 74837e72dba3a85466fb4ed94982daf09dc1508f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 10:37:34 +0100 Subject: [PATCH 027/309] More internal meicine. Refactoring for greater clarity --- .../sim-batch-management/es2plus/es2plus.go | 107 ++++++++---------- 1 file changed, 46 insertions(+), 61 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 79b7ea383..a30515841 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -65,48 +65,48 @@ type ProfileStatus struct { LockFlag bool `json:"lockFlag"` } -/// -/// Protocol code -/// - -func Es2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { - return ES2PlusGetProfileStatusRequest{ - Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, - IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, - } -} - -func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { +// +// Generating new ES2Plus clients +// - var result = new(ES2ProfileStatusResponse) - es2plusCommand := "getProfileStatus" - payload := Es2PlusStatusRequest(iccid, client.requesterId, "oo") - err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, &payload, result) - return result, err - /* - payload := Es2PlusStatusRequest(iccid, client.requesterId, "oo") // Use a goroutine to generate function call identifiers - jsonStrB, err := json.Marshal(&payload) - if err != nil { - return nil, err - } +type Es2PlusClient struct { + httpClient *http.Client + hostport string + requesterId string +} - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) +func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClient) { + return &Es2PlusClient { + httpClient: newHttpClient(certFilePath, keyFilePath), + hostport: hostport, + requesterId: requesterId, + } +} +func newHttpClient(certFilePath string, keyFilePath string) *http.Client { + cert, err := tls.LoadX509KeyPair( + certFilePath, + keyFilePath) if err != nil { - return nil, err + log.Fatalf("server: loadkeys: %s", err) } - - var result = new(ES2ProfileStatusResponse) - err = json.Unmarshal(responseBytes, &result) - if err != nil { - return nil, err + config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &config, + }, } - return result, err */ + return client } +/// +/// Generic protocol code +/// + + // formatRequest generates ascii representation of a request func formatRequest(r *http.Request) string { @@ -174,38 +174,23 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma } -type Es2PlusClient struct { - httpClient *http.Client - hostport string - requesterId string -} +/// +/// Externally visible API for Es2Plus protocol +/// -func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClient) { - return &Es2PlusClient { - httpClient: newHttpClient(certFilePath, keyFilePath), - hostport: hostport, - requesterId: requesterId, - } +func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { + return ES2PlusGetProfileStatusRequest{ + Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, + IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + } } - -// TODO: This is now just a http client, but we should extend the _external_ interface -// generate a es2plus endpoint, that contains the endpoint url, and a generator -// for function invocation IDs. This will also require som reengineering of the -// rest of the API. -func newHttpClient(certFilePath string, keyFilePath string) *http.Client { - cert, err := tls.LoadX509KeyPair( - certFilePath, - keyFilePath) - if err != nil { - log.Fatalf("server: loadkeys: %s", err) - } - config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &config, - }, - } - return client +func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { + var result = new(ES2ProfileStatusResponse) + es2plusCommand := "getProfileStatus" + payload := newEs2PlusStatusRequest(iccid, client.requesterId, "oo") // TODO: Add a proper function call identifier + err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, &payload, result) + return result, err } + From 73d4b12214b0a915d3f1d2066273cbe8602803e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 10:40:21 +0100 Subject: [PATCH 028/309] Cleaning up --- .../sim-batch-management/es2plus/es2plus.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index a30515841..b26fbaa4c 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -107,8 +107,11 @@ func newHttpClient(certFilePath string, keyFilePath string) *http.Client { /// - -// formatRequest generates ascii representation of a request +// +// Function used during debugging to print requests before they are +// sent over the wire. Very useful, should not be deleted even though it +// is not used right now. +// func formatRequest(r *http.Request) string { // Create return string var request []string @@ -154,12 +157,14 @@ func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client) ([]byte, error) { - fmt.Println(string(jsonStrB)) + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") resp, err := httpClient.Do(req) + + if err != nil { return nil, err } From 45bc85c0e0d3b5942faa1be0c0e41d1636ac616d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 10:45:46 +0100 Subject: [PATCH 029/309] Add more comments --- .../sim-batch-management/es2plus/es2plus.go | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index b26fbaa4c..b98e1b141 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -141,35 +141,48 @@ func formatRequest(r *http.Request) string { func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { + // Serialize payload as json. jsonStrB, err := json.Marshal(payload) if err != nil { return err } + // Get the result of the HTTP POST as a byte array + // that can be deserialized into json. Fail fast + // an error has been detected. responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) if err != nil { return err } - err = json.Unmarshal(responseBytes, result) - return err + // Return error code from deserialisation, result is put into + // result via referenced object. + return json.Unmarshal(responseBytes, result) } func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client) ([]byte, error) { + // Build and execute an ES2+ protocol request by using the serialised JSON in the + // byte array jsonStrB in a POST requrest. Set up the required + // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") resp, err := httpClient.Do(req) - + // On http protocol failure return quickly if err != nil { return nil, err } + // TODO Should check response headers here! - // (in particular X-admin-protocol) + // (in particular X-admin-protocol) and fail if not OK. + + + // Get payload bytes from response body and return them. + // Return an error if the bytes can't be retrieved. response, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err From 6d903b42389fc414e332d7741ac5ee8f975259c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:05:26 +0100 Subject: [PATCH 030/309] Make tracing configurable(ish) --- go.mod | 1 + go.sum | 2 ++ .../sim-batch-management/es2plus/es2plus.go | 30 +++++++++++++++---- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index eb89dbb49..7d5540395 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/google/go-cmp v0.3.1 // indirect + github.com/google/uuid v1.1.1 github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v1.11.0 github.com/pkg/errors v0.8.1 // indirect diff --git a/go.sum b/go.sum index 62b2b7ef3..854b55ca1 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index b98e1b141..63c6df793 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "strings" + "github.com/google/uuid" ) /// @@ -75,6 +76,8 @@ type Es2PlusClient struct { httpClient *http.Client hostport string requesterId string + printPayload bool + printHeaders bool } func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClient) { @@ -82,6 +85,8 @@ func Client (certFilePath string, keyFilePath string, hostport string, requester httpClient: newHttpClient(certFilePath, keyFilePath), hostport: hostport, requesterId: requesterId, + printPayload: false, + printHeaders: false, } } @@ -147,10 +152,14 @@ func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand return err } + if client.printPayload { + fmt.Print("Payload ->", string(jsonStrB)) + } + // Get the result of the HTTP POST as a byte array // that can be deserialized into json. Fail fast // an error has been detected. - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient) + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient, client.printHeaders) if err != nil { return err } @@ -161,15 +170,21 @@ func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand } -func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client) ([]byte, error) { +func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client, printHeaders bool) ([]byte, error) { // Build and execute an ES2+ protocol request by using the serialised JSON in the - // byte array jsonStrB in a POST requrest. Set up the required + // byte array jsonStrB in a POST request. Set up the required // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") + + + if printHeaders { + fmt.Println("Request -> %s\n", formatRequest(req)) + } + resp, err := httpClient.Do(req) // On http protocol failure return quickly @@ -206,8 +221,13 @@ func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, f func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { var result = new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - payload := newEs2PlusStatusRequest(iccid, client.requesterId, "oo") // TODO: Add a proper function call identifier - err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, &payload, result) + uuid, err := uuid.NewRandom() + if err != nil { + return nil, err + } + functionCallIdentifier := uuid.URN() + payload := newEs2PlusStatusRequest(iccid, client.requesterId, functionCallIdentifier) + err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, &payload, result) return result, err } From a52c664c5a278ea97b108b33a5188194f0e80c7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:08:31 +0100 Subject: [PATCH 031/309] more symmetry --- sim-administration/sim-batch-management/es2plus/es2plus.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 63c6df793..62f21ede4 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -211,8 +211,8 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// Externally visible API for Es2Plus protocol /// -func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) ES2PlusGetProfileStatusRequest { - return ES2PlusGetProfileStatusRequest{ +func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) *ES2PlusGetProfileStatusRequest { + return &ES2PlusGetProfileStatusRequest{ Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } @@ -227,7 +227,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, } functionCallIdentifier := uuid.URN() payload := newEs2PlusStatusRequest(iccid, client.requesterId, functionCallIdentifier) - err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, &payload, result) + err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) return result, err } From 6a243f8f9316ecc6fc3a6425ae111d2a55e6d16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:13:29 +0100 Subject: [PATCH 032/309] Refactoring uuid generation into separate method --- .../sim-batch-management/es2plus/es2plus.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 62f21ede4..69d5006ae 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -211,6 +211,14 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// Externally visible API for Es2Plus protocol /// +func newUuid() (string, error) { + uuid, err := uuid.NewRandom() + if err != nil { + return "", err + } + return uuid.URN(), nil +} + func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) *ES2PlusGetProfileStatusRequest { return &ES2PlusGetProfileStatusRequest{ Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, @@ -221,13 +229,12 @@ func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, f func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { var result = new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - uuid, err := uuid.NewRandom() - if err != nil { - return nil, err - } - functionCallIdentifier := uuid.URN() + functionCallIdentifier, err := newUuid() + if err != nil { + return nil, err + } payload := newEs2PlusStatusRequest(iccid, client.requesterId, functionCallIdentifier) - err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) + err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) return result, err } From 5728591b66ecbec3e5c6b57b6949eacde0ffd4c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:15:44 +0100 Subject: [PATCH 033/309] One more step of refactoring --- .../sim-batch-management/es2plus/es2plus.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 69d5006ae..c6d5bd1de 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -219,21 +219,26 @@ func newUuid() (string, error) { return uuid.URN(), nil } -func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string, functionCallIdentifier string) *ES2PlusGetProfileStatusRequest { +func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string) (*ES2PlusGetProfileStatusRequest, error) { + functionCallIdentifier, err := newUuid() + if err != nil { + return nil, err + } + return &ES2PlusGetProfileStatusRequest{ Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, - } + }, nil } func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { var result = new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - functionCallIdentifier, err := newUuid() - if err != nil { + + payload, err := newEs2PlusStatusRequest(iccid, client.requesterId) + if err != nil { return nil, err } - payload := newEs2PlusStatusRequest(iccid, client.requesterId, functionCallIdentifier) err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) return result, err } From e0c19228b491ca4c4cde6c2844585ad1957c405b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:29:37 +0100 Subject: [PATCH 034/309] More refactoring --- .../sim-batch-management/es2plus/es2plus.go | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index c6d5bd1de..6f408c04e 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -143,6 +143,22 @@ func formatRequest(r *http.Request) string { return strings.Join(request, "\n") } +func newUuid() (string, error) { + uuid, err := uuid.NewRandom() + if err != nil { + return "", err + } + return uuid.URN(), nil +} + +func newEs2plusHeader(client *Es2PlusClient) (*ES2PlusHeader, error) { + functionCallIdentifier, err := newUuid() + if err != nil { + return nil, err + } + return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.requesterId}, nil +} + func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { @@ -211,22 +227,10 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// Externally visible API for Es2Plus protocol /// -func newUuid() (string, error) { - uuid, err := uuid.NewRandom() - if err != nil { - return "", err - } - return uuid.URN(), nil -} - -func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string) (*ES2PlusGetProfileStatusRequest, error) { - functionCallIdentifier, err := newUuid() - if err != nil { - return nil, err - } +func newEs2PlusStatusRequest(iccid string, header *ES2PlusHeader) (*ES2PlusGetProfileStatusRequest, error) { return &ES2PlusGetProfileStatusRequest{ - Header: ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: functionRequesterIdentifier}, + Header: *header, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, }, nil } @@ -234,8 +238,9 @@ func newEs2PlusStatusRequest(iccid string, functionRequesterIdentifier string) ( func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { var result = new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" + header,err := newEs2plusHeader(client) - payload, err := newEs2PlusStatusRequest(iccid, client.requesterId) + payload, err := newEs2PlusStatusRequest(iccid, header) if err != nil { return nil, err } From f9db07ef89c45dd0fa113385a1522f10510e01d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:34:03 +0100 Subject: [PATCH 035/309] Optimizing away unklikely and impossible error situations --- .../sim-batch-management/es2plus/es2plus.go | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 6f408c04e..47a9d5d4c 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -151,12 +151,12 @@ func newUuid() (string, error) { return uuid.URN(), nil } -func newEs2plusHeader(client *Es2PlusClient) (*ES2PlusHeader, error) { +func newEs2plusHeader(client *Es2PlusClient) (*ES2PlusHeader) { functionCallIdentifier, err := newUuid() if err != nil { - return nil, err + panic(err) } - return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.requesterId}, nil + return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.requesterId} } @@ -228,23 +228,19 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// -func newEs2PlusStatusRequest(iccid string, header *ES2PlusHeader) (*ES2PlusGetProfileStatusRequest, error) { +func newEs2PlusStatusRequest(iccid string, header *ES2PlusHeader) (*ES2PlusGetProfileStatusRequest) { return &ES2PlusGetProfileStatusRequest{ Header: *header, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, - }, nil + } } func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { var result = new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - header,err := newEs2plusHeader(client) - - payload, err := newEs2PlusStatusRequest(iccid, header) - if err != nil { - return nil, err - } - err = marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) + header := newEs2plusHeader(client) + payload := newEs2PlusStatusRequest(iccid, header) + err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) return result, err } From d5e7c1850fc930dfdf18fddda39b9a9f06ddb524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:35:38 +0100 Subject: [PATCH 036/309] Open coding payload generation --- .../sim-batch-management/es2plus/es2plus.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 47a9d5d4c..5ec0950a2 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -228,18 +228,14 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// -func newEs2PlusStatusRequest(iccid string, header *ES2PlusHeader) (*ES2PlusGetProfileStatusRequest) { - return &ES2PlusGetProfileStatusRequest{ - Header: *header, - IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, - } -} - func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { var result = new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" header := newEs2plusHeader(client) - payload := newEs2PlusStatusRequest(iccid, header) + payload := &ES2PlusGetProfileStatusRequest{ + Header: *header, + IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + } err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) return result, err } From 70122c5d29b177044b96b06ab6b5805e83d3dc4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:36:41 +0100 Subject: [PATCH 037/309] Optimizing away last line --- sim-administration/sim-batch-management/es2plus/es2plus.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 5ec0950a2..3a8958cde 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -236,8 +236,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, Header: *header, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - err := marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) - return result, err + return result, marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) } From 020ef09b95167cb177ed65d953b77a4bb6667227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 11:38:03 +0100 Subject: [PATCH 038/309] Fix typo --- sim-administration/sim-batch-management/es2plus/es2plus.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 3a8958cde..be7cbd8d7 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -160,7 +160,7 @@ func newEs2plusHeader(client *Es2PlusClient) (*ES2PlusHeader) { } -func marshalUnmarshalGeneriEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { +func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { // Serialize payload as json. jsonStrB, err := json.Marshal(payload) @@ -236,7 +236,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, Header: *header, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - return result, marshalUnmarshalGeneriEs2plusCommand(client, es2plusCommand, payload, result) + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) } From 619cc8da5215a6f677f1767784d7b95bf498def5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 13:15:14 +0100 Subject: [PATCH 039/309] Refactor to deliver result of single iccid query --- .../sim-batch-management/es2plus/es2plus.go | 18 ++++++++++-- .../sim-batch-management/es2plus_smoketest.go | 29 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index be7cbd8d7..5273813eb 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "strings" + "errors" "github.com/google/uuid" ) @@ -228,15 +229,26 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// -func GetStatus(client *Es2PlusClient, iccid string) (*ES2ProfileStatusResponse, error) { - var result = new(ES2ProfileStatusResponse) +func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { + result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" header := newEs2plusHeader(client) payload := &ES2PlusGetProfileStatusRequest{ Header: *header, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + if (len(result.ProfileStatusList) == 0) { + return nil, nil + } else if (len(result.ProfileStatusList) == 1) { + return &(result.ProfileStatusList[0]), nil + } else { + return nil, errors.New("GetStatus returned more than one profile!") + } } diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 7adc082df..5d3878827 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -25,6 +25,7 @@ func main() { flag.Parse() + // TODO: Move the actual ICCID into the caller function iccid := "8965030119040000067" client := es2plus.Client(*certFilePath, *keyFilePath, *hostport, *requesterId) @@ -33,6 +34,34 @@ func main() { panic(err) } + // TODO: Assert that the state is "AVAILABLE" fmt.Println("result -> ", result) + + /** + // TODO: Generate a full roundtrip taking some suitable profile through a proper + // activation, and reset. + result, err := es2plus.Activate(client, iccid) + if err != nil { + panic(err) + } + + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + // Make some assertion about the status at this point + result, err := es2plus.Reset(client, iccid) + if err != nil { + panic(err) + } + + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + // Make some assertion about the status at this point +*/ } From 78db1387f44980008d3d837274abad72654fe572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 13:23:55 +0100 Subject: [PATCH 040/309] More refactoring to shape external API to be nicer for uses (me) --- .../sim-batch-management/es2plus/es2plus.go | 28 ++++++++++++++++++- .../sim-batch-management/es2plus_smoketest.go | 2 +- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 5273813eb..06ef8fa05 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -245,7 +245,33 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { if (len(result.ProfileStatusList) == 0) { return nil, nil } else if (len(result.ProfileStatusList) == 1) { - return &(result.ProfileStatusList[0]), nil + returnvalue := result.ProfileStatusList[0] + return &returnvalue, nil + } else { + return nil, errors.New("GetStatus returned more than one profile!") + } +} + + + +func RecoverProfile(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { + result := new(ES2ProfileStatusResponse) + es2plusCommand := "getProfileStatus" + header := newEs2plusHeader(client) + payload := &ES2PlusGetProfileStatusRequest{ + Header: *header, + IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + } + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + if (len(result.ProfileStatusList) == 0) { + return nil, nil + } else if (len(result.ProfileStatusList) == 1) { + returnvalue := result.ProfileStatusList[0] + return &returnvalue, nil } else { return nil, errors.New("GetStatus returned more than one profile!") } diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 5d3878827..0702cc82a 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -36,7 +36,7 @@ func main() { // TODO: Assert that the state is "AVAILABLE" - fmt.Println("result -> ", result) + fmt.Println("result -> ", result.State) /** // TODO: Generate a full roundtrip taking some suitable profile through a proper From 6ca882d3cbf4bc06a6312d3bba3eff785a076b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 13:46:19 +0100 Subject: [PATCH 041/309] Added code to recover profile, but not tested yet --- .../sim-batch-management/es2plus/es2plus.go | 45 ++++++++++--------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 06ef8fa05..c736c76e2 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -41,6 +41,8 @@ type ES2PlusResponseHeader struct { FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` } + + // // Status code invocation. // @@ -68,6 +70,21 @@ type ProfileStatus struct { } +// +// Profile reset +// + +type ES2PlusRecoverProfileRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + ProfileStatus string `json:"profileStatus"` + } + +type ES2PlusRecoverProfileResponse struct { + Header ES2PlusHeader `json:"header"` +} + + // // Generating new ES2Plus clients // @@ -253,28 +270,16 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { } - -func RecoverProfile(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { - result := new(ES2ProfileStatusResponse) - es2plusCommand := "getProfileStatus" +func RecoverProfile(client *Es2PlusClient, iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { + result := new(ES2PlusRecoverProfileResponse) + es2plusCommand := "recoverProfile" header := newEs2plusHeader(client) - payload := &ES2PlusGetProfileStatusRequest{ - Header: *header, - IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + payload := &ES2PlusRecoverProfileRequest{ + Header: *header, + Iccid: iccid, + ProfileStatus: targetState, } - err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { - return nil, err - } - - if (len(result.ProfileStatusList) == 0) { - return nil, nil - } else if (len(result.ProfileStatusList) == 1) { - returnvalue := result.ProfileStatusList[0] - return &returnvalue, nil - } else { - return nil, errors.New("GetStatus returned more than one profile!") - } + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) } From 50fdc35ac5ea65fbc68ab47721e0914c2fdf5aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 13:50:25 +0100 Subject: [PATCH 042/309] The reset thing doesn't work, but now we hatve a test that proves that --- .../sim-batch-management/es2plus_smoketest.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 0702cc82a..9ecf64878 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -33,10 +33,24 @@ func main() { if err != nil { panic(err) } + fmt.Println("result1 -> ", result.State) + result2, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") + if err != nil { + panic(err) + } + + fmt.Println("result2 -> ", result2) + + result, err = es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + fmt.Println("result3 -> ", result.State) // TODO: Assert that the state is "AVAILABLE" - fmt.Println("result -> ", result.State) + /** // TODO: Generate a full roundtrip taking some suitable profile through a proper From 08bfad31f54e23319fcf64c8f03c398131eba556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 13:58:13 +0100 Subject: [PATCH 043/309] Still not able to send this particular profile back to available --- .../sim-batch-management/es2plus/es2plus.go | 30 ++++++++++++++++++- .../sim-batch-management/es2plus_smoketest.go | 18 +++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index c736c76e2..16e63ecae 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -71,7 +71,7 @@ type ProfileStatus struct { // -// Profile reset +// Profile reset invocation // type ES2PlusRecoverProfileRequest struct { @@ -85,6 +85,22 @@ type ES2PlusRecoverProfileResponse struct { } + +// +// Cancel order invocation +// + +type ES2PlusCancelOrderRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` + } + +type ES2PlusCancelOrderResponse struct { + Header ES2PlusHeader `json:"header"` +} + + // // Generating new ES2Plus clients // @@ -283,3 +299,15 @@ func RecoverProfile(client *Es2PlusClient, iccid string, targetState string) (*E } +func CancelOrder(client *Es2PlusClient, iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { + result := new(ES2PlusCancelOrderResponse) + es2plusCommand := "cancelOrder" + header := newEs2plusHeader(client) + payload := &ES2PlusCancelOrderRequest { + Header: *header, + Iccid: iccid, + FinalProfileStatusIndicator: targetState, + } + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) +} + diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 9ecf64878..c77c179cc 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -48,6 +48,24 @@ func main() { } fmt.Println("result3 -> ", result.State) + + + result4, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") + if err != nil { + panic(err) + } + + fmt.Println("result4 -> ", result4) + + result, err = es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + fmt.Println("result5 -> ", result.State) + + + // TODO: Assert that the state is "AVAILABLE" From a2dccab16a689df672e105832a02fc39b597b390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 14:03:26 +0100 Subject: [PATCH 044/309] Using another iccid worked better --- .../sim-batch-management/es2plus_smoketest.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index c77c179cc..612effb2c 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -17,16 +17,19 @@ func main() { keyFilePath := flag.String("key", "", "Certificate key file.") hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") requesterId := flag.String("requesterid", "", "ES2+ requester ID.") + iccidInput := flag.String("iccid", "", "Iccid of profile to manipulate") + + + flag.Parse() fmt.Printf("certFilePath = '%s'\n", *certFilePath) fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) fmt.Printf("hostport = '%s'\n", *hostport) fmt.Printf("requesterId = '%s'\n", *requesterId) + fmt.Printf("iccidInput = '%s'\n", *iccidInput) - flag.Parse() - - // TODO: Move the actual ICCID into the caller function - iccid := "8965030119040000067" + iccid := *iccidInput + client := es2plus.Client(*certFilePath, *keyFilePath, *hostport, *requesterId) result, err := es2plus.GetStatus(client, iccid) From bb3db57cfb9758fabe9d767427f2045b2d2ed277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 14:34:02 +0100 Subject: [PATCH 045/309] Now we can also do the downloadorder thing --- .../ostelco/sim/es2plus/Es2PlusEntities.kt | 2 +- .../sim-batch-management/es2plus/es2plus.go | 43 +++++++++++++++++-- .../sim-batch-management/es2plus_smoketest.go | 19 +++++++- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt index f1c23090d..cafe399be 100644 --- a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt +++ b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt @@ -76,7 +76,7 @@ data class Es2PlusDownloadOrder( @JsonInclude(JsonInclude.Include.NON_NULL) data class Es2DownloadOrderResponse( @JsonProperty("header") val header: ES2ResponseHeader = eS2SuccessResponseHeader(), - @JsonProperty("iccid") val iccid: String? = null + @JsonProperty("iccid") val iccid: String? = null ): Es2Response(header) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 16e63ecae..6b0a3c2da 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -32,7 +32,7 @@ type ES2PlusIccid struct { } type FunctionExecutionStatus struct { - FunctionExecutionStatusType string `json:"status"` // Should be an enumeration type + FunctionExecutionStatusType string `json:"status"` StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` } @@ -81,7 +81,7 @@ type ES2PlusRecoverProfileRequest struct { } type ES2PlusRecoverProfileResponse struct { - Header ES2PlusHeader `json:"header"` + Header ES2PlusResponseHeader `json:"header"` } @@ -97,9 +97,24 @@ type ES2PlusCancelOrderRequest struct { } type ES2PlusCancelOrderResponse struct { - Header ES2PlusHeader `json:"header"` + Header ES2PlusResponseHeader `json:"header"` +} + +// +// Download order invocation +// + +type ES2PlusDownloadOrderRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + Profiletype string `json:"profiletype,omitempty"` } +type ES2PlusDownloadOrderResponse struct { + Header ES2PlusResponseHeader `json:"header"` + Iccid string `json:"iccid"` +} // // Generating new ES2Plus clients @@ -311,3 +326,25 @@ func CancelOrder(client *Es2PlusClient, iccid string, targetState string) (*ES2P return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) } +func DownloadOrder(client *Es2PlusClient, iccid string) (*ES2PlusDownloadOrderResponse, error) { + result := new(ES2PlusDownloadOrderResponse) + es2plusCommand := "downloadOrder" + header := newEs2plusHeader(client) + payload := &ES2PlusDownloadOrderRequest { + Header: *header, + Iccid: iccid, + Eid: "", + Profiletype: "", + } + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + if ("Executed-Success" != executionStatus) { + return result, errors.New(fmt.Sprintf("ExecutionStatus was: ''%s'", executionStatus)) + } else { + return result, nil + } +} diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 612effb2c..591a0db5f 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -29,7 +29,7 @@ func main() { fmt.Printf("iccidInput = '%s'\n", *iccidInput) iccid := *iccidInput - + client := es2plus.Client(*certFilePath, *keyFilePath, *hostport, *requesterId) result, err := es2plus.GetStatus(client, iccid) @@ -67,10 +67,23 @@ func main() { fmt.Println("result5 -> ", result.State) + if result.State != "AVAILABLE" { + panic("Couldn't convert state of iccid into AVAILABLE") + } + + result6, err := es2plus.DownloadOrder(client, iccid) + fmt.Println("result6 -> ", result6) + if err != nil { + panic(err) + } - // TODO: Assert that the state is "AVAILABLE" + result7, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result7 -> ", result7) /** @@ -99,4 +112,6 @@ func main() { // Make some assertion about the status at this point */ + + fmt.Println("Success") } From b2b24faeafaabbee007bfce70a69cf8ec3d8df8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 14:49:48 +0100 Subject: [PATCH 046/309] Full ES2Plus (as far as we need) now implemented --- .../sim-batch-management/es2plus/es2plus.go | 51 +++++++++++++++++++ .../sim-batch-management/es2plus_smoketest.go | 13 +++++ 2 files changed, 64 insertions(+) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 6b0a3c2da..fd92ed167 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -116,6 +116,29 @@ type ES2PlusDownloadOrderResponse struct { Iccid string `json:"iccid"` } +// +// ConfirmOrder invocation +// + +type ES2PlusConfirmOrderRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + MatchingId string `json:"matchingId,omitempty"` + ConfirmationCode string `json:"confirmationCode,omitempty"` + SmdpAddress string `json:"smdpAddress,omitempty"` + ReleaseFlag bool `json:"releaseFlag"` +} + +type ES2PlusConfirmOrderResponse struct { + Header ES2PlusResponseHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + MatchingId string `json:"matchingId,omitempty"` + SmdpAddress string `json:"smdpAddress,omitempty"` +} + + // // Generating new ES2Plus clients // @@ -348,3 +371,31 @@ func DownloadOrder(client *Es2PlusClient, iccid string) (*ES2PlusDownloadOrderRe return result, nil } } + + +func ConfirmOrder(client *Es2PlusClient, iccid string) (*ES2PlusConfirmOrderResponse, error) { + result := new(ES2PlusConfirmOrderResponse) + es2plusCommand := "confirmOrder" + header := newEs2plusHeader(client) + payload := &ES2PlusConfirmOrderRequest { + Header: *header, + Iccid: iccid, + Eid: "", + ConfirmationCode: "", + MatchingId: "", + SmdpAddress: "", + ReleaseFlag: true, + } + + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + if ("Executed-Success" != executionStatus) { + return result, errors.New(fmt.Sprintf("ExecutionStatus was: ''%s'", executionStatus)) + } else { + return result, nil + } +} diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 591a0db5f..2ed367a28 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -86,6 +86,19 @@ func main() { fmt.Println("result7 -> ", result7) + result8, err := es2plus.ConfirmOrder(client, iccid) + fmt.Println("result8 -> ", result8) + if err != nil { + panic(err) + } + + + result9, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result9 -> ", result9) + /** // TODO: Generate a full roundtrip taking some suitable profile through a proper // activation, and reset. From ecc371cadc598fc77107b5077d7f76db09251562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 16:14:20 +0100 Subject: [PATCH 047/309] More todo --- sim-administration/sim-batch-management/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index 168b65331..6ab2a6422 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -25,6 +25,10 @@ comments near the top of the files for instructions on how to use them. # TODO * Make a build command that runs tests and reports test coverage (etc), make it part of the "build-all.go" script. +* Write some code that allows sim-batches to be kept in database. + - Pick up the location of the database and also other + secrets via an environment variable. + - Figure out if we can make multiple command-line programs that have equal value From f2164fa1a39deb09ce43409feea4f4a3bdf49733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 16:21:37 +0100 Subject: [PATCH 048/309] Beginning (functioning) es2plus stack written in golang --- .../sim-batch-management/es2plus/es2plus.go | 401 ++++++++++++++++++ .../sim-batch-management/es2plus_smoketest.go | 130 ++++++ 2 files changed, 531 insertions(+) create mode 100644 sim-administration/sim-batch-management/es2plus/es2plus.go create mode 100755 sim-administration/sim-batch-management/es2plus_smoketest.go diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go new file mode 100644 index 000000000..fd92ed167 --- /dev/null +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -0,0 +1,401 @@ +package es2plus + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "net/http" + "strings" + "errors" + "github.com/google/uuid" +) + +/// +/// Generic headers for invocations and responses +/// + +type ES2PlusHeader struct { + FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` + FunctionCallIdentifier string `json:"functionCallIdentifier"` +} + +type ES2PlusGetProfileStatusRequest struct { + Header ES2PlusHeader `json:"header"` + IccidList []ES2PlusIccid `json:"iccidList"` +} + +type ES2PlusIccid struct { + Iccid string `json:"iccid"` +} + +type FunctionExecutionStatus struct { + FunctionExecutionStatusType string `json:"status"` + StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` +} + + +type ES2PlusResponseHeader struct { + FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` +} + + + +// +// Status code invocation. +// + +type ES2PlusStatusCodeData struct { + SubjectCode string `json:"subjectCode"` + ReasonCode string `json:"reasonCode"` + SubjectIdentifier string `json:"subjectIdentifier"` + Message string `json:"message"` +} + +type ES2ProfileStatusResponse struct { + Header ES2PlusResponseHeader `json:"header"` + ProfileStatusList []ProfileStatus `json:"profileStatusList"` + CompletionTimestamp string `json:"completionTimestamp"` +} + +type ProfileStatus struct { + StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` + ACToken string `json:"acToken"` + State string `json:"state"` + Eid string `json:"eid"` + Iccid string `json:"iccid"` + LockFlag bool `json:"lockFlag"` +} + + +// +// Profile reset invocation +// + +type ES2PlusRecoverProfileRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + ProfileStatus string `json:"profileStatus"` + } + +type ES2PlusRecoverProfileResponse struct { + Header ES2PlusResponseHeader `json:"header"` +} + + + +// +// Cancel order invocation +// + +type ES2PlusCancelOrderRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` + } + +type ES2PlusCancelOrderResponse struct { + Header ES2PlusResponseHeader `json:"header"` +} + +// +// Download order invocation +// + +type ES2PlusDownloadOrderRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + Profiletype string `json:"profiletype,omitempty"` +} + +type ES2PlusDownloadOrderResponse struct { + Header ES2PlusResponseHeader `json:"header"` + Iccid string `json:"iccid"` +} + +// +// ConfirmOrder invocation +// + +type ES2PlusConfirmOrderRequest struct { + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + MatchingId string `json:"matchingId,omitempty"` + ConfirmationCode string `json:"confirmationCode,omitempty"` + SmdpAddress string `json:"smdpAddress,omitempty"` + ReleaseFlag bool `json:"releaseFlag"` +} + +type ES2PlusConfirmOrderResponse struct { + Header ES2PlusResponseHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + MatchingId string `json:"matchingId,omitempty"` + SmdpAddress string `json:"smdpAddress,omitempty"` +} + + +// +// Generating new ES2Plus clients +// + + +type Es2PlusClient struct { + httpClient *http.Client + hostport string + requesterId string + printPayload bool + printHeaders bool +} + +func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClient) { + return &Es2PlusClient { + httpClient: newHttpClient(certFilePath, keyFilePath), + hostport: hostport, + requesterId: requesterId, + printPayload: false, + printHeaders: false, + } +} + +func newHttpClient(certFilePath string, keyFilePath string) *http.Client { + cert, err := tls.LoadX509KeyPair( + certFilePath, + keyFilePath) + if err != nil { + log.Fatalf("server: loadkeys: %s", err) + } + config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &config, + }, + } + return client +} + + +/// +/// Generic protocol code +/// + + +// +// Function used during debugging to print requests before they are +// sent over the wire. Very useful, should not be deleted even though it +// is not used right now. +// +func formatRequest(r *http.Request) string { + // Create return string + var request []string + // Add the request string + url := fmt.Sprintf("%v %v %v", r.Method, r.URL, r.Proto) + request = append(request, url) + // Add the host + request = append(request, fmt.Sprintf("Host: %v", r.Host)) + // Loop through headers + for name, headers := range r.Header { + name = strings.ToLower(name) + for _, h := range headers { + request = append(request, fmt.Sprintf("%v: %v", name, h)) + } + } + + // If this is a POST, add post data + if r.Method == "POST" { + r.ParseForm() + request = append(request, "\n") + request = append(request, r.Form.Encode()) + } + // Return the request as a string + return strings.Join(request, "\n") +} + +func newUuid() (string, error) { + uuid, err := uuid.NewRandom() + if err != nil { + return "", err + } + return uuid.URN(), nil +} + +func newEs2plusHeader(client *Es2PlusClient) (*ES2PlusHeader) { + functionCallIdentifier, err := newUuid() + if err != nil { + panic(err) + } + return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.requesterId} +} + + +func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { + + // Serialize payload as json. + jsonStrB, err := json.Marshal(payload) + if err != nil { + return err + } + + if client.printPayload { + fmt.Print("Payload ->", string(jsonStrB)) + } + + // Get the result of the HTTP POST as a byte array + // that can be deserialized into json. Fail fast + // an error has been detected. + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient, client.printHeaders) + if err != nil { + return err + } + + // Return error code from deserialisation, result is put into + // result via referenced object. + return json.Unmarshal(responseBytes, result) +} + + +func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client, printHeaders bool) ([]byte, error) { + + // Build and execute an ES2+ protocol request by using the serialised JSON in the + // byte array jsonStrB in a POST request. Set up the required + // headers for ES2+ and content type. + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) + req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") + req.Header.Set("Content-Type", "application/json") + + + if printHeaders { + fmt.Println("Request -> %s\n", formatRequest(req)) + } + + resp, err := httpClient.Do(req) + + // On http protocol failure return quickly + if err != nil { + return nil, err + } + + // TODO Should check response headers here! + // (in particular X-admin-protocol) and fail if not OK. + + + // Get payload bytes from response body and return them. + // Return an error if the bytes can't be retrieved. + response, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + responseBytes := []byte(response) + return responseBytes, nil +} + + +/// +/// Externally visible API for Es2Plus protocol +/// + + +func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { + result := new(ES2ProfileStatusResponse) + es2plusCommand := "getProfileStatus" + header := newEs2plusHeader(client) + payload := &ES2PlusGetProfileStatusRequest{ + Header: *header, + IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + } + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + if (len(result.ProfileStatusList) == 0) { + return nil, nil + } else if (len(result.ProfileStatusList) == 1) { + returnvalue := result.ProfileStatusList[0] + return &returnvalue, nil + } else { + return nil, errors.New("GetStatus returned more than one profile!") + } +} + + +func RecoverProfile(client *Es2PlusClient, iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { + result := new(ES2PlusRecoverProfileResponse) + es2plusCommand := "recoverProfile" + header := newEs2plusHeader(client) + payload := &ES2PlusRecoverProfileRequest{ + Header: *header, + Iccid: iccid, + ProfileStatus: targetState, + } + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) +} + + +func CancelOrder(client *Es2PlusClient, iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { + result := new(ES2PlusCancelOrderResponse) + es2plusCommand := "cancelOrder" + header := newEs2plusHeader(client) + payload := &ES2PlusCancelOrderRequest { + Header: *header, + Iccid: iccid, + FinalProfileStatusIndicator: targetState, + } + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) +} + +func DownloadOrder(client *Es2PlusClient, iccid string) (*ES2PlusDownloadOrderResponse, error) { + result := new(ES2PlusDownloadOrderResponse) + es2plusCommand := "downloadOrder" + header := newEs2plusHeader(client) + payload := &ES2PlusDownloadOrderRequest { + Header: *header, + Iccid: iccid, + Eid: "", + Profiletype: "", + } + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + if ("Executed-Success" != executionStatus) { + return result, errors.New(fmt.Sprintf("ExecutionStatus was: ''%s'", executionStatus)) + } else { + return result, nil + } +} + + +func ConfirmOrder(client *Es2PlusClient, iccid string) (*ES2PlusConfirmOrderResponse, error) { + result := new(ES2PlusConfirmOrderResponse) + es2plusCommand := "confirmOrder" + header := newEs2plusHeader(client) + payload := &ES2PlusConfirmOrderRequest { + Header: *header, + Iccid: iccid, + Eid: "", + ConfirmationCode: "", + MatchingId: "", + SmdpAddress: "", + ReleaseFlag: true, + } + + err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + if ("Executed-Success" != executionStatus) { + return result, errors.New(fmt.Sprintf("ExecutionStatus was: ''%s'", executionStatus)) + } else { + return result, nil + } +} diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go new file mode 100755 index 000000000..2ed367a28 --- /dev/null +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -0,0 +1,130 @@ +//usr/bin/env go run "$0" "$@"; exit "$?" +package main + +import ( + "flag" + "fmt" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" +) + +/// +/// Main. The rest should be put into a library. +/// + +func main() { + + certFilePath := flag.String("cert", "", "Certificate pem file.") + keyFilePath := flag.String("key", "", "Certificate key file.") + hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") + requesterId := flag.String("requesterid", "", "ES2+ requester ID.") + iccidInput := flag.String("iccid", "", "Iccid of profile to manipulate") + + + flag.Parse() + + fmt.Printf("certFilePath = '%s'\n", *certFilePath) + fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) + fmt.Printf("hostport = '%s'\n", *hostport) + fmt.Printf("requesterId = '%s'\n", *requesterId) + fmt.Printf("iccidInput = '%s'\n", *iccidInput) + + iccid := *iccidInput + + client := es2plus.Client(*certFilePath, *keyFilePath, *hostport, *requesterId) + + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result1 -> ", result.State) + + result2, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") + if err != nil { + panic(err) + } + + fmt.Println("result2 -> ", result2) + + result, err = es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + fmt.Println("result3 -> ", result.State) + + + result4, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") + if err != nil { + panic(err) + } + + fmt.Println("result4 -> ", result4) + + result, err = es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + fmt.Println("result5 -> ", result.State) + + if result.State != "AVAILABLE" { + panic("Couldn't convert state of iccid into AVAILABLE") + } + + + result6, err := es2plus.DownloadOrder(client, iccid) + fmt.Println("result6 -> ", result6) + if err != nil { + panic(err) + } + + + result7, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result7 -> ", result7) + + + result8, err := es2plus.ConfirmOrder(client, iccid) + fmt.Println("result8 -> ", result8) + if err != nil { + panic(err) + } + + + result9, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result9 -> ", result9) + + /** + // TODO: Generate a full roundtrip taking some suitable profile through a proper + // activation, and reset. + result, err := es2plus.Activate(client, iccid) + if err != nil { + panic(err) + } + + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + // Make some assertion about the status at this point + result, err := es2plus.Reset(client, iccid) + if err != nil { + panic(err) + } + + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + + // Make some assertion about the status at this point +*/ + + fmt.Println("Success") +} From 4b50e282ab1553aa507f2991aff375bab241d07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 16:30:23 +0100 Subject: [PATCH 049/309] Removing dead code, adding a final test --- .../sim-batch-management/es2plus_smoketest.go | 39 ++++++------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 2ed367a28..667bf1c48 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -7,10 +7,16 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" ) -/// -/// Main. The rest should be put into a library. -/// - +// +// This is a "smoketest", a program that is formulated somewhat as a test, +// and I've used to develop the es2plus library. It is not formulated as a +// unit test, since it requires access to an external resource and I don't feel it +// is appropriate to run unit tests against that external resource. Some degree of +// deliberation seems appropriate. +// +// The program illustrates the API being used, and logs progress in the state of the +// profile being manipulated. +// func main() { certFilePath := flag.String("cert", "", "Certificate pem file.") @@ -99,32 +105,11 @@ func main() { } fmt.Println("result9 -> ", result9) - /** - // TODO: Generate a full roundtrip taking some suitable profile through a proper - // activation, and reset. - result, err := es2plus.Activate(client, iccid) - if err != nil { - panic(err) - } - - result, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - - // Make some assertion about the status at this point - result, err := es2plus.Reset(client, iccid) - if err != nil { - panic(err) - } - result, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) + if result.State != "RELEASED" { + panic("Couldn't convert state of iccid into RELEASED") } - // Make some assertion about the status at this point -*/ fmt.Println("Success") } From e59860fbb70911ebb82041a15cbee26fe821c7fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 16:31:27 +0100 Subject: [PATCH 050/309] Fix bug --- go.mod | 1 + go.sum | 2 ++ sim-administration/sim-batch-management/es2plus_smoketest.go | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8ea2428ce..be36069f4 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.13 require ( github.com/google/go-cmp v0.3.1 // indirect + github.com/google/uuid v1.1.1 github.com/pkg/errors v0.8.1 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 802d8ce53..bbd920b61 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 667bf1c48..f6e9eb5e8 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -106,7 +106,7 @@ func main() { fmt.Println("result9 -> ", result9) - if result.State != "RELEASED" { + if result9.State != "RELEASED" { panic("Couldn't convert state of iccid into RELEASED") } From b20c6c195d5f2a00efe704f59ac50eaf478ef28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 18:21:49 +0100 Subject: [PATCH 051/309] Getting started with kingpin --- go.mod | 3 ++ go.sum | 12 ++++++ .../sim-batch-management/es2plus_smoketest.go | 32 ++++++++++++---- .../kingpin-exploration.go | 37 +++++++++++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) create mode 100755 sim-administration/sim-batch-management/kingpin-exploration.go diff --git a/go.mod b/go.mod index 7d5540395..a71ad5ff5 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module github.com/ostelco/ostelco-core go 1.13 require ( + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/google/uuid v1.1.1 github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v1.11.0 github.com/pkg/errors v0.8.1 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/go.sum b/go.sum index 854b55ca1..e75ce857b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,8 @@ +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -11,5 +16,12 @@ github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 2ed367a28..47b64717e 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -2,7 +2,6 @@ package main import ( - "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" ) @@ -11,16 +10,33 @@ import ( /// Main. The rest should be put into a library. /// -func main() { - certFilePath := flag.String("cert", "", "Certificate pem file.") - keyFilePath := flag.String("key", "", "Certificate key file.") - hostport := flag.String("hostport", "", "host:port of ES2+ endpoint.") - requesterId := flag.String("requesterid", "", "ES2+ requester ID.") - iccidInput := flag.String("iccid", "", "Iccid of profile to manipulate") +import "gopkg.in/alecthomas/kingpin.v2" + +var ( + debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() + + smoketest = kingpin.Command("es2plusSmoketest", "Register a new user.") + smoketestCertFilePath = smoketest.Arg("cert", "Certificate pem file.").Required().String() + smoketestKeyFilePath = smoketest.Arg("key", "Certificate key file.").Required().String() + smoketestHostport = smoketest.Arg("hostport", "host:port of ES2+ endpoint.").Required().String() + smoketestRequesterId = smoketest.Arg("requesterid", "ES2+ requester ID.").Required().String() + smoketestIccidInput = smoketest.Arg("iccid", "Iccid of profile to manipulate").Required().String() + + // TODO: Check if this can be used for the key files. + // postImage = post.Flag("image", "image to post").ExistingFile() +) +func main() { + switch kingpin.Parse() { + + // Handle the command + case "smoketest": + es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) + } +} - flag.Parse() +func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { fmt.Printf("certFilePath = '%s'\n", *certFilePath) fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) diff --git a/sim-administration/sim-batch-management/kingpin-exploration.go b/sim-administration/sim-batch-management/kingpin-exploration.go new file mode 100755 index 000000000..198672a4f --- /dev/null +++ b/sim-administration/sim-batch-management/kingpin-exploration.go @@ -0,0 +1,37 @@ +//usr/bin/env go run "$0" "$@"; exit "$?" + +package main + +import "gopkg.in/alecthomas/kingpin.v2" + +var ( + debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() + serverIP = kingpin.Flag("server", "server address").Default("127.0.0.1").IP() + + register = kingpin.Command("register", "Register a new user.") + registerNick = register.Arg("nick", "nickname for user").Required().String() + registerName = register.Arg("name", "name of user").Required().String() + + post = kingpin.Command("post", "Post a message to a channel.") + postImage = post.Flag("image", "image to post").ExistingFile() + postChannel = post.Arg("channel", "channel to post to").Required().String() + postText = post.Arg("text", "text to post").String() +) + +func main() { + switch kingpin.Parse() { + // Register user + case "register": + println("register command") + + // Post message + case "post": + println("post command") + /* + if *postImage != nil { + } + if *postText != "" { + } + */ + } +} From daddf3179db83bd9c4eb3e8d6d6ef2bcc02d1868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 18:22:06 +0100 Subject: [PATCH 052/309] Deletingm unused file --- .../kingpin-exploration.go | 37 ------------------- 1 file changed, 37 deletions(-) delete mode 100755 sim-administration/sim-batch-management/kingpin-exploration.go diff --git a/sim-administration/sim-batch-management/kingpin-exploration.go b/sim-administration/sim-batch-management/kingpin-exploration.go deleted file mode 100755 index 198672a4f..000000000 --- a/sim-administration/sim-batch-management/kingpin-exploration.go +++ /dev/null @@ -1,37 +0,0 @@ -//usr/bin/env go run "$0" "$@"; exit "$?" - -package main - -import "gopkg.in/alecthomas/kingpin.v2" - -var ( - debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - serverIP = kingpin.Flag("server", "server address").Default("127.0.0.1").IP() - - register = kingpin.Command("register", "Register a new user.") - registerNick = register.Arg("nick", "nickname for user").Required().String() - registerName = register.Arg("name", "name of user").Required().String() - - post = kingpin.Command("post", "Post a message to a channel.") - postImage = post.Flag("image", "image to post").ExistingFile() - postChannel = post.Arg("channel", "channel to post to").Required().String() - postText = post.Arg("text", "text to post").String() -) - -func main() { - switch kingpin.Parse() { - // Register user - case "register": - println("register command") - - // Post message - case "post": - println("post command") - /* - if *postImage != nil { - } - if *postText != "" { - } - */ - } -} From 3309aa09f222c2aa1146c8b8e44fc5cf8da508ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 19:41:45 +0100 Subject: [PATCH 053/309] Modify smoketest to use kingpin command line parsing library, this is in preparation of morphing the app into a central app for all the various command line utilities I've been planning to make --- .../sim-batch-management/es2plus_smoketest.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 47b64717e..7d37a6860 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -4,24 +4,22 @@ package main import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" + "gopkg.in/alecthomas/kingpin.v2" ) /// /// Main. The rest should be put into a library. /// - -import "gopkg.in/alecthomas/kingpin.v2" - var ( debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - smoketest = kingpin.Command("es2plusSmoketest", "Register a new user.") - smoketestCertFilePath = smoketest.Arg("cert", "Certificate pem file.").Required().String() - smoketestKeyFilePath = smoketest.Arg("key", "Certificate key file.").Required().String() - smoketestHostport = smoketest.Arg("hostport", "host:port of ES2+ endpoint.").Required().String() - smoketestRequesterId = smoketest.Arg("requesterid", "ES2+ requester ID.").Required().String() - smoketestIccidInput = smoketest.Arg("iccid", "Iccid of profile to manipulate").Required().String() + smoketest = kingpin.Command("es2plusSmoketest", "Smoketest the ES2+ protocol.") + smoketestCertFilePath = smoketest.Flag("cert", "Certificate pem file.").Required().String() + smoketestKeyFilePath = smoketest.Flag("key", "Certificate key file.").Required().String() + smoketestHostport = smoketest.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() + smoketestRequesterId = smoketest.Flag("requesterid", "ES2+ requester ID.").Required().String() + smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() // TODO: Check if this can be used for the key files. // postImage = post.Flag("image", "image to post").ExistingFile() @@ -31,7 +29,7 @@ func main() { switch kingpin.Parse() { // Handle the command - case "smoketest": + case "es2plusSmoketest": es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) } } From 55cc25a8528b1ce44325bb4c95513b9549456496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 20:14:03 +0100 Subject: [PATCH 054/309] Extend the es2plus_smoketest.go command to also do conversion of .out files into hss input files. --- .../sim-batch-management/es2plus_smoketest.go | 71 ++++++++----------- .../outfile_to_hss_input_converter.go | 11 +-- .../outfile_to_hss_input_converter_lib.go | 19 +++-- 3 files changed, 46 insertions(+), 55 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 7d37a6860..ed93c7c89 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -4,6 +4,7 @@ package main import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "gopkg.in/alecthomas/kingpin.v2" ) @@ -12,14 +13,28 @@ import ( /// var ( - debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - - smoketest = kingpin.Command("es2plusSmoketest", "Smoketest the ES2+ protocol.") - smoketestCertFilePath = smoketest.Flag("cert", "Certificate pem file.").Required().String() - smoketestKeyFilePath = smoketest.Flag("key", "Certificate key file.").Required().String() - smoketestHostport = smoketest.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() - smoketestRequesterId = smoketest.Flag("requesterid", "ES2+ requester ID.").Required().String() - smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() + // TODO: Enable, but also make it have an effect. + // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() + + // + // Smoketest of the ES2Plus interface + // + smoketest = kingpin.Command("es2plus-smoketest", "Smoketest the ES2+ protocol.") + smoketestCertFilePath = smoketest.Flag("cert", "Certificate pem file.").Required().String() + smoketestKeyFilePath = smoketest.Flag("key", "Certificate key file.").Required().String() + smoketestHostport = smoketest.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() + smoketestRequesterId = smoketest.Flag("requesterid", "ES2+ requester ID.").Required().String() + smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() + + // + // Convert an output (.out) file from an sim profile producer into an input file + // for Prime. + // + spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spUploadInputFile = smoketest.Flag("input-file", "path to .out file used as input file").Required().String() + + spUploadOutputFiePrefix = smoketest.Flag("output-file-prefix", + "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() // TODO: Check if this can be used for the key files. // postImage = post.Flag("image", "image to post").ExistingFile() @@ -27,13 +42,17 @@ var ( func main() { switch kingpin.Parse() { - - // Handle the command case "es2plusSmoketest": es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) + case "simProfilerUpload": + outfileconversion.ConvertInputfileToOutputfile(*inputFile, *outputFilePrefix) + default: + panic("Unknown command") } } +// TODO: Replace this with a set of commands that are directly useful as +// an alternative to the web interface. func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { fmt.Printf("certFilePath = '%s'\n", *certFilePath) @@ -66,7 +85,6 @@ func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *strin fmt.Println("result3 -> ", result.State) - result4, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") if err != nil { panic(err) @@ -85,60 +103,33 @@ func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *strin panic("Couldn't convert state of iccid into AVAILABLE") } - result6, err := es2plus.DownloadOrder(client, iccid) fmt.Println("result6 -> ", result6) if err != nil { panic(err) } - result7, err := es2plus.GetStatus(client, iccid) if err != nil { panic(err) } fmt.Println("result7 -> ", result7) - result8, err := es2plus.ConfirmOrder(client, iccid) fmt.Println("result8 -> ", result8) if err != nil { panic(err) } - result9, err := es2plus.GetStatus(client, iccid) if err != nil { panic(err) } fmt.Println("result9 -> ", result9) - /** - // TODO: Generate a full roundtrip taking some suitable profile through a proper - // activation, and reset. - result, err := es2plus.Activate(client, iccid) - if err != nil { - panic(err) - } - - result, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - - // Make some assertion about the status at this point - result, err := es2plus.Reset(client, iccid) - if err != nil { - panic(err) - } - - result, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) + if result.State != "RELEASED" { + panic("Couldn't convert state of sim profile into RELEASED") } - // Make some assertion about the status at this point -*/ - fmt.Println("Success") } diff --git a/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go b/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go index f94fc345c..f920e1940 100755 --- a/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go +++ b/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go @@ -31,7 +31,6 @@ package main import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" - "log" ) func main() { @@ -39,13 +38,5 @@ func main() { fmt.Println("inputFile = ", inputFile) fmt.Println("outputFilePrefix = ", outputFilePrefix) - - outRecord := outfileconversion.ReadOutputFile(inputFile) - outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" - fmt.Println("outputFile = ", outputFile) - - err := outfileconversion.WriteHssCsvFile(outputFile, outRecord.Entries) - if err != nil { - log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") - } + outfileconversion.ConvertInputfileToOutputfile(inputFile, outputFilePrefix) } diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go index ea585148c..28c558eb9 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go @@ -21,12 +21,10 @@ const ( UNKNOWN_HEADER = "unknown" ) - /// /// Functions /// - func ParseOutputToHssConverterCommandLine() (string, string) { inputFile := flag.String("input-file", "not a valid filename", @@ -53,7 +51,6 @@ func ParseLineIntoKeyValueMap(line string, theMap map[string]string) { theMap[key] = value } - type ParserState struct { currentState string inputVariables map[string]string @@ -235,8 +232,6 @@ func isComment(s string) bool { return match } - - // fileExists checks if a file exists and is not a directory before we // try using it to prevent further errors. func fileExists(filename string) bool { @@ -277,3 +272,17 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { return f.Close() } +// +// Entrypoint +// + +func ConvertInputfileToOutputfile(inputFile string, outputFilePrefix string) { + outRecord := outfileconversion.ReadOutputFile(inputFile) + outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" + fmt.Println("outputFile = ", outputFile) + + err := outfileconversion.WriteHssCsvFile(outputFile, outRecord.Entries) + if err != nil { + log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") + } +} From c52acbcc5d937dc37dc4e789b048e088c8581461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 20:14:49 +0100 Subject: [PATCH 055/309] update comment --- sim-administration/sim-batch-management/es2plus_smoketest.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index ed93c7c89..3bb33e711 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" - "gopkg.in/alecthomas/kingpin.v2" ) /// @@ -52,7 +51,7 @@ func main() { } // TODO: Replace this with a set of commands that are directly useful as -// an alternative to the web interface. +// an alternative to the web interface (update, prepare, query, etc.) func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { fmt.Printf("certFilePath = '%s'\n", *certFilePath) From 0f1f9a30c9cc0e4ff5e2711e2acf829045e88d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 20:18:38 +0100 Subject: [PATCH 056/309] Delete ole entrypoint for .out to hss input file conversion. Replace with command in combined commandline interface. --- .../sim-batch-management/es2plus_smoketest.go | 33 ++++++++++++++- .../outfile_to_hss_input_converter.go | 42 ------------------- 2 files changed, 32 insertions(+), 43 deletions(-) delete mode 100755 sim-administration/sim-batch-management/outfile_to_hss_input_converter.go diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/es2plus_smoketest.go index 3bb33e711..5962c695d 100755 --- a/sim-administration/sim-batch-management/es2plus_smoketest.go +++ b/sim-administration/sim-batch-management/es2plus_smoketest.go @@ -29,6 +29,37 @@ var ( // Convert an output (.out) file from an sim profile producer into an input file // for Prime. // + + /** + * OLD COMMENTS: Not yet reworked into doc for this script, but mostly accurate + * nonetheless. + * + * This program is intended to be used from the command line, and will convert an + * output file from a sim card vendor into an input file for a HSS. The assumptions + * necessary for this to work are: + * + * * The SIM card vendor produces output files similar to the example .out file + * found in the same source directory as this program + * + * * The HSS accepts input as a CSV file, with header line 'ICCID, IMSI, KI' and subsequent + * lines containing ICCID/IMSI/Ki fields, all separated by commas. + * + * Needless to say, the outmost care should be taken when handling Ki values and + * this program must, as a matter of course, be considered a security risk, as + * must all software that touch SIM values. + * + * With that caveat in place, the usage of this program typically looks like + * this: + * + * ./outfile_to_hss_input_converter.go \ + * -input-file sample_out_file_for_testing.out + * -output-file-prefix ./hss-input-for- + * + * (followed by cryptographically strong erasure of the .out file, + * encapsulation of the .csv file in strong cryptography etc., none + * of which are handled by this script). + */ + spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") spUploadInputFile = smoketest.Flag("input-file", "path to .out file used as input file").Required().String() @@ -46,7 +77,7 @@ func main() { case "simProfilerUpload": outfileconversion.ConvertInputfileToOutputfile(*inputFile, *outputFilePrefix) default: - panic("Unknown command") + panic("Unknown command.") } } diff --git a/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go b/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go deleted file mode 100755 index f920e1940..000000000 --- a/sim-administration/sim-batch-management/outfile_to_hss_input_converter.go +++ /dev/null @@ -1,42 +0,0 @@ -//usr/bin/env go run "$0" "$@"; exit "$?" -/** - * This program is intended to be used from the command line, and will convert an - * output file from a sim card vendor into an input file for a HSS. The assumptions - * necessary for this to work are: - * - * * The SIM card vendor produces output files similar to the example .out file - * found in the same source directory as this program - * - * * The HSS accepts input as a CSV file, with header line 'ICCID, IMSI, KI' and subsequent - * lines containing ICCID/IMSI/Ki fields, all separated by commas. - * - * Needless to say, the outmost care should be taken when handling Ki values and - * this program must, as a matter of course, be considered a security risk, as - * must all software that touch SIM values. - * - * With that caveat in place, the usage of this program typically looks like - * this: - * - * ./outfile_to_hss_input_converter.go \ - * -input-file sample_out_file_for_testing.out - * -output-file-prefix ./hss-input-for- - * - * (followed by cryptographically strong erasure of the .out file, - * encapsulation of the .csv file in strong cryptography etc., none - * of which are handled by this script). - */ - -package main - -import ( - "fmt" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" -) - -func main() { - inputFile, outputFilePrefix := outfileconversion.ParseOutputToHssConverterCommandLine() - - fmt.Println("inputFile = ", inputFile) - fmt.Println("outputFilePrefix = ", outputFilePrefix) - outfileconversion.ConvertInputfileToOutputfile(inputFile, outputFilePrefix) -} From 29c83b32cb2b937eb74ec2c3e8d2197fda9e1ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 30 Oct 2019 20:21:40 +0100 Subject: [PATCH 057/309] Rename entrypoint to one more fitting the new role (sim batch mangement, not smoketest for es2+) --- .../{es2plus_smoketest.go => sim-batch-mgt.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sim-administration/sim-batch-management/{es2plus_smoketest.go => sim-batch-mgt.go} (100%) diff --git a/sim-administration/sim-batch-management/es2plus_smoketest.go b/sim-administration/sim-batch-management/sim-batch-mgt.go similarity index 100% rename from sim-administration/sim-batch-management/es2plus_smoketest.go rename to sim-administration/sim-batch-management/sim-batch-mgt.go From daa808566d9be58cb4b793e3166931d23bb686e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 13:50:45 +0100 Subject: [PATCH 058/309] Whitespace --- sim-administration/sim-batch-management/sim-batch-mgt.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 5962c695d..ca1a7bcbb 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -62,7 +62,6 @@ var ( spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") spUploadInputFile = smoketest.Flag("input-file", "path to .out file used as input file").Required().String() - spUploadOutputFiePrefix = smoketest.Flag("output-file-prefix", "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() From e6c6fe0345664a6972f1252f7ce92b5bb9b30436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 14:26:41 +0100 Subject: [PATCH 059/309] Adding the functionality of the upload-sim-mgt.go to sim-batch-mgt.go --- .../sim-batch-management/sim-batch-mgt.go | 54 ++++++++++++++++++- .../uploadtoprime/upload-sim-batch-lib.go | 44 ++++++++++++--- 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index ca1a7bcbb..6de384b89 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -5,6 +5,8 @@ import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" + "gopkg.in/alecthomas/kingpin.v2" ) /// @@ -60,13 +62,42 @@ var ( * of which are handled by this script). */ - spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spUploadInputFile = smoketest.Flag("input-file", "path to .out file used as input file").Required().String() + spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spUploadInputFile = smoketest.Flag("input-file", "path to .out file used as input file").Required().String() spUploadOutputFiePrefix = smoketest.Flag("output-file-prefix", "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() // TODO: Check if this can be used for the key files. // postImage = post.Flag("image", "image to post").ExistingFile() + + // + // Upload of batch data to Prime + // + pbu = kingpin.Command("prime-batch-upload", "Generate a shellscript that will upload a batch of profiles to Prime.") + pbuFirstIccid = pbu.Flag("first-rawIccid", + "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() + pbuLastIccid = pbu.Flag("last-rawIccid", + "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() + pbuFirstIMSI = pbu.Flag("first-imsi", "First IMSI in batch").Required().String() + pbuLastIMSI = pbu.Flag("last-imsi", "Last IMSI in batch").Required().String() + pbuFirstMsisdn = pbu.Flag("first-msisdn", "First MSISDN in batch").Required().String() + pbuLastMsisdn = pbu.Flag("last-msisdn", "Last MSISDN in batch").Required().String() + pbuProfileType = pbu.Flag("profile-type", "SIM profile type").Required().String() + pbuBatchLengthString = pbu.Flag( + "batch-Quantity", + "Number of sim cards in batch").Required().String() + + // XXX Legal values are Loltel and M1 at this time, how to configure that + // flexibly? Eventually by putting them in a database and consulting it during + // command execution, but for now, just by squinting. + + pbuHssVendor = pbu.Flag("hss-vendor", "The HSS vendor").Required().Default("M1").String() + pbuUploadHostname = pbu.Flag("upload-hostname", "host to upload batch to").Required().Default("localhost").String() + pbuUploadPortnumber = pbu.Flag("upload-portnumber", "port to upload to").Required().Default("8080").String() + pbuProfileVendor = pbu.Flag("profile-vendor", "Vendor of SIM profiles").Required().Default("Idemia").String() + pbuInitialHlrActivationStatusOfProfiles = pbu.Flag( + "initial-hlr-activation-status-of-profiles", + "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Required().Default("ACTIVATED").String() ) func main() { @@ -75,6 +106,25 @@ func main() { es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) case "simProfilerUpload": outfileconversion.ConvertInputfileToOutputfile(*inputFile, *outputFilePrefix) + case "sim-profile-upload": + // TODO: Combine these two into something inside uploadtoprime. + // It's unecessary to break the batch thingy open in this way. + var batch = uploadtoprime.OutputBatchFromCommandLineParameters( + pbuFirstIccid, + pbuLastIccid, + pbuLastIMSI, + pbuFirstIMSI, + pbuLastMsisdn, + pbuFirstMsisdn, + pbuUploadHostname, + pbuUploadPortnumber, + pbuHssVendor, + pbuInitialHlrActivationStatusOfProfiles, + pbuProfileType, + pbuProfileVendor, + pbuBatchLengthString) + var csvPayload = uploadtoprime.GenerateCsvPayload(batch) + uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) default: panic("Unknown command.") } diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index 40cc3f93c..4a64ec445 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -147,8 +147,6 @@ func checkProfileType(name string, potentialProfileName string) { } } - - func IccidWithoutLuhnChecksum(s string) string { return loltelutils.TrimSuffix(s, 1) } @@ -198,6 +196,36 @@ func ParseUploadFileGeneratorCommmandline() model.OutputBatch { // flag.Parse() + return OutputBatchFromCommandLineParameters( + firstIccid, + lastIccid, + lastIMSI, + firstIMSI, + lastMsisdn, + firstMsisdn, + uploadHostname, + uploadPortnumber, + hssVendor, + initialHlrActivationStatusOfProfiles, + profileType, + profileVendor, + batchLengthString) +} + +func OutputBatchFromCommandLineParameters(firstIccid *string, + lastIccid *string, + lastIMSI *string, + firstIMSI *string, + lastMsisdn *string, + firstMsisdn *string, + uploadHostname *string, + uploadPortnumber *string, + hssVendor *string, + initialHlrActivationStatusOfProfiles *string, + profileType *string, + profileVendor *string, + batchLengthString *string) model.OutputBatch { + // // Check parameters for syntactic correctness and // semantic sanity. @@ -289,13 +317,13 @@ func ParseInputFileGeneratorCommmandline() model.InputBatch { // we need to up our Go-Fu before we can make flag.Parse(arguments) work return model.InputBatch{ - Customer: "Footel", + Customer: "Footel", ProfileType: "BAR_FOOTEL_STD", - OrderDate: "20191007", - BatchNo: "2019100701", - Quantity: 10, - FirstIccid: 894700000000002214, - FirstImsi: 242017100012213} + OrderDate: "20191007", + BatchNo: "2019100701", + Quantity: 10, + FirstIccid: 894700000000002214, + FirstImsi: 242017100012213} } func GenerateInputFile(batch model.InputBatch) string { From c9745580e800372fa488f9f909127128538df99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 14:27:04 +0100 Subject: [PATCH 060/309] delete now obsolete upload-sim-batch.go goscript. --- .../sim-batch-management/upload-sim-batch.go | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100755 sim-administration/sim-batch-management/upload-sim-batch.go diff --git a/sim-administration/sim-batch-management/upload-sim-batch.go b/sim-administration/sim-batch-management/upload-sim-batch.go deleted file mode 100755 index 71e5b93b2..000000000 --- a/sim-administration/sim-batch-management/upload-sim-batch.go +++ /dev/null @@ -1,15 +0,0 @@ -//usr/bin/env go run "$0" "$@"; exit "$?" - -package main - -import "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - -func main() { - batch := uploadtoprime.ParseUploadFileGeneratorCommmandline() - - // TODO: Combine these two into something inside uploadtoprime. - // It's unecessary to break the batch thingy open in this way. - var csvPayload = uploadtoprime.GenerateCsvPayload(batch) - - uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) -} \ No newline at end of file From 40f17a622ddb00475cec744923d5adda8e2d10b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 14:29:17 +0100 Subject: [PATCH 061/309] nuke comment that hasn't aged well. --- sim-administration/sim-batch-management/sim-batch-mgt.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 6de384b89..e63965044 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -9,10 +9,6 @@ import ( "gopkg.in/alecthomas/kingpin.v2" ) -/// -/// Main. The rest should be put into a library. -/// - var ( // TODO: Enable, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() From 52ce9061cedb4156e72398d0733c2fc267ced016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 15:08:17 +0100 Subject: [PATCH 062/309] Cleaning up fallout from previous refactoring, also adding es2+ commands to the mix so that individual profiles can be manipulated --- go.mod | 2 +- .../outfile_to_hss_input_converter_lib.go | 4 +- .../sim-batch-management/sim-batch-mgt.go | 84 ++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index a71ad5ff5..97997727f 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,6 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v1.11.0 github.com/pkg/errors v0.8.1 // indirect - gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect + gopkg.in/alecthomas/kingpin.v2 v2.2.6 gotest.tools v2.2.0+incompatible ) diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go index 28c558eb9..af6c1f6bc 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go @@ -277,11 +277,11 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { // func ConvertInputfileToOutputfile(inputFile string, outputFilePrefix string) { - outRecord := outfileconversion.ReadOutputFile(inputFile) + outRecord := ReadOutputFile(inputFile) outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" fmt.Println("outputFile = ", outputFile) - err := outfileconversion.WriteHssCsvFile(outputFile, outRecord.Entries) + err := WriteHssCsvFile(outputFile, outRecord.Entries) if err != nil { log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index e63965044..ebdd9690f 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -23,6 +23,17 @@ var ( smoketestRequesterId = smoketest.Flag("requesterid", "ES2+ requester ID.").Required().String() smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() + + es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") + es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile ").Required().String() + es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() + es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Required().String() + es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() + es2KeyFilePath = es2.Flag("key", "Certificate key file.").Required().String() + es2Hostport = es2.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() + es2RequesterId = es2.Flag("requesterid", "ES2+ requester ID.").Required().String() + + // // Convert an output (.out) file from an sim profile producer into an input file // for Prime. @@ -59,8 +70,8 @@ var ( */ spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spUploadInputFile = smoketest.Flag("input-file", "path to .out file used as input file").Required().String() - spUploadOutputFiePrefix = smoketest.Flag("output-file-prefix", + spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() + spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() // TODO: Check if this can be used for the key files. @@ -87,21 +98,23 @@ var ( // flexibly? Eventually by putting them in a database and consulting it during // command execution, but for now, just by squinting. - pbuHssVendor = pbu.Flag("hss-vendor", "The HSS vendor").Required().Default("M1").String() - pbuUploadHostname = pbu.Flag("upload-hostname", "host to upload batch to").Required().Default("localhost").String() - pbuUploadPortnumber = pbu.Flag("upload-portnumber", "port to upload to").Required().Default("8080").String() - pbuProfileVendor = pbu.Flag("profile-vendor", "Vendor of SIM profiles").Required().Default("Idemia").String() + pbuHssVendor = pbu.Flag("hss-vendor", "The HSS vendor").Default("M1").String() + pbuUploadHostname = pbu.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() + pbuUploadPortnumber = pbu.Flag("upload-portnumber", "port to upload to").Default("8080").String() + pbuProfileVendor = pbu.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() pbuInitialHlrActivationStatusOfProfiles = pbu.Flag( "initial-hlr-activation-status-of-profiles", - "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Required().Default("ACTIVATED").String() + "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() ) func main() { - switch kingpin.Parse() { - case "es2plusSmoketest": + + cmd := kingpin.Parse() + switch cmd { + case "es2plus-smoketest": es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) case "simProfilerUpload": - outfileconversion.ConvertInputfileToOutputfile(*inputFile, *outputFilePrefix) + outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "sim-profile-upload": // TODO: Combine these two into something inside uploadtoprime. // It's unecessary to break the batch thingy open in this way. @@ -121,13 +134,59 @@ func main() { pbuBatchLengthString) var csvPayload = uploadtoprime.GenerateCsvPayload(batch) uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) + + case "es2": + client := es2plus.Client(*es2CertFilePath, *es2KeyFilePath, *es2Hostport, *es2RequesterId) + iccid := *es2iccid + switch *es2cmd { + case "get-status": + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result -> ", result.State) + case "recover-profile": + checkEs2TargetState(es2Target) + _, err := es2plus.RecoverProfile(client, iccid, *es2Target) + if err != nil { + panic(err) + } + case "download-order": + result, err := es2plus.DownloadOrder(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result -> ", result) + case "confirm-order": + result, err := es2plus.ConfirmOrder(client, iccid) + if err != nil { + panic(err) + } + fmt.Println("result -> ", result) + case "cancel-profile": + checkEs2TargetState(es2Target) + _, err := es2plus.CancelOrder(client, iccid, *es2Target) + if err != nil { + panic(err) + } + default: + panic("Unknown es2+ subcommand, try --help") + } default: - panic("Unknown command.") + panic(fmt.Sprintf("Unknown command: '%s'\n", cmd)) + } +} + +func checkEs2TargetState(target *string) { + if (*target != "AVAILABLE" ) { + panic("Target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") } } // TODO: Replace this with a set of commands that are directly useful as -// an alternative to the web interface (update, prepare, query, etc.) +// an alternative to the web interface (update, prepare, query, etc.). +// But do it only after a little more work has been done with respect to +// cacheing parameters etc, so that the command lines isn't becoming awful. func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { fmt.Printf("certFilePath = '%s'\n", *certFilePath) @@ -160,6 +219,7 @@ func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *strin fmt.Println("result3 -> ", result.State) + // XXX Should be CancelProfile result4, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") if err != nil { panic(err) From f1d02233999d5d4e9612799345650c8a38288558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 17:25:38 +0100 Subject: [PATCH 063/309] Just getting started on persistence --- .../sim-batch-management/sim-batch-mgt.go | 96 ++++++++++++++++--- 1 file changed, 81 insertions(+), 15 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index ebdd9690f..56442b35b 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -2,11 +2,14 @@ package main import ( + "database/sql" "fmt" + "github.com/jmoiron/sqlx" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" "gopkg.in/alecthomas/kingpin.v2" + "strconv" ) var ( @@ -23,17 +26,15 @@ var ( smoketestRequesterId = smoketest.Flag("requesterid", "ES2+ requester ID.").Required().String() smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() - - es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") - es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile ").Required().String() - es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() - es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Required().String() + es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") + es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile ").Required().String() + es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() + es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Required().String() es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() es2KeyFilePath = es2.Flag("key", "Certificate key file.").Required().String() es2Hostport = es2.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() es2RequesterId = es2.Flag("requesterid", "ES2+ requester ID.").Required().String() - // // Convert an output (.out) file from an sim profile producer into an input file // for Prime. @@ -69,8 +70,8 @@ var ( * of which are handled by this script). */ - spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() + spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() @@ -105,6 +106,8 @@ var ( pbuInitialHlrActivationStatusOfProfiles = pbu.Flag( "initial-hlr-activation-status-of-profiles", "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() + + batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") ) func main() { @@ -147,10 +150,11 @@ func main() { fmt.Println("result -> ", result.State) case "recover-profile": checkEs2TargetState(es2Target) - _, err := es2plus.RecoverProfile(client, iccid, *es2Target) + result, err := es2plus.RecoverProfile(client, iccid, *es2Target) if err != nil { panic(err) } + fmt.Println("result -> ", result) case "download-order": result, err := es2plus.DownloadOrder(client, iccid) if err != nil { @@ -172,21 +176,83 @@ func main() { default: panic("Unknown es2+ subcommand, try --help") } + case "batch": + fmt.Println("Doing the batch thing.") + doTheBatchThing() default: panic(fmt.Sprintf("Unknown command: '%s'\n", cmd)) } } -func checkEs2TargetState(target *string) { - if (*target != "AVAILABLE" ) { +func checkEs2TargetState(target *string) { + if *target != "AVAILABLE" { panic("Target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") } } -// TODO: Replace this with a set of commands that are directly useful as -// an alternative to the web interface (update, prepare, query, etc.). -// But do it only after a little more work has been done with respect to -// cacheing parameters etc, so that the command lines isn't becoming awful. +// Sqlx https://jmoiron.github.io/sqlx/ +// Person represents a person. +type Person struct { + ID int `db:"id" json:"id"` + Firstname string `db:"firstname" json:"firstname"` + Lastname string `db:"lastname" json:"lastname"` +} + +func doTheBatchThing() { + + fmt.Println("The batching getting started") + + // Get a reference to the database, create a table if it + // doesn't exist already. + + var db *sqlx.DB + + // exactly the same as the built-in + db, err := sqlx.Open("sqlite3", "./nraboy.db") + + database, err := sql.Open("sqlite3", "./nraboy.db") + if err != nil { + fmt.Errorf("open sql", err) + } + + // + statement, _ := database.Prepare("CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT)") + _, err = statement.Exec() + if err != nil { + fmt.Errorf("Failed to create table: ", err) + } + + // Insert a row. + statement, _ = database.Prepare("INSERT INTO people (firstname, lastname) VALUES (?, ?)") + _, err = statement.Exec("Nic", "Raboy") + if err != nil { + fmt.Errorf("Failed to insert row: ", err) + } + + // Query all the rows. + rows, _ := database.Query("SELECT id, firstname, lastname FROM people") + var id int + var firstname string + var lastname string + for rows.Next() { + rows.Scan(&id, &firstname, &lastname) + if err != nil { + fmt.Errorf("Failed to scan row: ", err) + } + fmt.Println(strconv.Itoa(id) + ": " + firstname + " " + lastname) + } + + fmt.Print("Foo->") + rowz, err := db.Queryx("SELECT * FROM people") + for rowz.Next() { + var p Person + err = rowz.StructScan(&p) + fmt.Println("The p = ", p) + } +} + +// TODO: This is just smoketest-code, delete it after +// the smoketest-script has been rewritten a bit to do the same thing. func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { fmt.Printf("certFilePath = '%s'\n", *certFilePath) From c1e1325a38cea9c58a63d0e08667d188a981bf2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 17:42:05 +0100 Subject: [PATCH 064/309] Starting to work on a test-driven persistence layer development. --- .../sim-batch-management/model/model.go | 7 +- .../sim-batch-management/sim-batch-mgt.go | 64 +---------------- .../sim-batch-management/storage/storage.go | 69 +++++++++++++++++++ .../storage/storage_test.go | 23 +++++++ 4 files changed, 96 insertions(+), 67 deletions(-) create mode 100644 sim-administration/sim-batch-management/storage/storage.go create mode 100644 sim-administration/sim-batch-management/storage/storage_test.go diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 6a92bbcf6..81d04006e 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -4,8 +4,8 @@ package model // package, then build a DAO interface a "store" package type OutputBatch struct { - ProfileType string - Url string + ProfileType string `db:"profileType" json:"profileType"` + Url string `db:"url" json:"url"` Length int FirstMsisdn int MsisdnIncrement int @@ -25,7 +25,6 @@ type InputBatch struct { FirstImsi int } - type OutputFileRecord struct { Filename string InputVariables map[string]string @@ -42,7 +41,6 @@ type OutputFileRecord struct { OutputFileName string } - type SimEntry struct { RawIccid string IccidWithChecksum string @@ -51,4 +49,3 @@ type SimEntry struct { Ki string OutputFileName string } - diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 56442b35b..deeb8e52c 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -7,6 +7,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/storage" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" "gopkg.in/alecthomas/kingpin.v2" "strconv" @@ -178,7 +179,7 @@ func main() { } case "batch": fmt.Println("Doing the batch thing.") - doTheBatchThing() + storage.doTheBatchThing() default: panic(fmt.Sprintf("Unknown command: '%s'\n", cmd)) } @@ -190,67 +191,6 @@ func checkEs2TargetState(target *string) { } } -// Sqlx https://jmoiron.github.io/sqlx/ -// Person represents a person. -type Person struct { - ID int `db:"id" json:"id"` - Firstname string `db:"firstname" json:"firstname"` - Lastname string `db:"lastname" json:"lastname"` -} - -func doTheBatchThing() { - - fmt.Println("The batching getting started") - - // Get a reference to the database, create a table if it - // doesn't exist already. - - var db *sqlx.DB - - // exactly the same as the built-in - db, err := sqlx.Open("sqlite3", "./nraboy.db") - - database, err := sql.Open("sqlite3", "./nraboy.db") - if err != nil { - fmt.Errorf("open sql", err) - } - - // - statement, _ := database.Prepare("CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT)") - _, err = statement.Exec() - if err != nil { - fmt.Errorf("Failed to create table: ", err) - } - - // Insert a row. - statement, _ = database.Prepare("INSERT INTO people (firstname, lastname) VALUES (?, ?)") - _, err = statement.Exec("Nic", "Raboy") - if err != nil { - fmt.Errorf("Failed to insert row: ", err) - } - - // Query all the rows. - rows, _ := database.Query("SELECT id, firstname, lastname FROM people") - var id int - var firstname string - var lastname string - for rows.Next() { - rows.Scan(&id, &firstname, &lastname) - if err != nil { - fmt.Errorf("Failed to scan row: ", err) - } - fmt.Println(strconv.Itoa(id) + ": " + firstname + " " + lastname) - } - - fmt.Print("Foo->") - rowz, err := db.Queryx("SELECT * FROM people") - for rowz.Next() { - var p Person - err = rowz.StructScan(&p) - fmt.Println("The p = ", p) - } -} - // TODO: This is just smoketest-code, delete it after // the smoketest-script has been rewritten a bit to do the same thing. func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { diff --git a/sim-administration/sim-batch-management/storage/storage.go b/sim-administration/sim-batch-management/storage/storage.go new file mode 100644 index 000000000..dde22efa4 --- /dev/null +++ b/sim-administration/sim-batch-management/storage/storage.go @@ -0,0 +1,69 @@ +package storage + +import ( + "database/sql" + "fmt" + "github.com/jmoiron/sqlx" + "strconv" +) + +// Sqlx https://jmoiron.github.io/sqlx/ +// Person represents a person. +type Person struct { + ID int `db:"id" json:"id"` + Firstname string `db:"firstname" json:"firstname"` + Lastname string `db:"lastname" json:"lastname"` +} + +func doTheBatchThing() { + + fmt.Println("The batching getting started") + + // Get a reference to the database, create a table if it + // doesn't exist already. + + var db *sqlx.DB + + // exactly the same as the built-in + db, err := sqlx.Open("sqlite3", "./nraboy.db") + + database, err := sql.Open("sqlite3", "./nraboy.db") + if err != nil { + panic("COuldn't open sql") + } + + // + statement, _ := database.Prepare("CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT)") + _, err = statement.Exec() + if err != nil { + panic("COuldn't create table") + } + + // Insert a row. + statement, _ = database.Prepare("INSERT INTO people (firstname, lastname) VALUES (?, ?)") + _, err = statement.Exec("Nic", "Raboy") + if err != nil { + panic("Failed to insert row: ") + } + + // Query all the rows. + rows, _ := database.Query("SELECT id, firstname, lastname FROM people") + var id int + var firstname string + var lastname string + for rows.Next() { + rows.Scan(&id, &firstname, &lastname) + if err != nil { + panic("Failed to scan row: ") + } + fmt.Println(strconv.Itoa(id) + ": " + firstname + " " + lastname) + } + + fmt.Print("Foo->") + rowz, err := db.Queryx("SELECT * FROM people") + for rowz.Next() { + var p Person + err = rowz.StructScan(&p) + fmt.Println("The p = ", p) + } +} diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go new file mode 100644 index 000000000..c27d96e87 --- /dev/null +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -0,0 +1,23 @@ +package storage + +import ( + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "testing" +) + +func TestMemoryDbPing(t *testing.T) { + + var db *sqlx.DB + + // exactly the same as the built-in + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + t.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) + } + // from a pre-existing sql.DB; note the required driverName + // db = sqlx.NewDb(sql.Open("sqlite3", ":memory:"), "sqlite3") + + // force a connection and test that it worked + err = db.Ping() +} From 3215cfe11a9d13171d0e58f66ebc07064420e8a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 17:47:05 +0100 Subject: [PATCH 065/309] Setting up the fixture for the test --- .../storage/storage_test.go | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index c27d96e87..fae207478 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -1,23 +1,39 @@ package storage import ( + "fmt" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" + "os" "testing" ) -func TestMemoryDbPing(t *testing.T) { +var db *sqlx.DB +var err error - var db *sqlx.DB +func TestMain(m *testing.M) { + setup() + code := m.Run() + shutdown() + os.Exit(code) +} +func setup() { // exactly the same as the built-in - db, err := sqlx.Open("sqlite3", ":memory:") + db, err = sqlx.Open("sqlite3", ":memory:") if err != nil { - t.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) + fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) } - // from a pre-existing sql.DB; note the required driverName - // db = sqlx.NewDb(sql.Open("sqlite3", ":memory:"), "sqlite3") +} + +func shutdown() { +} +// ... just to know that everything is sane. +func TestMemoryDbPing(t *testing.T) { // force a connection and test that it worked err = db.Ping() + if err != nil { + fmt.Errorf("Could not ping in-memory database. '%s'", err) + } } From 5a9435763dd3380beeb4302595c3e7844b3d3656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 22:03:08 +0100 Subject: [PATCH 066/309] still no failures, but a little cleaner. --- .../storage/storage_test.go | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index fae207478..cd5f15e26 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -5,10 +5,11 @@ import ( "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" "os" + "sync" "testing" ) -var db *sqlx.DB +var sdb *SimBatchDB var err error func TestMain(m *testing.M) { @@ -20,10 +21,13 @@ func TestMain(m *testing.M) { func setup() { // exactly the same as the built-in - db, err = sqlx.Open("sqlite3", ":memory:") + db, err := sqlx.Open("sqlite3", ":memory:") if err != nil { fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) } + + sdb = &SimBatchDB{db: db, mu: sync.Mutex{}} + } func shutdown() { @@ -31,9 +35,56 @@ func shutdown() { // ... just to know that everything is sane. func TestMemoryDbPing(t *testing.T) { - // force a connection and test that it worked - err = db.Ping() + sdb.mu.Lock() + defer sdb.mu.Unlock() + err = sdb.db.Ping() if err != nil { fmt.Errorf("Could not ping in-memory database. '%s'", err) } } + +/* TODO: Just for reference, use the one in model instead. +type InputBatch struct { + Customer string + ProfileType string + OrderDate string + BatchNo string + Quantity int + FirstIccid int + FirstImsi int +} +*/ + +func TestGenerateInputBatchTable(t *testing.T) { + GenerateInputBatchTable(sdb) + // Try a CRUD here, spread it out over multiple methods, and our work is done. +} + +// +// GROWTH ZONE: Where the implementation grows, eventually being moved into +// its appropriate location. +// + +type SimBatchDB struct { + db *sqlx.DB + mu sync.Mutex +} + +func GenerateInputBatchTable(sdb *SimBatchDB) { + sdb.mu.Lock() + defer sdb.mu.Unlock() + foo := `CREATE TABLE IF NOT EXISTS SIMBATCH.INPUT_BATCH ( + CUSTOMER VARCHAR, + PROFILE_TYPE VARCHAR NOT NULL, + ORDER_DATE VARCHAR NOT NULL, + BATCH_NO VARCHAR NOT NULL, + QUANTITY INTEGER NOT NULL, + FIRST_ICCID BIGINT, + FIRST_IMSI BIGINT + )` + + _, err := sdb.db.Exec(foo) + if err != nil { + fmt.Errorf("Table creation failed. '%s'", err) + } +} From cdab323496be6477c0de2bc5916bce155092765b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 31 Oct 2019 22:03:27 +0100 Subject: [PATCH 067/309] mark extension point as TODO --- sim-administration/sim-batch-management/storage/storage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index cd5f15e26..dd17755a8 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -57,7 +57,7 @@ type InputBatch struct { func TestGenerateInputBatchTable(t *testing.T) { GenerateInputBatchTable(sdb) - // Try a CRUD here, spread it out over multiple methods, and our work is done. + // TODO: Try a CRUD here, spread it out over multiple methods, and our work is done. } // From d2f170522b94ca2e5fa440256275dbbde0e14d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 11:13:09 +0100 Subject: [PATCH 068/309] Ok, this doesn't fail. Doesn't mean it does the right thing, but that's what we'll find out now. --- go.mod | 1 + go.sum | 4 +++ .../sim-batch-management/sim-batch-mgt.go | 4 --- .../storage/storage_test.go | 28 +++++++++++++++++-- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 97997727f..4ec71321b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/go-ozzo/ozzo-dbx v1.0.15 // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/google/uuid v1.1.1 github.com/jmoiron/sqlx v1.2.0 diff --git a/go.sum b/go.sum index e75ce857b..ffd017944 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,9 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-ozzo/ozzo-dbx v1.0.15 h1:cahBHmsueUkRDhSM71f6TLfd1LtYN4XDce7iCblSYKA= +github.com/go-ozzo/ozzo-dbx v1.0.15/go.mod h1:48NXRgCxSU5JUSW+EA5lnokrL7ql1j6Qh2RsWeh6Fvs= +github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -10,6 +13,7 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index deeb8e52c..e2a86c43d 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -2,15 +2,11 @@ package main import ( - "database/sql" "fmt" - "github.com/jmoiron/sqlx" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/storage" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - "gopkg.in/alecthomas/kingpin.v2" - "strconv" ) var ( diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index dd17755a8..87ca93979 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "os" "sync" "testing" @@ -58,6 +59,29 @@ type InputBatch struct { func TestGenerateInputBatchTable(t *testing.T) { GenerateInputBatchTable(sdb) // TODO: Try a CRUD here, spread it out over multiple methods, and our work is done. + + theBatch := model.InputBatch{ + Customer: "foo", + ProfileType: "banana", + OrderDate: "apple", + BatchNo: "100", + Quantity: 100, + FirstIccid: 1234567890123456789, + FirstImsi: 123456789012345, + } + + result := sdb.db.MustExec("INSERT INTO INPUT_BATCH (CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI) values (?,?,?,?,?,?,?) ", + theBatch.Customer, + theBatch.ProfileType, + theBatch.OrderDate, + theBatch.BatchNo, + theBatch.Quantity, + theBatch.FirstIccid, + theBatch.FirstImsi, + ) + + fmt.Println("The batch = ", result) + } // @@ -73,8 +97,8 @@ type SimBatchDB struct { func GenerateInputBatchTable(sdb *SimBatchDB) { sdb.mu.Lock() defer sdb.mu.Unlock() - foo := `CREATE TABLE IF NOT EXISTS SIMBATCH.INPUT_BATCH ( - CUSTOMER VARCHAR, + foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( + CUSTOMER VARCHAR NOT NULL, PROFILE_TYPE VARCHAR NOT NULL, ORDER_DATE VARCHAR NOT NULL, BATCH_NO VARCHAR NOT NULL, From 73c2efa6df16f3d893604cb54039709225d95c6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 11:32:02 +0100 Subject: [PATCH 069/309] nicely failing test. --- .../storage/storage_test.go | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index 87ca93979..b953de0cc 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -1,11 +1,14 @@ package storage import ( + // "database/sql" "fmt" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" + "gotest.tools/assert" "os" + "reflect" "sync" "testing" ) @@ -70,7 +73,7 @@ func TestGenerateInputBatchTable(t *testing.T) { FirstImsi: 123456789012345, } - result := sdb.db.MustExec("INSERT INTO INPUT_BATCH (CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI) values (?,?,?,?,?,?,?) ", + insertionResult := sdb.db.MustExec("INSERT INTO INPUT_BATCH (CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI) values (?,?,?,?,?,?,?) ", theBatch.Customer, theBatch.ProfileType, theBatch.OrderDate, @@ -80,8 +83,42 @@ func TestGenerateInputBatchTable(t *testing.T) { theBatch.FirstImsi, ) - fmt.Println("The batch = ", result) + fmt.Println("The batch = ", insertionResult) + rows, err := sdb.db.Query("select CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI ") + if err != nil { + fmt.Errorf("Reading query failed '%s'", err) + } + + assert.Assert(t, rows != nil, "Rows shouldn't be nil") + + noOfRows := 0 + for rows.Next() { + var Customer string + var ProfileType string + var OrderDate string + var BatchNo string + var Quantity int + var FirstIccid int + var FirstImsi int + err = rows.Scan(&Customer, &ProfileType, &OrderDate, &BatchNo, &Quantity, &FirstIccid, &FirstImsi) + + queryResult := model.InputBatch{ + Customer: Customer, + ProfileType: ProfileType, + OrderDate: OrderDate, + BatchNo: BatchNo, + Quantity: Quantity, + FirstIccid: FirstIccid, + FirstImsi: FirstImsi, + } + + if !reflect.DeepEqual(theBatch, queryResult) { + fmt.Errorf("Read/write inequality for input batch") + } + noOfRows += 1 + } + assert.Equal(t, noOfRows, 1) } // From 600b6d6a071c8a0c58aed60a638907a0582ec99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 11:35:03 +0100 Subject: [PATCH 070/309] Updated comment. --- sim-administration/sim-batch-management/storage/storage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index b953de0cc..8ab951623 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -90,7 +90,7 @@ func TestGenerateInputBatchTable(t *testing.T) { fmt.Errorf("Reading query failed '%s'", err) } - assert.Assert(t, rows != nil, "Rows shouldn't be nil") + assert.Assert(t, rows != nil, "Rows shouldn't be nil in prepared roundtrip") noOfRows := 0 for rows.Next() { From 8fff56269fa22b1bdca937ce56bce88f4b86961c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 12:53:49 +0100 Subject: [PATCH 071/309] pass a bunch of tests, now we can discuss optimizations. --- .../sim-batch-management/storage/storage_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index 8ab951623..6ae973a0f 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -24,8 +24,8 @@ func TestMain(m *testing.M) { } func setup() { - // exactly the same as the built-in db, err := sqlx.Open("sqlite3", ":memory:") + // db, err := sqlx.Open("sqlite3", "foobar.db") if err != nil { fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) } @@ -85,7 +85,7 @@ func TestGenerateInputBatchTable(t *testing.T) { fmt.Println("The batch = ", insertionResult) - rows, err := sdb.db.Query("select CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI ") + rows, err := sdb.db.Query("select CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI FROM INPUT_BATCH") if err != nil { fmt.Errorf("Reading query failed '%s'", err) } From 628395396b5a1972958da4c5b8a058faf7d70df7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 16:15:33 +0100 Subject: [PATCH 072/309] Advanced scanning now seems to work. --- .../sim-batch-management/model/model.go | 15 +++--- .../storage/storage_test.go | 48 ++++++++++++------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 81d04006e..17f84faf0 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -16,13 +16,14 @@ type OutputBatch struct { } type InputBatch struct { - Customer string - ProfileType string - OrderDate string - BatchNo string - Quantity int - FirstIccid int - FirstImsi int + Id uint64 `db:"id" json:"id"` + Customer string `db:"customer" json:"customer"` + ProfileType string `db:"profileType" json:"profileType"` + OrderDate string `db:"orderDate" json:"orderDate"` + BatchNo string `db:"batchNo" json:"batchNo"` + Quantity int `db:"quantity" json:"quantity"` + FirstIccid string `db:"firstIccid" json:"firstIccid"` + FirstImsi string `db:"firstImsi" json:"firstImsi"` } type OutputFileRecord struct { diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index 6ae973a0f..fe8eed524 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -24,8 +24,8 @@ func TestMain(m *testing.M) { } func setup() { - db, err := sqlx.Open("sqlite3", ":memory:") - // db, err := sqlx.Open("sqlite3", "foobar.db") + // db, err := sqlx.Open("sqlite3", ":memory:") + db, err := sqlx.Open("sqlite3", "foobar.db") if err != nil { fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) } @@ -69,11 +69,11 @@ func TestGenerateInputBatchTable(t *testing.T) { OrderDate: "apple", BatchNo: "100", Quantity: 100, - FirstIccid: 1234567890123456789, - FirstImsi: 123456789012345, + FirstIccid: "1234567890123456789", + FirstImsi: "123456789012345", } - insertionResult := sdb.db.MustExec("INSERT INTO INPUT_BATCH (CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI) values (?,?,?,?,?,?,?) ", + insertionResult := sdb.db.MustExec("INSERT INTO INPUT_BATCH (customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?) ", theBatch.Customer, theBatch.ProfileType, theBatch.OrderDate, @@ -85,7 +85,7 @@ func TestGenerateInputBatchTable(t *testing.T) { fmt.Println("The batch = ", insertionResult) - rows, err := sdb.db.Query("select CUSTOMER, PROFILE_TYPE, ORDER_DATE, BATCH_NO, QUANTITY, FIRST_ICCID, FIRST_IMSI FROM INPUT_BATCH") + rows, err := sdb.db.Query("select id, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi FROM INPUT_BATCH") if err != nil { fmt.Errorf("Reading query failed '%s'", err) } @@ -94,16 +94,18 @@ func TestGenerateInputBatchTable(t *testing.T) { noOfRows := 0 for rows.Next() { + var id uint64 var Customer string var ProfileType string var OrderDate string var BatchNo string var Quantity int - var FirstIccid int - var FirstImsi int - err = rows.Scan(&Customer, &ProfileType, &OrderDate, &BatchNo, &Quantity, &FirstIccid, &FirstImsi) + var FirstIccid string + var FirstImsi string + err = rows.Scan(&id, &Customer, &ProfileType, &OrderDate, &BatchNo, &Quantity, &FirstIccid, &FirstImsi) queryResult := model.InputBatch{ + Id: id, Customer: Customer, ProfileType: ProfileType, OrderDate: OrderDate, @@ -119,6 +121,15 @@ func TestGenerateInputBatchTable(t *testing.T) { noOfRows += 1 } assert.Equal(t, noOfRows, 1) + + var result2 model.InputBatch + err = sdb.db.Get(&result2, "select * from INPUT_BATCH limit 1") + if err != nil { + fmt.Errorf("Get query failed '%s'", err) + } + if !reflect.DeepEqual(theBatch, result2) { + fmt.Errorf("Read/write inequality for input batch") + } } // @@ -135,17 +146,18 @@ func GenerateInputBatchTable(sdb *SimBatchDB) { sdb.mu.Lock() defer sdb.mu.Unlock() foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( - CUSTOMER VARCHAR NOT NULL, - PROFILE_TYPE VARCHAR NOT NULL, - ORDER_DATE VARCHAR NOT NULL, - BATCH_NO VARCHAR NOT NULL, - QUANTITY INTEGER NOT NULL, - FIRST_ICCID BIGINT, - FIRST_IMSI BIGINT + id integer primary key autoincrement, + customer VARCHAR NOT NULL, + profileType VARCHAR NOT NULL, + orderDate VARCHAR NOT NULL, + batchNo VARCHAR NOT NULL, + quantity INTEGER NOT NULL, + firstIccid VARCHAR, + firstImsi VARCHAR )` - _, err := sdb.db.Exec(foo) + result, err := sdb.db.Exec(foo) if err != nil { - fmt.Errorf("Table creation failed. '%s'", err) + fmt.Errorf("Table creation failed. '%s', '%s'", err, result) } } From cfd6a09ac018b484b0b4ffc1bfec1472e597ca3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 16:16:51 +0100 Subject: [PATCH 073/309] adding some notes in comment. To be deleted later. --- .../sim-batch-management/storage/storage_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index fe8eed524..e38aadce4 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -57,6 +57,13 @@ type InputBatch struct { FirstIccid int FirstImsi int } + +course := Course{} +courses := []Course{} + +db.Get(&course, "SELECT name AS course_name FROM courses LIMIT 1") +db.Select(&courses, "SELECT name AS course_name FROM courses") + */ func TestGenerateInputBatchTable(t *testing.T) { From ee0a49868a6896098cd35b166b8a3522833d279d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 16:33:53 +0100 Subject: [PATCH 074/309] CR part now done. This could be sufficient to get traction. UD is probably trivial. --- .../sim-batch-management/model/model.go | 3 ++- .../storage/storage_test.go | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 17f84faf0..6f4a255cf 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -16,7 +16,8 @@ type OutputBatch struct { } type InputBatch struct { - Id uint64 `db:"id" json:"id"` + Id int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` Customer string `db:"customer" json:"customer"` ProfileType string `db:"profileType" json:"profileType"` OrderDate string `db:"orderDate" json:"orderDate"` diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index e38aadce4..9850e696a 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -24,8 +24,8 @@ func TestMain(m *testing.M) { } func setup() { - // db, err := sqlx.Open("sqlite3", ":memory:") - db, err := sqlx.Open("sqlite3", "foobar.db") + db, err := sqlx.Open("sqlite3", ":memory:") + // db, err := sqlx.Open("sqlite3", "foobar.db") if err != nil { fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) } @@ -71,6 +71,7 @@ func TestGenerateInputBatchTable(t *testing.T) { // TODO: Try a CRUD here, spread it out over multiple methods, and our work is done. theBatch := model.InputBatch{ + Name: "SOME UNIQUE NAME", Customer: "foo", ProfileType: "banana", OrderDate: "apple", @@ -80,7 +81,8 @@ func TestGenerateInputBatchTable(t *testing.T) { FirstImsi: "123456789012345", } - insertionResult := sdb.db.MustExec("INSERT INTO INPUT_BATCH (customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?) ", + res := sdb.db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", + theBatch.Name, theBatch.Customer, theBatch.ProfileType, theBatch.OrderDate, @@ -90,9 +92,12 @@ func TestGenerateInputBatchTable(t *testing.T) { theBatch.FirstImsi, ) - fmt.Println("The batch = ", insertionResult) + theBatch.Id, err = res.LastInsertId() + if err != nil { + fmt.Errorf("Getting last inserted id failed '%s'", err) + } - rows, err := sdb.db.Query("select id, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi FROM INPUT_BATCH") + rows, err := sdb.db.Query("select id, name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi FROM INPUT_BATCH") if err != nil { fmt.Errorf("Reading query failed '%s'", err) } @@ -101,7 +106,8 @@ func TestGenerateInputBatchTable(t *testing.T) { noOfRows := 0 for rows.Next() { - var id uint64 + var id int64 + var Name string var Customer string var ProfileType string var OrderDate string @@ -109,10 +115,11 @@ func TestGenerateInputBatchTable(t *testing.T) { var Quantity int var FirstIccid string var FirstImsi string - err = rows.Scan(&id, &Customer, &ProfileType, &OrderDate, &BatchNo, &Quantity, &FirstIccid, &FirstImsi) + err = rows.Scan(&id, &Name, &Customer, &ProfileType, &OrderDate, &BatchNo, &Quantity, &FirstIccid, &FirstImsi) queryResult := model.InputBatch{ Id: id, + Name: Name, Customer: Customer, ProfileType: ProfileType, OrderDate: OrderDate, @@ -154,6 +161,7 @@ func GenerateInputBatchTable(sdb *SimBatchDB) { defer sdb.mu.Unlock() foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( id integer primary key autoincrement, + name VARCHAR NOT NULL, customer VARCHAR NOT NULL, profileType VARCHAR NOT NULL, orderDate VARCHAR NOT NULL, From 3949a6f4f5f91d18b40f734632a6a02cc9313507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 16:38:27 +0100 Subject: [PATCH 075/309] Refactoring into an inserter method. --- .../storage/storage_test.go | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index 9850e696a..d2c512971 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -66,6 +66,25 @@ db.Select(&courses, "SELECT name AS course_name FROM courses") */ +func InsertInputBatch(theBatch *model.InputBatch) { + + res := sdb.db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", + (*theBatch).Name, + (*theBatch).Customer, + (*theBatch).ProfileType, + (*theBatch).OrderDate, + (*theBatch).BatchNo, + (*theBatch).Quantity, + (*theBatch).FirstIccid, + (*theBatch).FirstImsi, + ) + + theBatch.Id, err = res.LastInsertId() + if err != nil { + fmt.Errorf("Getting last inserted id failed '%s'", err) + } +} + func TestGenerateInputBatchTable(t *testing.T) { GenerateInputBatchTable(sdb) // TODO: Try a CRUD here, spread it out over multiple methods, and our work is done. @@ -81,21 +100,7 @@ func TestGenerateInputBatchTable(t *testing.T) { FirstImsi: "123456789012345", } - res := sdb.db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", - theBatch.Name, - theBatch.Customer, - theBatch.ProfileType, - theBatch.OrderDate, - theBatch.BatchNo, - theBatch.Quantity, - theBatch.FirstIccid, - theBatch.FirstImsi, - ) - - theBatch.Id, err = res.LastInsertId() - if err != nil { - fmt.Errorf("Getting last inserted id failed '%s'", err) - } + InsertInputBatch(&theBatch) rows, err := sdb.db.Query("select id, name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi FROM INPUT_BATCH") if err != nil { From 7eeb1865a5f5f2fb6b9cc4ed1635b84f1facf39e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 16:40:23 +0100 Subject: [PATCH 076/309] add a little todo --- .../sim-batch-management/storage/storage_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index d2c512971..1448a93e4 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -87,7 +87,6 @@ func InsertInputBatch(theBatch *model.InputBatch) { func TestGenerateInputBatchTable(t *testing.T) { GenerateInputBatchTable(sdb) - // TODO: Try a CRUD here, spread it out over multiple methods, and our work is done. theBatch := model.InputBatch{ Name: "SOME UNIQUE NAME", @@ -102,6 +101,10 @@ func TestGenerateInputBatchTable(t *testing.T) { InsertInputBatch(&theBatch) + // XXX Refactor into some "get all", "getById", "getByName" methods + // and it will all be awsome. Continue that to completion for input + // batches. Wrap it in an interface, and hook that interface up to + // the command line processor. Rinse&repeat. rows, err := sdb.db.Query("select id, name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi FROM INPUT_BATCH") if err != nil { fmt.Errorf("Reading query failed '%s'", err) From e104b03d83bdfb8df9b1e0bb98a92315709fdae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 22:43:42 +0100 Subject: [PATCH 077/309] pass some more tests. --- .../storage/storage_test.go | 51 +++++++------------ 1 file changed, 17 insertions(+), 34 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index 1448a93e4..c3f867390 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -105,44 +105,12 @@ func TestGenerateInputBatchTable(t *testing.T) { // and it will all be awsome. Continue that to completion for input // batches. Wrap it in an interface, and hook that interface up to // the command line processor. Rinse&repeat. - rows, err := sdb.db.Query("select id, name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi FROM INPUT_BATCH") + allBatches, err := getAllInputBatches() if err != nil { fmt.Errorf("Reading query failed '%s'", err) } - assert.Assert(t, rows != nil, "Rows shouldn't be nil in prepared roundtrip") - - noOfRows := 0 - for rows.Next() { - var id int64 - var Name string - var Customer string - var ProfileType string - var OrderDate string - var BatchNo string - var Quantity int - var FirstIccid string - var FirstImsi string - err = rows.Scan(&id, &Name, &Customer, &ProfileType, &OrderDate, &BatchNo, &Quantity, &FirstIccid, &FirstImsi) - - queryResult := model.InputBatch{ - Id: id, - Name: Name, - Customer: Customer, - ProfileType: ProfileType, - OrderDate: OrderDate, - BatchNo: BatchNo, - Quantity: Quantity, - FirstIccid: FirstIccid, - FirstImsi: FirstImsi, - } - - if !reflect.DeepEqual(theBatch, queryResult) { - fmt.Errorf("Read/write inequality for input batch") - } - noOfRows += 1 - } - assert.Equal(t, noOfRows, 1) + assert.Equal(t, len(allBatches), 1) var result2 model.InputBatch err = sdb.db.Get(&result2, "select * from INPUT_BATCH limit 1") @@ -152,6 +120,21 @@ func TestGenerateInputBatchTable(t *testing.T) { if !reflect.DeepEqual(theBatch, result2) { fmt.Errorf("Read/write inequality for input batch") } + + foo, _ := getInputBatchById(1) + if !reflect.DeepEqual(foo, theBatch) { + fmt.Errorf("getBatchById failed") + } +} + +func getAllInputBatches() ([]model.InputBatch, error) { + result := []model.InputBatch{} + return result, sdb.db.Select(&result, "SELECT * from INPUT_BATCH") +} + +func getInputBatchById(id int64) (*model.InputBatch, error) { + var result model.InputBatch + return &result, sdb.db.Get(&result, "select * from INPUT_BATCH where id = ?", id) } // From 8de1a2a011a12fd62bad06037e0f221ce0c476d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:00:35 +0100 Subject: [PATCH 078/309] The shape of the interface is emerging now. --- .../storage/storage_test.go | 128 ++++++++---------- 1 file changed, 56 insertions(+), 72 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index c3f867390..55b453501 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -47,50 +47,13 @@ func TestMemoryDbPing(t *testing.T) { } } -/* TODO: Just for reference, use the one in model instead. -type InputBatch struct { - Customer string - ProfileType string - OrderDate string - BatchNo string - Quantity int - FirstIccid int - FirstImsi int -} - -course := Course{} -courses := []Course{} - -db.Get(&course, "SELECT name AS course_name FROM courses LIMIT 1") -db.Select(&courses, "SELECT name AS course_name FROM courses") - -*/ - -func InsertInputBatch(theBatch *model.InputBatch) { - - res := sdb.db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", - (*theBatch).Name, - (*theBatch).Customer, - (*theBatch).ProfileType, - (*theBatch).OrderDate, - (*theBatch).BatchNo, - (*theBatch).Quantity, - (*theBatch).FirstIccid, - (*theBatch).FirstImsi, - ) - - theBatch.Id, err = res.LastInsertId() - if err != nil { - fmt.Errorf("Getting last inserted id failed '%s'", err) - } -} -func TestGenerateInputBatchTable(t *testing.T) { - GenerateInputBatchTable(sdb) +func TestInputBatchRoundtrip(t *testing.T) { + GenerateTables(sdb) theBatch := model.InputBatch{ Name: "SOME UNIQUE NAME", - Customer: "foo", + Customer: "firstInputBatch", ProfileType: "banana", OrderDate: "apple", BatchNo: "100", @@ -99,55 +62,78 @@ func TestGenerateInputBatchTable(t *testing.T) { FirstImsi: "123456789012345", } - InsertInputBatch(&theBatch) - - // XXX Refactor into some "get all", "getById", "getByName" methods - // and it will all be awsome. Continue that to completion for input - // batches. Wrap it in an interface, and hook that interface up to - // the command line processor. Rinse&repeat. - allBatches, err := getAllInputBatches() + sdb.Create(&theBatch) + allBatches, err := sdb.GetAllInputBatches() if err != nil { fmt.Errorf("Reading query failed '%s'", err) } assert.Equal(t, len(allBatches), 1) - var result2 model.InputBatch - err = sdb.db.Get(&result2, "select * from INPUT_BATCH limit 1") - if err != nil { - fmt.Errorf("Get query failed '%s'", err) - } - if !reflect.DeepEqual(theBatch, result2) { - fmt.Errorf("Read/write inequality for input batch") - } - - foo, _ := getInputBatchById(1) - if !reflect.DeepEqual(foo, theBatch) { + firstInputBatch, _ := sdb.GetInputBatchById(1) + if !reflect.DeepEqual(firstInputBatch, theBatch) { fmt.Errorf("getBatchById failed") } } -func getAllInputBatches() ([]model.InputBatch, error) { +// +// GROWTH ZONE: Where the implementation grows, eventually being moved into +// its appropriate location. +// + +type SimBatchDB struct { + db *sqlx.DB + mu sync.Mutex +} + +// Store is the storage interface for ds. +type Store interface { + + //Generate all the tables if not already there. + GenerateTables() error + + // Input Batches + Create(doc *model.InputBatch) error + GetAllInputBatches(id string) ([]model.InputBatch, error) + GetInputBatchById(id int64) (*model.InputBatch, error) + GetInputBatchByName(id string) (*model.InputBatch, error) +} + +func (sdb SimBatchDB) GetAllInputBatches() ([]model.InputBatch, error) { result := []model.InputBatch{} return result, sdb.db.Select(&result, "SELECT * from INPUT_BATCH") } -func getInputBatchById(id int64) (*model.InputBatch, error) { +func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.InputBatch, error) { var result model.InputBatch return &result, sdb.db.Get(&result, "select * from INPUT_BATCH where id = ?", id) } -// -// GROWTH ZONE: Where the implementation grows, eventually being moved into -// its appropriate location. -// +func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.InputBatch, error) { + var result model.InputBatch + return &result, sdb.db.Get(&result, "select * from INPUT_BATCH where name = ?", name) +} -type SimBatchDB struct { - db *sqlx.DB - mu sync.Mutex +func (sdb SimBatchDB) Create(theBatch *model.InputBatch) { + + res := sdb.db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", + (*theBatch).Name, + (*theBatch).Customer, + (*theBatch).ProfileType, + (*theBatch).OrderDate, + (*theBatch).BatchNo, + (*theBatch).Quantity, + (*theBatch).FirstIccid, + (*theBatch).FirstImsi, + ) + + theBatch.Id, err = res.LastInsertId() + if err != nil { + fmt.Errorf("Getting last inserted id failed '%s'", err) + } } -func GenerateInputBatchTable(sdb *SimBatchDB) { +func (sdb *SimBatchDB) GenerateTables error { sdb.mu.Lock() defer sdb.mu.Unlock() foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( @@ -162,8 +148,6 @@ func GenerateInputBatchTable(sdb *SimBatchDB) { firstImsi VARCHAR )` - result, err := sdb.db.Exec(foo) - if err != nil { - fmt.Errorf("Table creation failed. '%s', '%s'", err, result) - } + _, err := sdb.db.Exec(foo) + return err } From 6806d06a857fd8fa881af0fd73310efb15a062e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:01:51 +0100 Subject: [PATCH 079/309] Even better shape --- .../sim-batch-management/storage/storage_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go index 55b453501..47963ae2d 100644 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ b/sim-administration/sim-batch-management/storage/storage_test.go @@ -47,9 +47,8 @@ func TestMemoryDbPing(t *testing.T) { } } - func TestInputBatchRoundtrip(t *testing.T) { - GenerateTables(sdb) + sdb.GenerateTables() theBatch := model.InputBatch{ Name: "SOME UNIQUE NAME", @@ -133,7 +132,7 @@ func (sdb SimBatchDB) Create(theBatch *model.InputBatch) { } } -func (sdb *SimBatchDB) GenerateTables error { +func (sdb *SimBatchDB) GenerateTables() error { sdb.mu.Lock() defer sdb.mu.Unlock() foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( From 69489a0c3bcbb6187afff39dae4d3da170c7daf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:33:56 +0100 Subject: [PATCH 080/309] Basic shape in order now. Time to get busy for real! --- .../sim-batch-management/model/model.go | 36 +++-- .../sim-batch-management/sim-batch-mgt.go | 3 +- .../sim-batch-management/storage/storage.go | 69 -------- .../storage/storage_test.go | 152 ------------------ .../sim-batch-management/store/store.go | 72 +++++++++ .../sim-batch-management/store/store_test.go | 72 +++++++++ 6 files changed, 167 insertions(+), 237 deletions(-) delete mode 100644 sim-administration/sim-batch-management/storage/storage.go delete mode 100644 sim-administration/sim-batch-management/storage/storage_test.go create mode 100644 sim-administration/sim-batch-management/store/store.go create mode 100644 sim-administration/sim-batch-management/store/store_test.go diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 6f4a255cf..8a5f0d8ca 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -1,20 +1,16 @@ package model -// TODO: Put all records used to manage workflows in this -// package, then build a DAO interface a "store" package - -type OutputBatch struct { - ProfileType string `db:"profileType" json:"profileType"` - Url string `db:"url" json:"url"` - Length int - FirstMsisdn int - MsisdnIncrement int - FirstIccid int - IccidIncrement int - FirstImsi int - ImsiIncrement int -} +// TODO: There are now multiple structs that model batches. +// It's probably a good idea to harmonize these so that it's +// only one type of batch info that's being read, and then have +// various ways to combine the misc. sources of batch information +// that lets partial information from multiple records be harmonized +// in a common persisted record that is then used for the bulk of the +// processing. +// +// Batches as read from the input files +// type InputBatch struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` @@ -27,6 +23,18 @@ type InputBatch struct { FirstImsi string `db:"firstImsi" json:"firstImsi"` } +type OutputBatch struct { + ProfileType string `db:"profileType" json:"profileType"` + Url string `db:"url" json:"url"` + Length int + FirstMsisdn int + MsisdnIncrement int + FirstIccid int + IccidIncrement int + FirstImsi int + ImsiIncrement int +} + type OutputFileRecord struct { Filename string InputVariables map[string]string diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index e2a86c43d..dc07bb474 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/storage" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" ) @@ -175,7 +174,7 @@ func main() { } case "batch": fmt.Println("Doing the batch thing.") - storage.doTheBatchThing() + // storage.doTheBatchThing() default: panic(fmt.Sprintf("Unknown command: '%s'\n", cmd)) } diff --git a/sim-administration/sim-batch-management/storage/storage.go b/sim-administration/sim-batch-management/storage/storage.go deleted file mode 100644 index dde22efa4..000000000 --- a/sim-administration/sim-batch-management/storage/storage.go +++ /dev/null @@ -1,69 +0,0 @@ -package storage - -import ( - "database/sql" - "fmt" - "github.com/jmoiron/sqlx" - "strconv" -) - -// Sqlx https://jmoiron.github.io/sqlx/ -// Person represents a person. -type Person struct { - ID int `db:"id" json:"id"` - Firstname string `db:"firstname" json:"firstname"` - Lastname string `db:"lastname" json:"lastname"` -} - -func doTheBatchThing() { - - fmt.Println("The batching getting started") - - // Get a reference to the database, create a table if it - // doesn't exist already. - - var db *sqlx.DB - - // exactly the same as the built-in - db, err := sqlx.Open("sqlite3", "./nraboy.db") - - database, err := sql.Open("sqlite3", "./nraboy.db") - if err != nil { - panic("COuldn't open sql") - } - - // - statement, _ := database.Prepare("CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT)") - _, err = statement.Exec() - if err != nil { - panic("COuldn't create table") - } - - // Insert a row. - statement, _ = database.Prepare("INSERT INTO people (firstname, lastname) VALUES (?, ?)") - _, err = statement.Exec("Nic", "Raboy") - if err != nil { - panic("Failed to insert row: ") - } - - // Query all the rows. - rows, _ := database.Query("SELECT id, firstname, lastname FROM people") - var id int - var firstname string - var lastname string - for rows.Next() { - rows.Scan(&id, &firstname, &lastname) - if err != nil { - panic("Failed to scan row: ") - } - fmt.Println(strconv.Itoa(id) + ": " + firstname + " " + lastname) - } - - fmt.Print("Foo->") - rowz, err := db.Queryx("SELECT * FROM people") - for rowz.Next() { - var p Person - err = rowz.StructScan(&p) - fmt.Println("The p = ", p) - } -} diff --git a/sim-administration/sim-batch-management/storage/storage_test.go b/sim-administration/sim-batch-management/storage/storage_test.go deleted file mode 100644 index 47963ae2d..000000000 --- a/sim-administration/sim-batch-management/storage/storage_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package storage - -import ( - // "database/sql" - "fmt" - "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" - "gotest.tools/assert" - "os" - "reflect" - "sync" - "testing" -) - -var sdb *SimBatchDB -var err error - -func TestMain(m *testing.M) { - setup() - code := m.Run() - shutdown() - os.Exit(code) -} - -func setup() { - db, err := sqlx.Open("sqlite3", ":memory:") - // db, err := sqlx.Open("sqlite3", "foobar.db") - if err != nil { - fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) - } - - sdb = &SimBatchDB{db: db, mu: sync.Mutex{}} - -} - -func shutdown() { -} - -// ... just to know that everything is sane. -func TestMemoryDbPing(t *testing.T) { - sdb.mu.Lock() - defer sdb.mu.Unlock() - err = sdb.db.Ping() - if err != nil { - fmt.Errorf("Could not ping in-memory database. '%s'", err) - } -} - -func TestInputBatchRoundtrip(t *testing.T) { - sdb.GenerateTables() - - theBatch := model.InputBatch{ - Name: "SOME UNIQUE NAME", - Customer: "firstInputBatch", - ProfileType: "banana", - OrderDate: "apple", - BatchNo: "100", - Quantity: 100, - FirstIccid: "1234567890123456789", - FirstImsi: "123456789012345", - } - - sdb.Create(&theBatch) - allBatches, err := sdb.GetAllInputBatches() - if err != nil { - fmt.Errorf("Reading query failed '%s'", err) - } - - assert.Equal(t, len(allBatches), 1) - - firstInputBatch, _ := sdb.GetInputBatchById(1) - if !reflect.DeepEqual(firstInputBatch, theBatch) { - fmt.Errorf("getBatchById failed") - } -} - -// -// GROWTH ZONE: Where the implementation grows, eventually being moved into -// its appropriate location. -// - -type SimBatchDB struct { - db *sqlx.DB - mu sync.Mutex -} - -// Store is the storage interface for ds. -type Store interface { - - //Generate all the tables if not already there. - GenerateTables() error - - // Input Batches - Create(doc *model.InputBatch) error - GetAllInputBatches(id string) ([]model.InputBatch, error) - GetInputBatchById(id int64) (*model.InputBatch, error) - GetInputBatchByName(id string) (*model.InputBatch, error) -} - -func (sdb SimBatchDB) GetAllInputBatches() ([]model.InputBatch, error) { - result := []model.InputBatch{} - return result, sdb.db.Select(&result, "SELECT * from INPUT_BATCH") -} - -func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.InputBatch, error) { - var result model.InputBatch - return &result, sdb.db.Get(&result, "select * from INPUT_BATCH where id = ?", id) -} - -func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.InputBatch, error) { - var result model.InputBatch - return &result, sdb.db.Get(&result, "select * from INPUT_BATCH where name = ?", name) -} - -func (sdb SimBatchDB) Create(theBatch *model.InputBatch) { - - res := sdb.db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", - (*theBatch).Name, - (*theBatch).Customer, - (*theBatch).ProfileType, - (*theBatch).OrderDate, - (*theBatch).BatchNo, - (*theBatch).Quantity, - (*theBatch).FirstIccid, - (*theBatch).FirstImsi, - ) - - theBatch.Id, err = res.LastInsertId() - if err != nil { - fmt.Errorf("Getting last inserted id failed '%s'", err) - } -} - -func (sdb *SimBatchDB) GenerateTables() error { - sdb.mu.Lock() - defer sdb.mu.Unlock() - foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( - id integer primary key autoincrement, - name VARCHAR NOT NULL, - customer VARCHAR NOT NULL, - profileType VARCHAR NOT NULL, - orderDate VARCHAR NOT NULL, - batchNo VARCHAR NOT NULL, - quantity INTEGER NOT NULL, - firstIccid VARCHAR, - firstImsi VARCHAR - )` - - _, err := sdb.db.Exec(foo) - return err -} diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go new file mode 100644 index 000000000..3907a3c68 --- /dev/null +++ b/sim-administration/sim-batch-management/store/store.go @@ -0,0 +1,72 @@ +package store + +import ( + "fmt" + "github.com/jmoiron/sqlx" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" +) + +type Store interface { + GenerateTables() error + + Create(theBatch *model.InputBatch) error + GetAllInputBatches(id string) ([]model.InputBatch, error) + GetInputBatchById(id int64) (*model.InputBatch, error) + GetInputBatchByName(id string) (*model.InputBatch, error) +} + +type SimBatchDB struct { + Db *sqlx.DB +} + +func (sdb SimBatchDB) GetAllInputBatches() ([]model.InputBatch, error) { + result := []model.InputBatch{} + return result, sdb.Db.Select(&result, "SELECT * from INPUT_BATCH") +} + +func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.InputBatch, error) { + var result model.InputBatch + return &result, sdb.Db.Get(&result, "select * from INPUT_BATCH where id = ?", id) +} + +func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.InputBatch, error) { + var result model.InputBatch + return &result, sdb.Db.Get(&result, "select * from INPUT_BATCH where name = ?", name) +} + +func (sdb SimBatchDB) Create(theBatch *model.InputBatch) { + + res := sdb.Db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", + (*theBatch).Name, + (*theBatch).Customer, + (*theBatch).ProfileType, + (*theBatch).OrderDate, + (*theBatch).BatchNo, + (*theBatch).Quantity, + (*theBatch).FirstIccid, + (*theBatch).FirstImsi, + ) + + id, err := res.LastInsertId() + if err != nil { + fmt.Errorf("Getting last inserted id failed '%s'", err) + } + theBatch.Id = id +} + +func (sdb *SimBatchDB) GenerateTables() error { + foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( + id integer primary key autoincrement, + name VARCHAR NOT NULL, + customer VARCHAR NOT NULL, + profileType VARCHAR NOT NULL, + orderDate VARCHAR NOT NULL, + batchNo VARCHAR NOT NULL, + quantity INTEGER NOT NULL, + firstIccid VARCHAR, + firstImsi VARCHAR + )` + + _, err := sdb.Db.Exec(foo) + return err +} diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go new file mode 100644 index 000000000..5b2fee4d4 --- /dev/null +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -0,0 +1,72 @@ +package store + +import ( + "fmt" + "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" + "gotest.tools/assert" + "os" + "reflect" + "testing" +) + +var sdb *store.SimBatchDB +var err error + +func TestMain(m *testing.M) { + setup() + code := m.Run() + shutdown() + os.Exit(code) +} + +func setup() { + db, err := sqlx.Open("sqlite3", ":memory:") + // db, err := sqlx.Open("sqlite3", "foobar.db") + if err != nil { + fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) + } + + sdb = &store.SimBatchDB{Db: db} + sdb.GenerateTables() +} + +func shutdown() { +} + +// ... just to know that everything is sane. +func TestMemoryDbPing(t *testing.T) { + err = sdb.Db.Ping() + if err != nil { + fmt.Errorf("Could not ping in-memory database. '%s'", err) + } +} + +func TestInputBatchRoundtrip(t *testing.T) { + + theBatch := model.InputBatch{ + Name: "SOME UNIQUE NAME", + Customer: "firstInputBatch", + ProfileType: "banana", + OrderDate: "apple", + BatchNo: "100", + Quantity: 100, + FirstIccid: "1234567890123456789", + FirstImsi: "123456789012345", + } + + sdb.Create(&theBatch) + allBatches, err := sdb.GetAllInputBatches() + if err != nil { + fmt.Errorf("Reading query failed '%s'", err) + } + + assert.Equal(t, len(allBatches), 1) + + firstInputBatch, _ := sdb.GetInputBatchById(1) + if !reflect.DeepEqual(firstInputBatch, theBatch) { + fmt.Errorf("getBatchById failed") + } +} From 055b43d7d3ee71b661ed96680643bb703ebdd264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:38:45 +0100 Subject: [PATCH 081/309] Better creation of database instance --- .../sim-batch-management/store/store.go | 16 ++++++++++++++++ .../sim-batch-management/store/store_test.go | 9 +-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 3907a3c68..d396fb70c 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -19,6 +19,22 @@ type SimBatchDB struct { Db *sqlx.DB } +func NewInMemoryDatabase() *SimBatchDB { + db, err := sqlx.Open("sqlite3", ":memory:") + if err != nil { + fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) + } + return &SimBatchDB{Db: db} +} + +func NewFileSqliteDatabase(path string) *SimBatchDB { + db, err := sqlx.Open("sqlite3", "foobar.db") + if err != nil { + fmt.Errorf("Didn't manage to open sqlite3 file database. '%s'", err) + } + return &SimBatchDB{Db: db} +} + func (sdb SimBatchDB) GetAllInputBatches() ([]model.InputBatch, error) { result := []model.InputBatch{} return result, sdb.Db.Select(&result, "SELECT * from INPUT_BATCH") diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 5b2fee4d4..f4d448cac 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -2,7 +2,6 @@ package store import ( "fmt" - "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" @@ -23,13 +22,7 @@ func TestMain(m *testing.M) { } func setup() { - db, err := sqlx.Open("sqlite3", ":memory:") - // db, err := sqlx.Open("sqlite3", "foobar.db") - if err != nil { - fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) - } - - sdb = &store.SimBatchDB{Db: db} + sdb = store.NewInMemoryDatabase() sdb.GenerateTables() } From f4a178cfd03f9ca73c5bd6e1fc60371c635e34d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:46:55 +0100 Subject: [PATCH 082/309] dotting i's and dashig t-s --- go.mod | 2 +- .../sim-batch-management/store/store.go | 19 +++++++++++++------ .../sim-batch-management/store/store_test.go | 5 ++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 4ec71321b..e14f2307f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/google/uuid v1.1.1 github.com/jmoiron/sqlx v1.2.0 github.com/mattn/go-sqlite3 v1.11.0 - github.com/pkg/errors v0.8.1 // indirect + github.com/pkg/errors v0.8.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gotest.tools v2.2.0+incompatible ) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index d396fb70c..38915cc99 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/jmoiron/sqlx" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" + "os" ) type Store interface { @@ -19,20 +20,26 @@ type SimBatchDB struct { Db *sqlx.DB } -func NewInMemoryDatabase() *SimBatchDB { +func NewInMemoryDatabase() (*SimBatchDB, error) { db, err := sqlx.Open("sqlite3", ":memory:") if err != nil { - fmt.Errorf("Didn't manage to open sqlite3 in-memory database. '%s'", err) + return nil, err } - return &SimBatchDB{Db: db} + return &SimBatchDB{Db: db}, nil } -func NewFileSqliteDatabase(path string) *SimBatchDB { +func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (*SimBatchDB, error) { + variableValue := os.Getenv(variablename) + db, err := OpenFileSqliteDatabase(variableValue) + return db, err +} + +func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { db, err := sqlx.Open("sqlite3", "foobar.db") if err != nil { - fmt.Errorf("Didn't manage to open sqlite3 file database. '%s'", err) + return nil, err } - return &SimBatchDB{Db: db} + return &SimBatchDB{Db: db}, nil } func (sdb SimBatchDB) GetAllInputBatches() ([]model.InputBatch, error) { diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index f4d448cac..f4720543b 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -22,7 +22,10 @@ func TestMain(m *testing.M) { } func setup() { - sdb = store.NewInMemoryDatabase() + sdb, err = store.NewInMemoryDatabase() + if err != nil { + fmt.Errorf("Couldn't open new in memory database") + } sdb.GenerateTables() } From c7a19d60adaddd226354737c5c90d7be3ba49d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:56:44 +0100 Subject: [PATCH 083/309] Deleting now obsolete fingering exercise in persistence. --- .../sim-batch-management/sim-batch-db.go | 82 ------------------- 1 file changed, 82 deletions(-) delete mode 100644 sim-administration/sim-batch-management/sim-batch-db.go diff --git a/sim-administration/sim-batch-management/sim-batch-db.go b/sim-administration/sim-batch-management/sim-batch-db.go deleted file mode 100644 index 1b8ac630c..000000000 --- a/sim-administration/sim-batch-management/sim-batch-db.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -// A simple "finger exercise" to get to know sqlite in golang. Purely -// exploratory, no specific goal in sight. -// NOTE: It will asap be rewritten to manage a persistent store of -// batch parameters, and also be able to facilitate a workflow -// sheparding the orders from inception to deployment. -// Todo: -// * Introduce common model package for existing structs. -// * Extend model with metedata for persistence. -// * Write DAO based on this file, but make it properly -// unit tested using in-memory database. -// * Figure out how to keep config somewhere safe without -// having to type too much. - -import ( - "database/sql" - "fmt" - "github.com/jmoiron/sqlx" - "strconv" - - _ "github.com/mattn/go-sqlite3" -) - -// Sqlx https://jmoiron.github.io/sqlx/ -// Person represents a person. -type Person struct { - ID int `db:"id" json:"id"` - Firstname string `db:"firstname" json:"firstname"` - Lastname string `db:"lastname" json:"lastname"` -} - -func main_not() { - - // Get a reference to the database, create a table if it - // doesn't exist already. - - var db *sqlx.DB - - // exactly the same as the built-in - db, err := sqlx.Open("sqlite3", "./nraboy.db") - - database, err := sql.Open("sqlite3", "./nraboy.db") - if err != nil { - fmt.Errorf("open sql", err) - } - - // - statement, _ := database.Prepare("CREATE TABLE IF NOT EXISTS people (id INTEGER PRIMARY KEY, firstname TEXT, lastname TEXT)") - _, err = statement.Exec() - if err != nil { - fmt.Errorf("Failed to create table: ", err) - } - - // Insert a row. - statement, _ = database.Prepare("INSERT INTO people (firstname, lastname) VALUES (?, ?)") - _, err = statement.Exec("Nic", "Raboy") - if err != nil { - fmt.Errorf("Failed to insert row: ", err) - } - - // Query all the rows. - rows, _ := database.Query("SELECT id, firstname, lastname FROM people") - var id int - var firstname string - var lastname string - for rows.Next() { - rows.Scan(&id, &firstname, &lastname) - if err != nil { - fmt.Errorf("Failed to scan row: ", err) - } - fmt.Println(strconv.Itoa(id) + ": " + firstname + " " + lastname) - } - - fmt.Print("Foo->") - rowz, err := db.Queryx("SELECT * FROM people") - for rowz.Next() { - var p Person - err = rowz.StructScan(&p) - fmt.Println("The p = ", p) - } -} From 203bf501dfdded1d262f7e80cbba3fc86bbb3336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 1 Nov 2019 23:56:59 +0100 Subject: [PATCH 084/309] Fix typo --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index dc07bb474..4afd0595e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -112,7 +112,7 @@ func main() { switch cmd { case "es2plus-smoketest": es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) - case "simProfilerUpload": + case "sim-profile-upload": outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "sim-profile-upload": // TODO: Combine these two into something inside uploadtoprime. From 270801d4276bad10a1831d189c9926c36ced03f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 00:12:02 +0100 Subject: [PATCH 085/309] update todo --- .../sim-batch-management/TODO.md | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index aa3827208..6fc674648 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,18 +1,19 @@ An informal TODO list for the sim batch management tool == -1. Make an RDBMS that handles sim card workflows. -2. It must by necessity be able to handle free lists of - imsis, msisdns etc. -3. As today, it should be possible to -generate- those lists - from parameters, where that makes sense. In general howeve,r - in particular for production use, this will not be the case - and we need to cater for that. -4. The programme should -initially- be wholly command line - oriented, with a database using sqlite. -5. At some (much) later stage, it may make sense to put it - in some cloud, somewhere. -6. The interfaces to external parties will be +1. Make persistence-model for generated sim profiles. This database should then be used + to generate uploads of various kinds to prime. This essentially means mirroring + and extending the data model for sim profiles currently in Prime. +1. Rewrite upload-sim-batch-lib-test.go to be part of sim-batch-mgt.go, + during that process: + * Persist the batch data. + * List available batches, show the status. + * Make a command that can print the input file for that batch. + * Find some clever way to get legacy batches into the database without typing too much. +1. Add crypto resources so that the program can talk to external parties. +1. Figure out how to handle workflows. Be explicit! +1. Handle both parameterized lists of MSISDNs and list-based input. +1. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determind) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or From 2dc6b42f8f6bc0972d7815b14dc5941ab0894f57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 00:16:58 +0100 Subject: [PATCH 086/309] Add uniqueness constraint on batch name (used for addressing the batch from the comman dline) --- sim-administration/sim-batch-management/store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 38915cc99..95f71437c 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -80,7 +80,7 @@ func (sdb SimBatchDB) Create(theBatch *model.InputBatch) { func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( id integer primary key autoincrement, - name VARCHAR NOT NULL, + name VARCHAR NOT NULL UNIQUE, customer VARCHAR NOT NULL, profileType VARCHAR NOT NULL, orderDate VARCHAR NOT NULL, From 431d8d7e1334c61a18cd3b8d7bdb350ac9239bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 11:17:55 +0100 Subject: [PATCH 087/309] Slowly starting the conversion towards holding state in a database --- .../sim-batch-management/sim-batch-mgt.go | 38 ++++++++++++++++++- .../uploadtoprime/upload-sim-batch-lib.go | 8 ++-- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 4afd0595e..f79513df3 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -3,6 +3,7 @@ package main import ( "fmt" + "gopkg.in/alecthomas/kingpin.v2" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" @@ -103,7 +104,40 @@ var ( "initial-hlr-activation-status-of-profiles", "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() + // TODO ??? batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") + + declareBatch = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") + + // + // Set up command line parsing + // + firstIccid = declareBatch.Flag("first-rawIccid", + "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() + lastIccid = declareBatch.Flag("last-rawIccid", + "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() + firstIMSI = declareBatch.Flag("first-imsi", "First IMSI in batch").Required().String() + lastIMSI = declareBatch.Flag("last-imsi", "Last IMSI in batch").Required().String() + firstMsisdn = declareBatch.Flag("first-msisdn", "First MSISDN in batch").Required().String() + lastMsisdn = declareBatch.Flag("last-msisdn", "Last MSISDN in batch").Required().String() + profileType = declareBatch.Flag("profile-type", "SIM profile type").Required().String() + batchLengthString = declareBatch.Flag( + "batch-quantity", + "Number of sim cards in batch").Required().String() + + hssVendor = declareBatch.Flag("hss-vendor", "The HSS vendor").Default("M1").String() + uploadHostname = + declareBatch.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() + uploadPortnumber = + declareBatch.Flag("upload-portnumber", "port to upload to").Default("8080").String() + + profileVendor = + declareBatch.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia",).String() + + initialHlrActivationStatusOfProfiles = + declareBatch.Flag( + "initial-hlr-activation-status-of-profiles", + "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() ) func main() { @@ -114,7 +148,9 @@ func main() { es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) case "sim-profile-upload": outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) - case "sim-profile-upload": + case "declare-batch": + fmt.Println("Declare batch") + case "prime-batch-upload": // TODO: Combine these two into something inside uploadtoprime. // It's unecessary to break the batch thingy open in this way. var batch = uploadtoprime.OutputBatchFromCommandLineParameters( diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index 4a64ec445..74c47cd76 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -322,8 +322,8 @@ func ParseInputFileGeneratorCommmandline() model.InputBatch { OrderDate: "20191007", BatchNo: "2019100701", Quantity: 10, - FirstIccid: 894700000000002214, - FirstImsi: 242017100012213} + FirstIccid: "894700000000002214", + FirstImsi: "242017100012213"} } func GenerateInputFile(batch model.InputBatch) string { @@ -338,8 +338,8 @@ func GenerateInputFile(batch model.InputBatch) string { "*INPUT VARIABLES\n" + "***************************************\n" + "var_In:\n" + - fmt.Sprintf(" ICCID: %d\n", batch.FirstIccid) + - fmt.Sprintf("IMSI: %d\n", batch.FirstImsi) + + fmt.Sprintf(" ICCID: %s\n", batch.FirstIccid) + + fmt.Sprintf("IMSI: %s\n", batch.FirstImsi) + "***************************************\n" + "*OUTPUT VARIABLES\n" + "***************************************\n" + From 149310a078a635d9dc887faafefe1e78804a51e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 11:37:47 +0100 Subject: [PATCH 088/309] More progress towards persistence (commandline without any arg line checking or persistence so far). --- .../sim-batch-management/.gitignore | 1 + .../sim-batch-management/sim-batch-mgt.go | 75 ++++++++++++++----- 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/sim-administration/sim-batch-management/.gitignore b/sim-administration/sim-batch-management/.gitignore index 64488d3a0..9e9c72b3e 100644 --- a/sim-administration/sim-batch-management/.gitignore +++ b/sim-administration/sim-batch-management/.gitignore @@ -1,2 +1,3 @@ *.sh es2pluswrapper.sh +batch-lifecycle-test.sh diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index f79513df3..d441b68a9 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -3,10 +3,11 @@ package main import ( "fmt" - "gopkg.in/alecthomas/kingpin.v2" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" + // Don' lose this! + "gopkg.in/alecthomas/kingpin.v2" ) var ( @@ -107,35 +108,35 @@ var ( // TODO ??? batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") - declareBatch = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") + db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") // // Set up command line parsing // - firstIccid = declareBatch.Flag("first-rawIccid", + dbFirstIccid = db.Flag("first-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - lastIccid = declareBatch.Flag("last-rawIccid", + dbLastIccid = db.Flag("last-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - firstIMSI = declareBatch.Flag("first-imsi", "First IMSI in batch").Required().String() - lastIMSI = declareBatch.Flag("last-imsi", "Last IMSI in batch").Required().String() - firstMsisdn = declareBatch.Flag("first-msisdn", "First MSISDN in batch").Required().String() - lastMsisdn = declareBatch.Flag("last-msisdn", "Last MSISDN in batch").Required().String() - profileType = declareBatch.Flag("profile-type", "SIM profile type").Required().String() - batchLengthString = declareBatch.Flag( + dbFirstIMSI = db.Flag("first-imsi", "First IMSI in batch").Required().String() + dbLastIMSI = db.Flag("last-imsi", "Last IMSI in batch").Required().String() + dbFirstMsisdn = db.Flag("first-msisdn", "First MSISDN in batch").Required().String() + dbLastMsisdn = db.Flag("last-msisdn", "Last MSISDN in batch").Required().String() + dbProfileType = db.Flag("profile-type", "SIM profile type").Required().String() + dbBatchLengthString = db.Flag( "batch-quantity", "Number of sim cards in batch").Required().String() - hssVendor = declareBatch.Flag("hss-vendor", "The HSS vendor").Default("M1").String() - uploadHostname = - declareBatch.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() - uploadPortnumber = - declareBatch.Flag("upload-portnumber", "port to upload to").Default("8080").String() + dbHssVendor = db.Flag("hss-vendor", "The HSS vendor").Default("M1").String() + dbUploadHostname = + db.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() + dbUploadPortnumber = + db.Flag("upload-portnumber", "port to upload to").Default("8080").String() - profileVendor = - declareBatch.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia",).String() + dbProfileVendor = + db.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() - initialHlrActivationStatusOfProfiles = - declareBatch.Flag( + dbInitialHlrActivationStatusOfProfiles = + db.Flag( "initial-hlr-activation-status-of-profiles", "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() ) @@ -150,6 +151,20 @@ func main() { outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "declare-batch": fmt.Println("Declare batch") + declareThisBatch( + *dbFirstIccid, + *dbLastIccid, + *dbFirstIMSI, + *dbLastIMSI, + *dbFirstMsisdn, + *dbLastMsisdn, + *dbProfileType, + *dbBatchLengthString, + *dbHssVendor, + *dbUploadHostname, + *dbUploadPortnumber, + *dbProfileVendor, + *dbInitialHlrActivationStatusOfProfiles) case "prime-batch-upload": // TODO: Combine these two into something inside uploadtoprime. // It's unecessary to break the batch thingy open in this way. @@ -305,3 +320,25 @@ func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *strin fmt.Println("Success") } + +// XXX Put this into a separate package at some point, "batch_editor" or something +// equally descriptive. +func declareThisBatch( + firstIccid string, + lastIccid string, + firstIMSI string, + lastIMSI string, + firstMsisdn string, + lastMsisdn string, + profileType string, + batchLengthString string, + hssVendor string, + uploadHostname string, + uploadPortnumber string, + profileVendor string, + initialHlrActivationStatusOfProfiles string) { + fmt.Println("HOhoho, now we're declaring a batch!") + // 1. Check all the arguments (methods already written). + // 2. Check that the name isn't already registred. + // 3. If it isn't, then persist it +} From a69e13a93ccbb081a7c3e72e97afc6c4c8d751f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:32:47 +0100 Subject: [PATCH 089/309] Refactor persistence code to use a new struct called "Batch", so that we don't have to fiddle with the old stuff. It's too brittle anyway. --- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 110 +++++++++++++++ .../luhn_test.go | 4 +- .../sim-batch-management/model/model.go | 12 ++ .../sim-batch-management/sim-batch-mgt.go | 133 ++++++++++++++---- .../sim-batch-management/store/store.go | 22 +-- .../sim-batch-management/store/store_test.go | 2 +- .../uploadtoprime/upload-sim-batch-lib.go | 124 ++-------------- 7 files changed, 254 insertions(+), 153 deletions(-) create mode 100644 sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go rename sim-administration/sim-batch-management/{uploadtoprime => fieldsyntaxchecks}/luhn_test.go (97%) diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go new file mode 100644 index 000000000..b0577bdbe --- /dev/null +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -0,0 +1,110 @@ +package fieldsyntaxchecks + +import ( + "fmt" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" + "log" + "net/url" + "regexp" + "strconv" + "strings" +) + +func generateControlDigit(luhnString string) int { + controlDigit := calculateChecksum(luhnString, true) % 10 + + if controlDigit != 0 { + controlDigit = 10 - controlDigit + } + + return controlDigit +} + +func calculateChecksum(luhnString string, double bool) int { + source := strings.Split(luhnString, "") + checksum := 0 + + for i := len(source) - 1; i > -1; i-- { + t, _ := strconv.ParseInt(source[i], 10, 8) + n := int(t) + + if double { + n = n * 2 + } + + double = !double + + if n >= 10 { + n = n - 9 + } + + checksum += n + } + return checksum +} + +func LuhnChecksum(number int) int { + return generateControlDigit(strconv.Itoa(number)) +} + +func IsICCID(s string) bool { + match, _ := regexp.MatchString("^\\d{18}\\d?\\d?$", s) + return match +} + +func CheckICCIDSyntax(name string, potentialIccid string) { + if !IsICCID(potentialIccid) { + log.Fatalf("Not a valid %s ICCID: '%s'. Must be 18 or 19 (or 20) digits (_including_ luhn checksum).", name, potentialIccid) + } + + stringWithoutLuhnChecksum := IccidWithoutLuhnChecksum(potentialIccid) + controlDigit := generateControlDigit(stringWithoutLuhnChecksum) + checksummedCandidate := fmt.Sprintf("%s%d", stringWithoutLuhnChecksum, controlDigit) + if checksummedCandidate != potentialIccid { + log.Fatalf("Not a valid ICCID: '%s'. Expected luhn checksom '%d'", potentialIccid, controlDigit) + } +} + +func IsIMSI(s string) bool { + match, _ := regexp.MatchString("^\\d{15}$", s) + return match +} + +func CheckIMSISyntax(name string, potentialIMSI string) { + if !IsIMSI(potentialIMSI) { + log.Fatalf("Not a valid %s IMSI: '%s'. Must be 15 digits.", name, potentialIMSI) + } +} + +func IsMSISDN(s string) bool { + match, _ := regexp.MatchString("^\\d+$", s) + return match +} + +func CheckMSISDNSyntax(name string, potentialMSISDN string) { + if !IsMSISDN(potentialMSISDN) { + log.Fatalf("Not a valid %s MSISDN: '%s'. Must be non-empty sequence of digits.", name, potentialMSISDN) + } +} + +func CheckURLSyntax(name string, theUrl string) { + _, err := url.ParseRequestURI(theUrl) + if err != nil { + log.Fatalf("Not a valid %s URL: '%s'.", name, theUrl) + } +} + +func IsProfileName(s string) bool { + match, _ := regexp.MatchString("^[A-Z][A-Z0-9_]*$", s) + return match +} + +func CheckProfileType(name string, potentialProfileName string) { + if !IsProfileName(potentialProfileName) { + log.Fatalf("Not a valid %s MSISDN: '%s'. Must be uppercase characters, numbers and underscores. ", name, potentialProfileName) + } +} + +func IccidWithoutLuhnChecksum(s string) string { + return loltelutils.TrimSuffix(s, 1) +} diff --git a/sim-administration/sim-batch-management/uploadtoprime/luhn_test.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go similarity index 97% rename from sim-administration/sim-batch-management/uploadtoprime/luhn_test.go rename to sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go index 524771436..563c7250d 100644 --- a/sim-administration/sim-batch-management/uploadtoprime/luhn_test.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go @@ -1,11 +1,9 @@ -package uploadtoprime - +package fieldsyntaxchecks import ( "testing" ) - func TestLuhn(t *testing.T) { validNumbers := []int{ 79927398713, diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 8a5f0d8ca..2d954d0f3 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -51,6 +51,18 @@ type OutputFileRecord struct { OutputFileName string } +type Batch struct { + Id int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` + Customer string `db:"customer" json:"customer"` + ProfileType string `db:"profileType" json:"profileType"` + OrderDate string `db:"orderDate" json:"orderDate"` + BatchNo string `db:"batchNo" json:"batchNo"` + Quantity int `db:"quantity" json:"quantity"` + FirstIccid string `db:"firstIccid" json:"firstIccid"` + FirstImsi string `db:"firstImsi" json:"firstImsi"` +} + type SimEntry struct { RawIccid string IccidWithChecksum string diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index d441b68a9..527daffa7 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -2,14 +2,20 @@ package main import ( + "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - // Don' lose this! - "gopkg.in/alecthomas/kingpin.v2" + + "log" + "strconv" ) +// "gopkg.in/alecthomas/kingpin.v2" var ( // TODO: Enable, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() @@ -108,37 +114,32 @@ var ( // TODO ??? batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") - db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") - // - // Set up command line parsing + // Declare a new batch // + db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") dbFirstIccid = db.Flag("first-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() dbLastIccid = db.Flag("last-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - dbFirstIMSI = db.Flag("first-imsi", "First IMSI in batch").Required().String() - dbLastIMSI = db.Flag("last-imsi", "Last IMSI in batch").Required().String() - dbFirstMsisdn = db.Flag("first-msisdn", "First MSISDN in batch").Required().String() - dbLastMsisdn = db.Flag("last-msisdn", "Last MSISDN in batch").Required().String() - dbProfileType = db.Flag("profile-type", "SIM profile type").Required().String() + dbFirstIMSI = db.Flag("first-imsi", "First IMSI in batch").Required().String() + dbLastIMSI = db.Flag("last-imsi", "Last IMSI in batch").Required().String() + dbFirstMsisdn = db.Flag("first-msisdn", "First MSISDN in batch").Required().String() + dbLastMsisdn = db.Flag("last-msisdn", "Last MSISDN in batch").Required().String() + dbProfileType = db.Flag("profile-type", "SIM profile type").Required().String() dbBatchLengthString = db.Flag( "batch-quantity", "Number of sim cards in batch").Required().String() - dbHssVendor = db.Flag("hss-vendor", "The HSS vendor").Default("M1").String() - dbUploadHostname = - db.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() - dbUploadPortnumber = - db.Flag("upload-portnumber", "port to upload to").Default("8080").String() + dbHssVendor = db.Flag("hss-vendor", "The HSS vendor").Default("M1").String() + dbUploadHostname = db.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() + dbUploadPortnumber = db.Flag("upload-portnumber", "port to upload to").Default("8080").String() - dbProfileVendor = - db.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() + dbProfileVendor = db.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() - dbInitialHlrActivationStatusOfProfiles = - db.Flag( - "initial-hlr-activation-status-of-profiles", - "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() + dbInitialHlrActivationStatusOfProfiles = db.Flag( + "initial-hlr-activation-status-of-profiles", + "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() ) func main() { @@ -336,9 +337,89 @@ func declareThisBatch( uploadHostname string, uploadPortnumber string, profileVendor string, - initialHlrActivationStatusOfProfiles string) { - fmt.Println("HOhoho, now we're declaring a batch!") - // 1. Check all the arguments (methods already written). - // 2. Check that the name isn't already registred. - // 3. If it isn't, then persist it + initialHlrActivationStatusOfProfiles string) model.OutputBatch { + fmt.Println("HOhoho, now we're declaring a batch!") + // 1. Check all the arguments (methods already written). + // 2. Check that the name isn't already registred. + // 3. If it isn't, then persist it + + // + // Check parameters for syntactic correctness and + // semantic sanity. + // + + fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", firstIccid) + fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", lastIccid) + fieldsyntaxchecks.CheckIMSISyntax("last-imsi", lastIMSI) + fieldsyntaxchecks.CheckIMSISyntax("first-imsi", firstIMSI) + fieldsyntaxchecks.CheckMSISDNSyntax("last-msisdn", lastMsisdn) + fieldsyntaxchecks.CheckMSISDNSyntax("first-msisdn", firstMsisdn) + + batchLength, err := strconv.Atoi(batchLengthString) + if err != nil { + log.Fatalf("Not a valid batch Quantity string '%s'.\n", batchLengthString) + } + + if batchLength <= 0 { + log.Fatalf("OutputBatch Quantity must be positive, but was '%d'", batchLength) + } + + uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", + uploadHostname, uploadPortnumber, hssVendor, profileVendor, initialHlrActivationStatusOfProfiles) + + fieldsyntaxchecks.CheckURLSyntax("uploadUrl", uploadUrl) + fieldsyntaxchecks.CheckProfileType("profile-type", profileType) + + // Convert to integers, and get lengths + msisdnIncrement := -1 + if firstMsisdn <= lastMsisdn { + msisdnIncrement = 1 + } + + log.Println("firstmsisdn = ", firstMsisdn) + log.Println("lastmsisdn = ", lastMsisdn) + log.Println("MsisdnIncrement = ", msisdnIncrement) + + var firstMsisdnInt, _ = strconv.Atoi(firstMsisdn) + var lastMsisdnInt, _ = strconv.Atoi(lastMsisdn) + var msisdnLen = lastMsisdnInt - firstMsisdnInt + 1 + if msisdnLen < 0 { + msisdnLen = -msisdnLen + } + + var firstImsiInt, _ = strconv.Atoi(firstIMSI) + var lastImsiInt, _ = strconv.Atoi(lastIMSI) + var imsiLen = lastImsiInt - firstImsiInt + 1 + + var firstIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(firstIccid)) + var lastIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(lastIccid)) + var iccidlen = lastIccidInt - firstIccidInt + 1 + + // Validate that lengths of sequences are equal in absolute + // values. + // TODO: Perhaps use some varargs trick of some sort here? + if loltelutils.Abs(msisdnLen) != loltelutils.Abs(iccidlen) || loltelutils.Abs(msisdnLen) != loltelutils.Abs(imsiLen) || batchLength != loltelutils.Abs(imsiLen) { + log.Printf("msisdnLen = %10d\n", msisdnLen) + log.Printf("iccidLen = %10d\n", iccidlen) + log.Printf("imsiLen = %10d\n", imsiLen) + log.Fatal("FATAL: msisdnLen, iccidLen and imsiLen are not identical.") + } + + tail := flag.Args() + if len(tail) != 0 { + log.Printf("Unknown parameters: %s", flag.Args()) + } + + // Return a correctly parsed batch + return model.OutputBatch{ + ProfileType: profileType, + Url: uploadUrl, + Length: loltelutils.Abs(iccidlen), + FirstIccid: firstIccidInt, + IccidIncrement: loltelutils.Sign(iccidlen), + FirstImsi: firstImsiInt, + ImsiIncrement: loltelutils.Sign(imsiLen), + FirstMsisdn: firstMsisdnInt, + MsisdnIncrement: msisdnIncrement, + } } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 95f71437c..b8cfbb22b 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -10,10 +10,10 @@ import ( type Store interface { GenerateTables() error - Create(theBatch *model.InputBatch) error - GetAllInputBatches(id string) ([]model.InputBatch, error) - GetInputBatchById(id int64) (*model.InputBatch, error) - GetInputBatchByName(id string) (*model.InputBatch, error) + Create(theBatch *model.Batch) error + GetAllInputBatches(id string) ([]model.Batch, error) + GetInputBatchById(id int64) (*model.Batch, error) + GetInputBatchByName(id string) (*model.Batch, error) } type SimBatchDB struct { @@ -42,22 +42,22 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } -func (sdb SimBatchDB) GetAllInputBatches() ([]model.InputBatch, error) { - result := []model.InputBatch{} +func (sdb SimBatchDB) GetAllInputBatches() ([]model.Batch, error) { + result := []model.Batch{} return result, sdb.Db.Select(&result, "SELECT * from INPUT_BATCH") } -func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.InputBatch, error) { - var result model.InputBatch +func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.Batch, error) { + var result model.Batch return &result, sdb.Db.Get(&result, "select * from INPUT_BATCH where id = ?", id) } -func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.InputBatch, error) { - var result model.InputBatch +func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { + var result model.Batch return &result, sdb.Db.Get(&result, "select * from INPUT_BATCH where name = ?", name) } -func (sdb SimBatchDB) Create(theBatch *model.InputBatch) { +func (sdb SimBatchDB) Create(theBatch *model.Batch) { res := sdb.Db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", (*theBatch).Name, diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index f4720543b..55adaaaeb 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -42,7 +42,7 @@ func TestMemoryDbPing(t *testing.T) { func TestInputBatchRoundtrip(t *testing.T) { - theBatch := model.InputBatch{ + theBatch := model.Batch{ Name: "SOME UNIQUE NAME", Customer: "firstInputBatch", ProfileType: "banana", diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index 74c47cd76..dd289861e 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -13,11 +13,10 @@ package uploadtoprime import ( "flag" "fmt" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "log" - "net/url" - "regexp" "strconv" "strings" ) @@ -30,43 +29,6 @@ func GeneratePostingCurlscript(url string, payload string) { fmt.Print("EOF\n") } -func generateControlDigit(luhnString string) int { - controlDigit := calculateChecksum(luhnString, true) % 10 - - if controlDigit != 0 { - controlDigit = 10 - controlDigit - } - - return controlDigit -} - -func calculateChecksum(luhnString string, double bool) int { - source := strings.Split(luhnString, "") - checksum := 0 - - for i := len(source) - 1; i > -1; i-- { - t, _ := strconv.ParseInt(source[i], 10, 8) - n := int(t) - - if double { - n = n * 2 - } - - double = !double - - if n >= 10 { - n = n - 9 - } - - checksum += n - } - return checksum -} - -func LuhnChecksum(number int) int { - return generateControlDigit(strconv.Itoa(number)) -} - func GenerateCsvPayload(batch model.OutputBatch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") @@ -77,7 +39,7 @@ func GenerateCsvPayload(batch model.OutputBatch) string { var msisdn = batch.FirstMsisdn for i := 0; i < batch.Length; i++ { - iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, LuhnChecksum(iccidWithoutLuhnChecksum)) + iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.ProfileType) sb.WriteString(line) @@ -89,68 +51,6 @@ func GenerateCsvPayload(batch model.OutputBatch) string { return sb.String() } -func isICCID(s string) bool { - match, _ := regexp.MatchString("^\\d{18}\\d?\\d?$", s) - return match -} - -func checkICCIDSyntax(name string, potentialIccid string) { - if !isICCID(potentialIccid) { - log.Fatalf("Not a valid %s ICCID: '%s'. Must be 18 or 19 (or 20) digits (_including_ luhn checksum).", name, potentialIccid) - } - - stringWithoutLuhnChecksum := IccidWithoutLuhnChecksum(potentialIccid) - controlDigit := generateControlDigit(stringWithoutLuhnChecksum) - checksummedCandidate := fmt.Sprintf("%s%d", stringWithoutLuhnChecksum, controlDigit) - if checksummedCandidate != potentialIccid { - log.Fatalf("Not a valid ICCID: '%s'. Expected luhn checksom '%d'", potentialIccid, controlDigit) - } -} - -func isIMSI(s string) bool { - match, _ := regexp.MatchString("^\\d{15}$", s) - return match -} - -func checkIMSISyntax(name string, potentialIMSI string) { - if !isIMSI(potentialIMSI) { - log.Fatalf("Not a valid %s IMSI: '%s'. Must be 15 digits.", name, potentialIMSI) - } -} - -func isMSISDN(s string) bool { - match, _ := regexp.MatchString("^\\d+$", s) - return match -} - -func checkMSISDNSyntax(name string, potentialMSISDN string) { - if !isMSISDN(potentialMSISDN) { - log.Fatalf("Not a valid %s MSISDN: '%s'. Must be non-empty sequence of digits.", name, potentialMSISDN) - } -} - -func checkURLSyntax(name string, theUrl string) { - _, err := url.ParseRequestURI(theUrl) - if err != nil { - log.Fatalf("Not a valid %s URL: '%s'.", name, theUrl) - } -} - -func isProfileName(s string) bool { - match, _ := regexp.MatchString("^[A-Z][A-Z0-9_]*$", s) - return match -} - -func checkProfileType(name string, potentialProfileName string) { - if !isProfileName(potentialProfileName) { - log.Fatalf("Not a valid %s MSISDN: '%s'. Must be uppercase characters, numbers and underscores. ", name, potentialProfileName) - } -} - -func IccidWithoutLuhnChecksum(s string) string { - return loltelutils.TrimSuffix(s, 1) -} - func ParseUploadFileGeneratorCommmandline() model.OutputBatch { // @@ -231,12 +131,12 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, // semantic sanity. // - checkICCIDSyntax("first-rawIccid", *firstIccid) - checkICCIDSyntax("last-rawIccid", *lastIccid) - checkIMSISyntax("last-imsi", *lastIMSI) - checkIMSISyntax("first-imsi", *firstIMSI) - checkMSISDNSyntax("last-msisdn", *lastMsisdn) - checkMSISDNSyntax("first-msisdn", *firstMsisdn) + fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", *firstIccid) + fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", *lastIccid) + fieldsyntaxchecks.CheckIMSISyntax("last-imsi", *lastIMSI) + fieldsyntaxchecks.CheckIMSISyntax("first-imsi", *firstIMSI) + fieldsyntaxchecks.CheckMSISDNSyntax("last-msisdn", *lastMsisdn) + fieldsyntaxchecks.CheckMSISDNSyntax("first-msisdn", *firstMsisdn) batchLength, err := strconv.Atoi(*batchLengthString) if err != nil { @@ -250,8 +150,8 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", *uploadHostname, *uploadPortnumber, *hssVendor, *profileVendor, *initialHlrActivationStatusOfProfiles) - checkURLSyntax("uploadUrl", uploadUrl) - checkProfileType("profile-type", *profileType) + fieldsyntaxchecks.CheckURLSyntax("uploadUrl", uploadUrl) + fieldsyntaxchecks.CheckProfileType("profile-type", *profileType) // Convert to integers, and get lengths msisdnIncrement := -1 @@ -274,8 +174,8 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, var lastImsiInt, _ = strconv.Atoi(*lastIMSI) var imsiLen = lastImsiInt - firstImsiInt + 1 - var firstIccidInt, _ = strconv.Atoi(IccidWithoutLuhnChecksum(*firstIccid)) - var lastIccidInt, _ = strconv.Atoi(IccidWithoutLuhnChecksum(*lastIccid)) + var firstIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(*firstIccid)) + var lastIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(*lastIccid)) var iccidlen = lastIccidInt - firstIccidInt + 1 // Validate that lengths of sequences are equal in absolute From e75797e11d0139b1ba6ef0adb147d6a695b62cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:33:52 +0100 Subject: [PATCH 090/309] add coment about toxic Ki --- sim-administration/sim-batch-management/model/model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 2d954d0f3..6459aa9e5 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -68,6 +68,6 @@ type SimEntry struct { IccidWithChecksum string IccidWithoutChecksum string Imsi string - Ki string + Ki string // XXX Toxic. Should never be stored persistently!! OutputFileName string } From ed5324f142bfdfa13d8224a517af21d1e67f1624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:44:15 +0100 Subject: [PATCH 091/309] rewrite a little more --- .../sim-batch-management/model/model.go | 23 +++++++++++-------- .../sim-batch-management/store/store.go | 10 ++++---- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 6459aa9e5..4aeebe4e7 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -51,6 +51,20 @@ type OutputFileRecord struct { OutputFileName string } +type SimEntry struct { + RawIccid string + IccidWithChecksum string + IccidWithoutChecksum string + Imsi string + Ki string // XXX Toxic. Should never be stored persistently!! + OutputFileName string +} + +// +// Below this line we grow the final persistence model. Eventually +// nothing below this line should be left. +// + type Batch struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` @@ -62,12 +76,3 @@ type Batch struct { FirstIccid string `db:"firstIccid" json:"firstIccid"` FirstImsi string `db:"firstImsi" json:"firstImsi"` } - -type SimEntry struct { - RawIccid string - IccidWithChecksum string - IccidWithoutChecksum string - Imsi string - Ki string // XXX Toxic. Should never be stored persistently!! - OutputFileName string -} diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index b8cfbb22b..7b36b9ef1 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -44,22 +44,22 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { func (sdb SimBatchDB) GetAllInputBatches() ([]model.Batch, error) { result := []model.Batch{} - return result, sdb.Db.Select(&result, "SELECT * from INPUT_BATCH") + return result, sdb.Db.Select(&result, "SELECT * from BATCH") } func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.Batch, error) { var result model.Batch - return &result, sdb.Db.Get(&result, "select * from INPUT_BATCH where id = ?", id) + return &result, sdb.Db.Get(&result, "select * from BATCH where id = ?", id) } func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { var result model.Batch - return &result, sdb.Db.Get(&result, "select * from INPUT_BATCH where name = ?", name) + return &result, sdb.Db.Get(&result, "select * from BATCH where name = ?", name) } func (sdb SimBatchDB) Create(theBatch *model.Batch) { - res := sdb.Db.MustExec("INSERT INTO INPUT_BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", + res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", (*theBatch).Name, (*theBatch).Customer, (*theBatch).ProfileType, @@ -78,7 +78,7 @@ func (sdb SimBatchDB) Create(theBatch *model.Batch) { } func (sdb *SimBatchDB) GenerateTables() error { - foo := `CREATE TABLE IF NOT EXISTS INPUT_BATCH ( + foo := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, customer VARCHAR NOT NULL, From 5b65c4435a02cb03f3d5e575b07da8df1668c534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:44:54 +0100 Subject: [PATCH 092/309] label the TODO list with TODO: --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 527daffa7..f639b61b9 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -339,6 +339,8 @@ func declareThisBatch( profileVendor string, initialHlrActivationStatusOfProfiles string) model.OutputBatch { fmt.Println("HOhoho, now we're declaring a batch!") + + // TODO: // 1. Check all the arguments (methods already written). // 2. Check that the name isn't already registred. // 3. If it isn't, then persist it From b652b392cf06cfaa60facec915180f637d20cd27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:46:27 +0100 Subject: [PATCH 093/309] Adding kingpin, which for some weird reason disapears all the time --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index f639b61b9..971bc7ce8 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -10,7 +10,7 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - + "gopkg.in/alecthomas/kingpin.v2" "log" "strconv" ) From ff20fba0eedd165f118443498a49109126b7be1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:51:05 +0100 Subject: [PATCH 094/309] Print fieldname when bugging out --- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 4 ++-- sim-administration/sim-batch-management/sim-batch-mgt.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go index b0577bdbe..50cb93c5a 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -54,14 +54,14 @@ func IsICCID(s string) bool { func CheckICCIDSyntax(name string, potentialIccid string) { if !IsICCID(potentialIccid) { - log.Fatalf("Not a valid %s ICCID: '%s'. Must be 18 or 19 (or 20) digits (_including_ luhn checksum).", name, potentialIccid) + log.Fatalf("Not a valid '%s' ICCID: '%s'. Must be 18 or 19 (or 20) digits (_including_ luhn checksum).", name, potentialIccid) } stringWithoutLuhnChecksum := IccidWithoutLuhnChecksum(potentialIccid) controlDigit := generateControlDigit(stringWithoutLuhnChecksum) checksummedCandidate := fmt.Sprintf("%s%d", stringWithoutLuhnChecksum, controlDigit) if checksummedCandidate != potentialIccid { - log.Fatalf("Not a valid ICCID: '%s'. Expected luhn checksom '%d'", potentialIccid, controlDigit) + log.Fatalf("Not a valid '%s' ICCID: '%s'. Expected luhn checksom '%d'", name, potentialIccid, controlDigit) } } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 971bc7ce8..d1bf69ec3 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -353,6 +353,7 @@ func declareThisBatch( fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", firstIccid) fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", lastIccid) fieldsyntaxchecks.CheckIMSISyntax("last-imsi", lastIMSI) + fieldsyntaxchecks.CheckIMSISyntax("first-imsi", firstIMSI) fieldsyntaxchecks.CheckMSISDNSyntax("last-msisdn", lastMsisdn) fieldsyntaxchecks.CheckMSISDNSyntax("first-msisdn", firstMsisdn) From 80e4081c1b9dec8a0fbe62bd15fb340d10a271c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 18:54:21 +0100 Subject: [PATCH 095/309] managed to correctly parse a batch --- sim-administration/sim-batch-management/sim-batch-mgt.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index d1bf69ec3..f7c26f9bc 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -379,10 +379,6 @@ func declareThisBatch( msisdnIncrement = 1 } - log.Println("firstmsisdn = ", firstMsisdn) - log.Println("lastmsisdn = ", lastMsisdn) - log.Println("MsisdnIncrement = ", msisdnIncrement) - var firstMsisdnInt, _ = strconv.Atoi(firstMsisdn) var lastMsisdnInt, _ = strconv.Atoi(lastMsisdn) var msisdnLen = lastMsisdnInt - firstMsisdnInt + 1 @@ -413,6 +409,7 @@ func declareThisBatch( log.Printf("Unknown parameters: %s", flag.Args()) } + fmt.Println("Correctly parsed a batch") // Return a correctly parsed batch return model.OutputBatch{ ProfileType: profileType, From 74ccc61721ce02ccd95b92cc92fe5b02d0755cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 19:11:06 +0100 Subject: [PATCH 096/309] Able to generate a batch entry --- .../sim-batch-management/model/model.go | 23 +++++++++++-------- .../sim-batch-management/sim-batch-mgt.go | 16 ++++++------- .../sim-batch-management/store/store.go | 17 ++++++++++++-- .../sim-batch-management/store/store_test.go | 2 ++ 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 4aeebe4e7..9ca1e4e97 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -66,13 +66,18 @@ type SimEntry struct { // type Batch struct { - Id int64 `db:"id" json:"id"` - Name string `db:"name" json:"name"` - Customer string `db:"customer" json:"customer"` - ProfileType string `db:"profileType" json:"profileType"` - OrderDate string `db:"orderDate" json:"orderDate"` - BatchNo string `db:"batchNo" json:"batchNo"` - Quantity int `db:"quantity" json:"quantity"` - FirstIccid string `db:"firstIccid" json:"firstIccid"` - FirstImsi string `db:"firstImsi" json:"firstImsi"` + Id int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` + Customer string `db:"customer" json:"customer"` + ProfileType string `db:"profileType" json:"profileType"` + OrderDate string `db:"orderDate" json:"orderDate"` + BatchNo string `db:"batchNo" json:"batchNo"` + Quantity int `db:"quantity" json:"quantity"` + FirstIccid string `db:"firstIccid" json:"firstIccid"` + FirstImsi string `db:"firstImsi" json:"firstImsi"` + Url string `db:"url" json:"url"` + MsisdnIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` + IccidIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` + ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` + FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index f7c26f9bc..35fc696b9 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -10,9 +10,9 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - "gopkg.in/alecthomas/kingpin.v2" "log" "strconv" + "gopkg.in/alecthomas/kingpin.v2" ) // "gopkg.in/alecthomas/kingpin.v2" @@ -337,8 +337,7 @@ func declareThisBatch( uploadHostname string, uploadPortnumber string, profileVendor string, - initialHlrActivationStatusOfProfiles string) model.OutputBatch { - fmt.Println("HOhoho, now we're declaring a batch!") + initialHlrActivationStatusOfProfiles string) model.Batch { // TODO: // 1. Check all the arguments (methods already written). @@ -409,17 +408,16 @@ func declareThisBatch( log.Printf("Unknown parameters: %s", flag.Args()) } - fmt.Println("Correctly parsed a batch") // Return a correctly parsed batch - return model.OutputBatch{ + return model.Batch{ ProfileType: profileType, Url: uploadUrl, - Length: loltelutils.Abs(iccidlen), - FirstIccid: firstIccidInt, + Quantity: loltelutils.Abs(iccidlen), + FirstIccid: firstIccid, IccidIncrement: loltelutils.Sign(iccidlen), - FirstImsi: firstImsiInt, + FirstImsi: firstIMSI, ImsiIncrement: loltelutils.Sign(imsiLen), - FirstMsisdn: firstMsisdnInt, + FirstMsisdn: firstMsisdn, MsisdnIncrement: msisdnIncrement, } } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 7b36b9ef1..fe1bb0958 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -59,7 +59,7 @@ func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) Create(theBatch *model.Batch) { - res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi) values (?,?,?,?,?,?,?,?) ", + res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstMsisdn, firstImsi, msisdnIncrement, iccidIncrement, firstMsisdn, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, (*theBatch).Customer, (*theBatch).ProfileType, @@ -68,6 +68,12 @@ func (sdb SimBatchDB) Create(theBatch *model.Batch) { (*theBatch).Quantity, (*theBatch).FirstIccid, (*theBatch).FirstImsi, + (*theBatch).FirstMsisdn, + (*theBatch).MsisdnIncrement, + (*theBatch).IccidIncrement, + (*theBatch).ImsiIncrement, + (*theBatch).FirstMsisdn, + (*theBatch).Url, ) id, err := res.LastInsertId() @@ -87,7 +93,14 @@ func (sdb *SimBatchDB) GenerateTables() error { batchNo VARCHAR NOT NULL, quantity INTEGER NOT NULL, firstIccid VARCHAR, - firstImsi VARCHAR + firstImsi VARCHAR, + firstMsisdn VARCHAR, + msisdnIncrement INTEGER, + imsiIncrement INTEGER, + iccidIncrement INTEGER, + iccidIncrement INTEGER, + firstMsisdn VARCHAR, + url VARCHAR )` _, err := sdb.Db.Exec(foo) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 55adaaaeb..401db041c 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -49,8 +49,10 @@ func TestInputBatchRoundtrip(t *testing.T) { OrderDate: "apple", BatchNo: "100", Quantity: 100, + Url: "http://vg.no", FirstIccid: "1234567890123456789", FirstImsi: "123456789012345", + MsisdnIncrement: -1, } sdb.Create(&theBatch) From f46a5565e28641d57f4388ad55d510f0550f7ba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 19:28:42 +0100 Subject: [PATCH 097/309] Getting our feet wet in actual persisting batches --- .../sim-batch-management/sim-batch-mgt.go | 11 +++++++++++ .../sim-batch-management/store/store.go | 1 + 2 files changed, 12 insertions(+) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 35fc696b9..b04e4b7e3 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -10,6 +10,7 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "log" "strconv" "gopkg.in/alecthomas/kingpin.v2" @@ -144,6 +145,13 @@ var ( func main() { + db, err := store.OpenFileSqliteDatabase("foobar.db") + if err != nil { + panic(fmt.Sprintf("Couldn't open sqlite database. '%s'", err)) + } + + db.GenerateTables() + cmd := kingpin.Parse() switch cmd { case "es2plus-smoketest": @@ -153,6 +161,7 @@ func main() { case "declare-batch": fmt.Println("Declare batch") declareThisBatch( + db, *dbFirstIccid, *dbLastIccid, *dbFirstIMSI, @@ -325,6 +334,7 @@ func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *strin // XXX Put this into a separate package at some point, "batch_editor" or something // equally descriptive. func declareThisBatch( + db *store.SimBatchDB, firstIccid string, lastIccid string, firstIMSI string, @@ -409,6 +419,7 @@ func declareThisBatch( } // Return a correctly parsed batch + // TODO: Batch name missing! return model.Batch{ ProfileType: profileType, Url: uploadUrl, diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index fe1bb0958..eda5a19ca 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -2,6 +2,7 @@ package store import ( "fmt" + _ "github.com/mattn/go-sqlite3" "github.com/jmoiron/sqlx" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "os" From 6e25b39bbf493f379539eaa58be729a1be7384f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sat, 2 Nov 2019 19:30:35 +0100 Subject: [PATCH 098/309] Getting our feet wet in actual persisting batches --- sim-administration/sim-batch-management/sim-batch-mgt.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index b04e4b7e3..9fb8e1e62 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -420,7 +420,7 @@ func declareThisBatch( // Return a correctly parsed batch // TODO: Batch name missing! - return model.Batch{ + myBatch := model.Batch{ ProfileType: profileType, Url: uploadUrl, Quantity: loltelutils.Abs(iccidlen), @@ -431,4 +431,6 @@ func declareThisBatch( FirstMsisdn: firstMsisdn, MsisdnIncrement: msisdnIncrement, } + + db.Create(&myBatch) } From da200fb25c1e483909407d8d8d591a4a45dec24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sun, 3 Nov 2019 18:40:38 +0100 Subject: [PATCH 099/309] Added a function to vet input parameters before storing in database, seems to work (as far as it is supposed to, so far) --- .../sim-batch-management/sim-batch-mgt.go | 110 +------------- .../sim-batch-management/store/store.go | 140 +++++++++++++++++- .../sim-batch-management/store/store_test.go | 45 ++++-- 3 files changed, 169 insertions(+), 126 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 9fb8e1e62..4b7e2ba99 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -9,11 +9,10 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" "log" "strconv" - "gopkg.in/alecthomas/kingpin.v2" ) // "gopkg.in/alecthomas/kingpin.v2" @@ -160,8 +159,7 @@ func main() { outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "declare-batch": fmt.Println("Declare batch") - declareThisBatch( - db, + DeclareBatch(db, *dbFirstIccid, *dbLastIccid, *dbFirstIMSI, @@ -330,107 +328,3 @@ func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *strin fmt.Println("Success") } - -// XXX Put this into a separate package at some point, "batch_editor" or something -// equally descriptive. -func declareThisBatch( - db *store.SimBatchDB, - firstIccid string, - lastIccid string, - firstIMSI string, - lastIMSI string, - firstMsisdn string, - lastMsisdn string, - profileType string, - batchLengthString string, - hssVendor string, - uploadHostname string, - uploadPortnumber string, - profileVendor string, - initialHlrActivationStatusOfProfiles string) model.Batch { - - // TODO: - // 1. Check all the arguments (methods already written). - // 2. Check that the name isn't already registred. - // 3. If it isn't, then persist it - - // - // Check parameters for syntactic correctness and - // semantic sanity. - // - - fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", firstIccid) - fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", lastIccid) - fieldsyntaxchecks.CheckIMSISyntax("last-imsi", lastIMSI) - - fieldsyntaxchecks.CheckIMSISyntax("first-imsi", firstIMSI) - fieldsyntaxchecks.CheckMSISDNSyntax("last-msisdn", lastMsisdn) - fieldsyntaxchecks.CheckMSISDNSyntax("first-msisdn", firstMsisdn) - - batchLength, err := strconv.Atoi(batchLengthString) - if err != nil { - log.Fatalf("Not a valid batch Quantity string '%s'.\n", batchLengthString) - } - - if batchLength <= 0 { - log.Fatalf("OutputBatch Quantity must be positive, but was '%d'", batchLength) - } - - uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", - uploadHostname, uploadPortnumber, hssVendor, profileVendor, initialHlrActivationStatusOfProfiles) - - fieldsyntaxchecks.CheckURLSyntax("uploadUrl", uploadUrl) - fieldsyntaxchecks.CheckProfileType("profile-type", profileType) - - // Convert to integers, and get lengths - msisdnIncrement := -1 - if firstMsisdn <= lastMsisdn { - msisdnIncrement = 1 - } - - var firstMsisdnInt, _ = strconv.Atoi(firstMsisdn) - var lastMsisdnInt, _ = strconv.Atoi(lastMsisdn) - var msisdnLen = lastMsisdnInt - firstMsisdnInt + 1 - if msisdnLen < 0 { - msisdnLen = -msisdnLen - } - - var firstImsiInt, _ = strconv.Atoi(firstIMSI) - var lastImsiInt, _ = strconv.Atoi(lastIMSI) - var imsiLen = lastImsiInt - firstImsiInt + 1 - - var firstIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(firstIccid)) - var lastIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(lastIccid)) - var iccidlen = lastIccidInt - firstIccidInt + 1 - - // Validate that lengths of sequences are equal in absolute - // values. - // TODO: Perhaps use some varargs trick of some sort here? - if loltelutils.Abs(msisdnLen) != loltelutils.Abs(iccidlen) || loltelutils.Abs(msisdnLen) != loltelutils.Abs(imsiLen) || batchLength != loltelutils.Abs(imsiLen) { - log.Printf("msisdnLen = %10d\n", msisdnLen) - log.Printf("iccidLen = %10d\n", iccidlen) - log.Printf("imsiLen = %10d\n", imsiLen) - log.Fatal("FATAL: msisdnLen, iccidLen and imsiLen are not identical.") - } - - tail := flag.Args() - if len(tail) != 0 { - log.Printf("Unknown parameters: %s", flag.Args()) - } - - // Return a correctly parsed batch - // TODO: Batch name missing! - myBatch := model.Batch{ - ProfileType: profileType, - Url: uploadUrl, - Quantity: loltelutils.Abs(iccidlen), - FirstIccid: firstIccid, - IccidIncrement: loltelutils.Sign(iccidlen), - FirstImsi: firstIMSI, - ImsiIncrement: loltelutils.Sign(imsiLen), - FirstMsisdn: firstMsisdn, - MsisdnIncrement: msisdnIncrement, - } - - db.Create(&myBatch) -} diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index eda5a19ca..abf16e4ec 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -1,11 +1,17 @@ package store import ( + "flag" "fmt" - _ "github.com/mattn/go-sqlite3" "github.com/jmoiron/sqlx" + _ "github.com/mattn/go-sqlite3" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" + + "log" "os" + "strconv" ) type Store interface { @@ -15,6 +21,22 @@ type Store interface { GetAllInputBatches(id string) ([]model.Batch, error) GetInputBatchById(id int64) (*model.Batch, error) GetInputBatchByName(id string) (*model.Batch, error) + + DeclareBatch( + db *SimBatchDB, + firstIccid string, + lastIccid string, + firstIMSI string, + lastIMSI string, + firstMsisdn string, + lastMsisdn string, + profileType string, + batchLengthString string, + hssVendor string, + uploadHostname string, + uploadPortnumber string, + profileVendor string, + initialHlrActivationStatusOfProfiles string) (*model.Batch, error) } type SimBatchDB struct { @@ -58,9 +80,9 @@ func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { return &result, sdb.Db.Get(&result, "select * from BATCH where name = ?", name) } -func (sdb SimBatchDB) Create(theBatch *model.Batch) { +func (sdb SimBatchDB) Create(theBatch *model.Batch) error { - res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstMsisdn, firstImsi, msisdnIncrement, iccidIncrement, firstMsisdn, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", + res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, (*theBatch).Customer, (*theBatch).ProfileType, @@ -73,7 +95,6 @@ func (sdb SimBatchDB) Create(theBatch *model.Batch) { (*theBatch).MsisdnIncrement, (*theBatch).IccidIncrement, (*theBatch).ImsiIncrement, - (*theBatch).FirstMsisdn, (*theBatch).Url, ) @@ -82,6 +103,7 @@ func (sdb SimBatchDB) Create(theBatch *model.Batch) { fmt.Errorf("Getting last inserted id failed '%s'", err) } theBatch.Id = id + return err } func (sdb *SimBatchDB) GenerateTables() error { @@ -99,11 +121,115 @@ func (sdb *SimBatchDB) GenerateTables() error { msisdnIncrement INTEGER, imsiIncrement INTEGER, iccidIncrement INTEGER, - iccidIncrement INTEGER, - firstMsisdn VARCHAR, url VARCHAR )` - _, err := sdb.Db.Exec(foo) return err } + +/** + * Create a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. + */ +func (sdb SimBatchDB) DeclareBatch( + firstIccid string, + lastIccid string, + firstIMSI string, + lastIMSI string, + firstMsisdn string, + lastMsisdn string, + profileType string, + batchLengthString string, + hssVendor string, + uploadHostname string, + uploadPortnumber string, + profileVendor string, + initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { + + // TODO: + // 1. Check all the arguments (methods already written). + // 2. Check that the name isn't already registred. + // 3. If it isn't, then persist it + + // + // Check parameters for syntactic correctness and + // semantic sanity. + // + + fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", firstIccid) + fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", lastIccid) + fieldsyntaxchecks.CheckIMSISyntax("last-imsi", lastIMSI) + + fieldsyntaxchecks.CheckIMSISyntax("first-imsi", firstIMSI) + fieldsyntaxchecks.CheckMSISDNSyntax("last-msisdn", lastMsisdn) + fieldsyntaxchecks.CheckMSISDNSyntax("first-msisdn", firstMsisdn) + + batchLength, err := strconv.Atoi(batchLengthString) + if err != nil { + log.Fatalf("Not a valid batch Quantity string '%s'.\n", batchLengthString) + } + + if batchLength <= 0 { + log.Fatalf("OutputBatch Quantity must be positive, but was '%d'", batchLength) + } + + uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", + uploadHostname, uploadPortnumber, hssVendor, profileVendor, initialHlrActivationStatusOfProfiles) + + fieldsyntaxchecks.CheckURLSyntax("uploadUrl", uploadUrl) + fieldsyntaxchecks.CheckProfileType("profile-type", profileType) + + // Convert to integers, and get lengths + msisdnIncrement := -1 + if firstMsisdn <= lastMsisdn { + msisdnIncrement = 1 + } + + var firstMsisdnInt, _ = strconv.Atoi(firstMsisdn) + var lastMsisdnInt, _ = strconv.Atoi(lastMsisdn) + var msisdnLen = lastMsisdnInt - firstMsisdnInt + 1 + if msisdnLen < 0 { + msisdnLen = -msisdnLen + } + + var firstImsiInt, _ = strconv.Atoi(firstIMSI) + var lastImsiInt, _ = strconv.Atoi(lastIMSI) + var imsiLen = lastImsiInt - firstImsiInt + 1 + + var firstIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(firstIccid)) + var lastIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(lastIccid)) + var iccidlen = lastIccidInt - firstIccidInt + 1 + + // Validate that lengths of sequences are equal in absolute + // values. + // TODO: Perhaps use some varargs trick of some sort here? + if loltelutils.Abs(msisdnLen) != loltelutils.Abs(iccidlen) || loltelutils.Abs(msisdnLen) != loltelutils.Abs(imsiLen) || batchLength != loltelutils.Abs(imsiLen) { + log.Printf("msisdnLen = %10d\n", msisdnLen) + log.Printf("iccidLen = %10d\n", iccidlen) + log.Printf("imsiLen = %10d\n", imsiLen) + log.Fatal("FATAL: msisdnLen, iccidLen and imsiLen are not identical.") + } + + tail := flag.Args() + if len(tail) != 0 { + log.Printf("Unknown parameters: %s", flag.Args()) + } + + // Return a correctly parsed batch + // TODO: Batch name missing! + myBatch := model.Batch{ + Name: "TODO fixme", + ProfileType: profileType, + Url: uploadUrl, + Quantity: loltelutils.Abs(iccidlen), + FirstIccid: firstIccid, + IccidIncrement: loltelutils.Sign(iccidlen), + FirstImsi: firstIMSI, + ImsiIncrement: loltelutils.Sign(imsiLen), + FirstMsisdn: firstMsisdn, + MsisdnIncrement: msisdnIncrement, + } + + // Create the new batch (checking for consistency not done!) + err = sdb.Create(&myBatch) + return &myBatch, err +} diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 401db041c..c9f0d932e 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -24,9 +24,12 @@ func TestMain(m *testing.M) { func setup() { sdb, err = store.NewInMemoryDatabase() if err != nil { - fmt.Errorf("Couldn't open new in memory database") + fmt.Errorf("Couldn't open new in memory database '%s", err) + } + err = sdb.GenerateTables() + if err != nil { + fmt.Errorf("Couldn't generate tables '%s'", err) } - sdb.GenerateTables() } func shutdown() { @@ -43,15 +46,15 @@ func TestMemoryDbPing(t *testing.T) { func TestInputBatchRoundtrip(t *testing.T) { theBatch := model.Batch{ - Name: "SOME UNIQUE NAME", - Customer: "firstInputBatch", - ProfileType: "banana", - OrderDate: "apple", - BatchNo: "100", - Quantity: 100, - Url: "http://vg.no", - FirstIccid: "1234567890123456789", - FirstImsi: "123456789012345", + Name: "SOME UNIQUE NAME", + Customer: "firstInputBatch", + ProfileType: "banana", + OrderDate: "apple", + BatchNo: "100", + Quantity: 100, + Url: "http://vg.no", + FirstIccid: "1234567890123456789", + FirstImsi: "123456789012345", MsisdnIncrement: -1, } @@ -68,3 +71,23 @@ func TestInputBatchRoundtrip(t *testing.T) { fmt.Errorf("getBatchById failed") } } + +func TestDeclareBatch(t *testing.T) { + _, err := sdb.DeclareBatch( + "89148000000745809013", // firstIccid string, + "89148000000745809013", // lastIccid string, + "242017100012213", // firstIMSI string, + "242017100012213", // lastIMSI string, + "47900184", // firstMsisdn string, + "47900184", // lastMsisdn string, + "BAR_FOOTEL_STD", //profileType string, + "1", // batchLengthString string, + "LOL", // hssVendor string, + "localhost", // uploadHostname string, + "8088", // uploadPortnumber string, + "snuff", // profileVendor string, + "ACTIVE") // initialHlrActivationStatusOfProfiles string + if err != nil { + panic(err) + } +} From 08ae240eaedf70e28eed25c60aa85bd27d331095 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sun, 3 Nov 2019 18:57:53 +0100 Subject: [PATCH 100/309] Steps toward also having an order date, but not there yet --- .../sim-batch-management/sim-batch-mgt.go | 11 ++++------- .../sim-batch-management/store/store.go | 5 ++++- .../sim-batch-management/store/store_test.go | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 4b7e2ba99..2fc5a574d 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -2,17 +2,12 @@ package main import ( - "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - "log" - "strconv" + "gopkg.in/alecthomas/kingpin.v2" ) // "gopkg.in/alecthomas/kingpin.v2" @@ -118,6 +113,7 @@ var ( // Declare a new batch // db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") + dbOrderDate = db.Flag("order-date", "Order date in format ddmmyyyy").Required().String() dbFirstIccid = db.Flag("first-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() dbLastIccid = db.Flag("last-rawIccid", @@ -159,7 +155,8 @@ func main() { outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "declare-batch": fmt.Println("Declare batch") - DeclareBatch(db, + db.DeclareBatch( + *dbOrderDate, *dbFirstIccid, *dbLastIccid, *dbFirstIMSI, diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index abf16e4ec..479cae06d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -82,8 +82,9 @@ func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) Create(theBatch *model.Batch) error { - res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?) ", + res := sdb.Db.MustExec("INSERT INTO BATCH (name, orderDate, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, + (*theBatch).OrderDate, (*theBatch).Customer, (*theBatch).ProfileType, (*theBatch).OrderDate, @@ -131,6 +132,7 @@ func (sdb *SimBatchDB) GenerateTables() error { * Create a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. */ func (sdb SimBatchDB) DeclareBatch( + orderDate string, firstIccid string, lastIccid string, firstIMSI string, @@ -217,6 +219,7 @@ func (sdb SimBatchDB) DeclareBatch( // Return a correctly parsed batch // TODO: Batch name missing! myBatch := model.Batch{ + OrderDate: orderDate, Name: "TODO fixme", ProfileType: profileType, Url: uploadUrl, diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index c9f0d932e..50408dd9c 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -74,6 +74,7 @@ func TestInputBatchRoundtrip(t *testing.T) { func TestDeclareBatch(t *testing.T) { _, err := sdb.DeclareBatch( + "20200101", // date string "89148000000745809013", // firstIccid string, "89148000000745809013", // lastIccid string, "242017100012213", // firstIMSI string, From 08cade199a083271cb89ee4b1ce6489fbb0683c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sun, 3 Nov 2019 20:45:36 +0100 Subject: [PATCH 101/309] Add more to the stored table (some dummy values) --- .../sim-batch-management/store/store.go | 8 +++++--- .../sim-batch-management/store/store_test.go | 11 ++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 479cae06d..cc33cd1dc 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -82,12 +82,12 @@ func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) Create(theBatch *model.Batch) error { - res := sdb.Db.MustExec("INSERT INTO BATCH (name, orderDate, customer, profileType, orderDate, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", + res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, + (*theBatch).Customer, (*theBatch).OrderDate, (*theBatch).Customer, (*theBatch).ProfileType, - (*theBatch).OrderDate, (*theBatch).BatchNo, (*theBatch).Quantity, (*theBatch).FirstIccid, @@ -220,7 +220,9 @@ func (sdb SimBatchDB) DeclareBatch( // TODO: Batch name missing! myBatch := model.Batch{ OrderDate: orderDate, - Name: "TODO fixme", + Customer: "NOT A CUSTOMER FIXME", + Name: "TODO fixme Name", + BatchNo: "TODO FIXME BatchNo", ProfileType: profileType, Url: uploadUrl, Quantity: loltelutils.Abs(iccidlen), diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 50408dd9c..460c5f5d9 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -47,9 +47,9 @@ func TestInputBatchRoundtrip(t *testing.T) { theBatch := model.Batch{ Name: "SOME UNIQUE NAME", + OrderDate: "20200101", Customer: "firstInputBatch", ProfileType: "banana", - OrderDate: "apple", BatchNo: "100", Quantity: 100, Url: "http://vg.no", @@ -67,13 +67,13 @@ func TestInputBatchRoundtrip(t *testing.T) { assert.Equal(t, len(allBatches), 1) firstInputBatch, _ := sdb.GetInputBatchById(1) - if !reflect.DeepEqual(firstInputBatch, theBatch) { + if !reflect.DeepEqual(*firstInputBatch, theBatch) { fmt.Errorf("getBatchById failed") } } func TestDeclareBatch(t *testing.T) { - _, err := sdb.DeclareBatch( + theBatch, err := sdb.DeclareBatch( "20200101", // date string "89148000000745809013", // firstIccid string, "89148000000745809013", // lastIccid string, @@ -91,4 +91,9 @@ func TestDeclareBatch(t *testing.T) { if err != nil { panic(err) } + + retrievedValue, _ := sdb.GetInputBatchById(theBatch.Id) + if !reflect.DeepEqual(*retrievedValue, *theBatch) { + fmt.Errorf("getBatchById failed") + } } From ee4a9b3f424debbd635df7f77927178eccd644b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sun, 3 Nov 2019 20:58:12 +0100 Subject: [PATCH 102/309] Now we're able to add full lines, but there is no test for referential integrity, so that's what we need to add now --- sim-administration/sim-batch-management/sim-batch-mgt.go | 8 +++++++- sim-administration/sim-batch-management/store/store.go | 9 ++++++--- .../sim-batch-management/store/store_test.go | 3 +++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 2fc5a574d..7ad935888 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -7,7 +7,7 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - "gopkg.in/alecthomas/kingpin.v2" + kingpin "gopkg.in/alecthomas/kingpin.v2" ) // "gopkg.in/alecthomas/kingpin.v2" @@ -113,6 +113,9 @@ var ( // Declare a new batch // db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") + dbName = db.Flag("name", "Unique name of this batch").Required().String() + dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() + dbBatchNo = db.Flag("batch-no", "Unique number of this batch (with respect to the profile vendor)").Required().String() dbOrderDate = db.Flag("order-date", "Order date in format ddmmyyyy").Required().String() dbFirstIccid = db.Flag("first-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() @@ -156,6 +159,9 @@ func main() { case "declare-batch": fmt.Println("Declare batch") db.DeclareBatch( + *dbName, + *dbCustomer, + *dbBatchNo, *dbOrderDate, *dbFirstIccid, *dbLastIccid, diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index cc33cd1dc..0b49f0ead 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -132,6 +132,9 @@ func (sdb *SimBatchDB) GenerateTables() error { * Create a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. */ func (sdb SimBatchDB) DeclareBatch( + name string, + customer string, + batchNo string, orderDate string, firstIccid string, lastIccid string, @@ -220,9 +223,9 @@ func (sdb SimBatchDB) DeclareBatch( // TODO: Batch name missing! myBatch := model.Batch{ OrderDate: orderDate, - Customer: "NOT A CUSTOMER FIXME", - Name: "TODO fixme Name", - BatchNo: "TODO FIXME BatchNo", + Customer: customer, + Name: name, + BatchNo: batchNo, ProfileType: profileType, Url: uploadUrl, Quantity: loltelutils.Abs(iccidlen), diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 460c5f5d9..876b45b43 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -74,6 +74,9 @@ func TestInputBatchRoundtrip(t *testing.T) { func TestDeclareBatch(t *testing.T) { theBatch, err := sdb.DeclareBatch( + "Name", + "Customer", + "8778fsda", // batch number "20200101", // date string "89148000000745809013", // firstIccid string, "89148000000745809013", // lastIccid string, From b2a1d640ef6d06997d83eb944cd8e62e1ee2997c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sun, 3 Nov 2019 21:33:35 +0100 Subject: [PATCH 103/309] This fails, lack of isolation between tests. Seems like tables are not dropped. Don't know why. --- .../sim-batch-management/model/model.go | 28 ++++++------- .../sim-batch-management/store/store.go | 19 ++++++--- .../sim-batch-management/store/store_test.go | 42 +++++++++++++++---- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 9ca1e4e97..7ab907184 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -66,18 +66,18 @@ type SimEntry struct { // type Batch struct { - Id int64 `db:"id" json:"id"` - Name string `db:"name" json:"name"` - Customer string `db:"customer" json:"customer"` - ProfileType string `db:"profileType" json:"profileType"` - OrderDate string `db:"orderDate" json:"orderDate"` - BatchNo string `db:"batchNo" json:"batchNo"` - Quantity int `db:"quantity" json:"quantity"` - FirstIccid string `db:"firstIccid" json:"firstIccid"` - FirstImsi string `db:"firstImsi" json:"firstImsi"` - Url string `db:"url" json:"url"` - MsisdnIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` - IccidIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` - ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` - FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` + Id int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` + Customer string `db:"customer" json:"customer"` + ProfileType string `db:"profileType" json:"profileType"` + OrderDate string `db:"orderDate" json:"orderDate"` + BatchNo string `db:"batchNo" json:"batchNo"` + Quantity int `db:"quantity" json:"quantity"` + FirstIccid string `db:"firstIccid" json:"firstIccid"` + FirstImsi string `db:"firstImsi" json:"firstImsi"` + Url string `db:"url" json:"url"` + MsisdnIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` + IccidIncrement int `db:"iccidIncrement" json:"msisdnIncrement"` + ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` + FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 0b49f0ead..8fc8f25e0 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -16,11 +16,12 @@ import ( type Store interface { GenerateTables() error + DropTables() error Create(theBatch *model.Batch) error - GetAllInputBatches(id string) ([]model.Batch, error) - GetInputBatchById(id int64) (*model.Batch, error) - GetInputBatchByName(id string) (*model.Batch, error) + GetAllBatches(id string) ([]model.Batch, error) + GetBatchById(id int64) (*model.Batch, error) + GetBatchByName(id string) (*model.Batch, error) DeclareBatch( db *SimBatchDB, @@ -65,17 +66,17 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } -func (sdb SimBatchDB) GetAllInputBatches() ([]model.Batch, error) { +func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { result := []model.Batch{} return result, sdb.Db.Select(&result, "SELECT * from BATCH") } -func (sdb SimBatchDB) GetInputBatchById(id int64) (*model.Batch, error) { +func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { var result model.Batch return &result, sdb.Db.Get(&result, "select * from BATCH where id = ?", id) } -func (sdb SimBatchDB) GetInputBatchByName(name string) (*model.Batch, error) { +func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { var result model.Batch return &result, sdb.Db.Get(&result, "select * from BATCH where name = ?", name) } @@ -128,6 +129,12 @@ func (sdb *SimBatchDB) GenerateTables() error { return err } +func (sdb *SimBatchDB) DropTables() error { + foo := `DROP TABLE BATCH` + _, err := sdb.Db.Exec(foo) + return err +} + /** * Create a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. */ diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 876b45b43..766462f34 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -22,6 +22,7 @@ func TestMain(m *testing.M) { } func setup() { + // sdb, err = store.OpenFileSqliteDatabase("bazunka.db") sdb, err = store.NewInMemoryDatabase() if err != nil { fmt.Errorf("Couldn't open new in memory database '%s", err) @@ -33,6 +34,10 @@ func setup() { } func shutdown() { + sdb.DropTables() + if err != nil { + fmt.Errorf("Couldn't drop tables '%s'", err) + } } // ... just to know that everything is sane. @@ -43,8 +48,7 @@ func TestMemoryDbPing(t *testing.T) { } } -func TestInputBatchRoundtrip(t *testing.T) { - +func injectTestBatch() *model.Batch { theBatch := model.Batch{ Name: "SOME UNIQUE NAME", OrderDate: "20200101", @@ -58,16 +62,36 @@ func TestInputBatchRoundtrip(t *testing.T) { MsisdnIncrement: -1, } - sdb.Create(&theBatch) - allBatches, err := sdb.GetAllInputBatches() + err := sdb.Create(&theBatch) + if err != nil { + panic(err) + } + return &theBatch +} + +func TestGetBatchById(t *testing.T) { + + theBatch := injectTestBatch() + + firstInputBatch, _ := sdb.GetBatchById(1) + if !reflect.DeepEqual(*firstInputBatch, *theBatch) { + fmt.Errorf("getBatchById failed") + } +} + +func TestGetAllBatches(t *testing.T) { + + theBatch := injectTestBatch() + + allBatches, err := sdb.GetAllBatches() if err != nil { fmt.Errorf("Reading query failed '%s'", err) } assert.Equal(t, len(allBatches), 1) - firstInputBatch, _ := sdb.GetInputBatchById(1) - if !reflect.DeepEqual(*firstInputBatch, theBatch) { + firstInputBatch := allBatches[0] + if !reflect.DeepEqual(firstInputBatch, theBatch) { fmt.Errorf("getBatchById failed") } } @@ -76,8 +100,8 @@ func TestDeclareBatch(t *testing.T) { theBatch, err := sdb.DeclareBatch( "Name", "Customer", - "8778fsda", // batch number - "20200101", // date string + "8778fsda", // batch number + "20200101", // date string "89148000000745809013", // firstIccid string, "89148000000745809013", // lastIccid string, "242017100012213", // firstIMSI string, @@ -95,7 +119,7 @@ func TestDeclareBatch(t *testing.T) { panic(err) } - retrievedValue, _ := sdb.GetInputBatchById(theBatch.Id) + retrievedValue, _ := sdb.GetBatchById(theBatch.Id) if !reflect.DeepEqual(*retrievedValue, *theBatch) { fmt.Errorf("getBatchById failed") } From bfb7d94198e8161320b554923ac9d4a1ea740bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Sun, 3 Nov 2019 21:40:50 +0100 Subject: [PATCH 104/309] Update TODO --- .../sim-batch-management/TODO.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index 6fc674648..325dbe193 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,21 +1,24 @@ An informal TODO list for the sim batch management tool == -1. Make persistence-model for generated sim profiles. This database should then be used - to generate uploads of various kinds to prime. This essentially means mirroring - and extending the data model for sim profiles currently in Prime. 1. Rewrite upload-sim-batch-lib-test.go to be part of sim-batch-mgt.go, during that process: - * Persist the batch data. - * List available batches, show the status. - * Make a command that can print the input file for that batch. - * Find some clever way to get legacy batches into the database without typing too much. + * Persist the batch data [Done] + * List available batches, show the status (use json). + * Persist access parameters for Prime(s) (prod, and pre-prod) (indirectly). + * Persist access parameters for sim vendors and HSSes + * Check referential integrity in data model so that batches don't refer to + HSSes or other types of entities. + * From the batch information, generate + - Input file to SIM vendor. + - Upload file for Prime instances. + * From output file, generate input file for HSS without storing Ki values. 1. Add crypto resources so that the program can talk to external parties. 1. Figure out how to handle workflows. Be explicit! 1. Handle both parameterized lists of MSISDNs and list-based input. 1. The interfaces to external parties will be - input/output files for profile generation. - - some kind of file (not yet determind) for msisdn lists. + - some kind of file (not yet determined) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or directly from the script later. In either case it will be assumed that tunnels are set up out of band, and From af9f11eb860896981fed4066282b0b44cc3f4310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 4 Nov 2019 13:48:00 +0100 Subject: [PATCH 105/309] We need to keep som tempfiles for ad-hoc testing that shouldn't be part of the repo --- sim-administration/sim-batch-management/.gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sim-administration/sim-batch-management/.gitignore b/sim-administration/sim-batch-management/.gitignore index 9e9c72b3e..8826fe49c 100644 --- a/sim-administration/sim-batch-management/.gitignore +++ b/sim-administration/sim-batch-management/.gitignore @@ -1,3 +1,5 @@ *.sh es2pluswrapper.sh batch-lifecycle-test.sh +tmp + From 28d210a8423690222546a4264ac4452cec457f46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 4 Nov 2019 16:05:53 +0100 Subject: [PATCH 106/309] Also print the batch length when it is found to be the wrong one --- .../sim-batch-management/uploadtoprime/upload-sim-batch-lib.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index dd289861e..608edd9e5 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -182,6 +182,7 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, // values. // TODO: Perhaps use some varargs trick of some sort here? if loltelutils.Abs(msisdnLen) != loltelutils.Abs(iccidlen) || loltelutils.Abs(msisdnLen) != loltelutils.Abs(imsiLen) || batchLength != loltelutils.Abs(imsiLen) { + log.Printf("batchLength = %10d\n", batchLength) log.Printf("msisdnLen = %10d\n", msisdnLen) log.Printf("iccidLen = %10d\n", iccidlen) log.Printf("imsiLen = %10d\n", imsiLen) From 3c7a4ce8035802900e1f9c0a433db1f9c145dc1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 4 Nov 2019 16:54:55 +0100 Subject: [PATCH 107/309] Attempting to make an activation command, so far it's failing. --- .../sim-batch-management/sim-batch-mgt.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 7ad935888..3d277f3da 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -28,7 +28,7 @@ var ( es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile ").Required().String() es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() - es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Required().String() + es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() es2KeyFilePath = es2.Flag("key", "Certificate key file.").Required().String() es2Hostport = es2.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() @@ -205,7 +205,7 @@ func main() { if err != nil { panic(err) } - fmt.Println("result -> ", result.State) + fmt.Println("iccid='%s', state='%s', acToken='%s'\n", iccid, result.State, result.ACToken) case "recover-profile": checkEs2TargetState(es2Target) result, err := es2plus.RecoverProfile(client, iccid, *es2Target) @@ -225,6 +225,21 @@ func main() { panic(err) } fmt.Println("result -> ", result) + case "activate-iccid": + _, err := es2plus.DownloadOrder(client, iccid) + if err != nil { + panic(err) + } + _, err = es2plus.ConfirmOrder(client, iccid) + if err != nil { + panic(err) + } + result, err := es2plus.GetStatus(client, iccid) + if err != nil { + panic(err) + } + fmt.Printf("%s, %s", iccid, result.ACToken) + case "cancel-profile": checkEs2TargetState(es2Target) _, err := es2plus.CancelOrder(client, iccid, *es2Target) From 39bee43ac5cb1e129627e82485760942b4cc59bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 10:43:52 +0100 Subject: [PATCH 108/309] Progress towards paralell activation of profiles --- .../sim-batch-management/es2plus/es2plus.go | 26 ++++++++++- .../sim-batch-management/sim-batch-mgt.go | 43 ++++++++++++++----- 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index fd92ed167..ed484874a 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -301,7 +301,8 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { - result := new(ES2ProfileStatusResponse) + + result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" header := newEs2plusHeader(client) payload := &ES2PlusGetProfileStatusRequest{ @@ -314,6 +315,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { } if (len(result.ProfileStatusList) == 0) { + fmt.Sprintf("No results found for iccid = '%s'", iccid) return nil, nil } else if (len(result.ProfileStatusList) == 1) { returnvalue := result.ProfileStatusList[0] @@ -399,3 +401,25 @@ func ConfirmOrder(client *Es2PlusClient, iccid string) (*ES2PlusConfirmOrderResp return result, nil } } + +func ActivateIccid(client *Es2PlusClient, iccid string) (*ProfileStatus, error){ + + result, err := GetStatus(client, iccid) + if err != nil { + panic(err) + } + + if result.ACToken == "" { + + _, err := DownloadOrder(client, iccid) + if err != nil { + return nil, err + } + _, err = ConfirmOrder(client, iccid) + if err != nil { + return nil, err + } + } + result, err = GetStatus(client, iccid) + return result, err +} diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 3d277f3da..1701bd3f1 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -2,12 +2,16 @@ package main import ( + "bufio" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" kingpin "gopkg.in/alecthomas/kingpin.v2" + "log" + "os" + "sync" ) // "gopkg.in/alecthomas/kingpin.v2" @@ -26,7 +30,7 @@ var ( smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") - es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile ").Required().String() + es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() @@ -197,6 +201,8 @@ func main() { uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) case "es2": + + // TODO: Vet all the parameters, they can very easily be bogus. client := es2plus.Client(*es2CertFilePath, *es2KeyFilePath, *es2Hostport, *es2RequesterId) iccid := *es2iccid switch *es2cmd { @@ -205,7 +211,8 @@ func main() { if err != nil { panic(err) } - fmt.Println("iccid='%s', state='%s', acToken='%s'\n", iccid, result.State, result.ACToken) + + fmt.Printf("iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) case "recover-profile": checkEs2TargetState(es2Target) result, err := es2plus.RecoverProfile(client, iccid, *es2Target) @@ -226,19 +233,35 @@ func main() { } fmt.Println("result -> ", result) case "activate-iccid": - _, err := es2plus.DownloadOrder(client, iccid) + result, err := es2plus.ActivateIccid(client, iccid) + if err != nil { panic(err) } - _, err = es2plus.ConfirmOrder(client, iccid) + fmt.Printf("%s, %s\n", iccid, result.ACToken) + + case "bulk-activate-iccids": + fmt.Printf("Ready to bulk activate some iccids\n") + + file, err := os.Open(iccid) if err != nil { - panic(err) + log.Fatal(err) } - result, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) + defer file.Close() + + scanner := bufio.NewScanner(file) + var mutex = &sync.Mutex{} + for scanner.Scan() { + mutex.Lock() + fmt.Println("Iccid = ", scanner.Text()) + mutex.Unlock() } - fmt.Printf("%s, %s", iccid, result.ACToken) + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + fmt.Println("Done") + case "cancel-profile": checkEs2TargetState(es2Target) @@ -247,7 +270,7 @@ func main() { panic(err) } default: - panic("Unknown es2+ subcommand, try --help") + panic(fmt.Sprintf("Unknown es2+ subcommand '%s', try --help", *es2cmd)) } case "batch": fmt.Println("Doing the batch thing.") From 6c5e5097e24c6c3f8acc1b2dd0d75ac23fd767d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 11:04:20 +0100 Subject: [PATCH 109/309] Now with waitgroup --- .../sim-batch-management/sim-batch-mgt.go | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 1701bd3f1..c46fab147 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -117,10 +117,10 @@ var ( // Declare a new batch // db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") - dbName = db.Flag("name", "Unique name of this batch").Required().String() - dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() - dbBatchNo = db.Flag("batch-no", "Unique number of this batch (with respect to the profile vendor)").Required().String() - dbOrderDate = db.Flag("order-date", "Order date in format ddmmyyyy").Required().String() + dbName = db.Flag("name", "Unique name of this batch").Required().String() + dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() + dbBatchNo = db.Flag("batch-no", "Unique number of this batch (with respect to the profile vendor)").Required().String() + dbOrderDate = db.Flag("order-date", "Order date in format ddmmyyyy").Required().String() dbFirstIccid = db.Flag("first-rawIccid", "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() dbLastIccid = db.Flag("last-rawIccid", @@ -249,20 +249,37 @@ func main() { } defer file.Close() + + var iccids []string scanner := bufio.NewScanner(file) var mutex = &sync.Mutex{} for scanner.Scan() { - mutex.Lock() - fmt.Println("Iccid = ", scanner.Text()) - mutex.Unlock() + iccid := scanner.Text() + iccids = append(iccids, iccid) + } + + var waitgroup sync.WaitGroup + for _, iccid := range (iccids) { + waitgroup.Add(1) + go func() { + for { + // Mutex the printing, so that it won't + // print on top of someone else already printing. + mutex.Lock() + fmt.Println("Iccid = ", iccid) + mutex.Unlock() + waitgroup.Done() + } + }() } + waitgroup.Wait() + if err := scanner.Err(); err != nil { log.Fatal(err) } fmt.Println("Done") - case "cancel-profile": checkEs2TargetState(es2Target) _, err := es2plus.CancelOrder(client, iccid, *es2Target) From afc38191a7260d16e085d055583fafffc6abdcae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 11:13:01 +0100 Subject: [PATCH 110/309] Now works in simulated case --- .../sim-batch-management/sim-batch-mgt.go | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index c46fab147..214aee6de 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -249,28 +249,18 @@ func main() { } defer file.Close() - - var iccids []string scanner := bufio.NewScanner(file) var mutex = &sync.Mutex{} + var waitgroup sync.WaitGroup for scanner.Scan() { iccid := scanner.Text() - iccids = append(iccids, iccid) - } - - var waitgroup sync.WaitGroup - for _, iccid := range (iccids) { waitgroup.Add(1) - go func() { - for { - // Mutex the printing, so that it won't - // print on top of someone else already printing. - mutex.Lock() - fmt.Println("Iccid = ", iccid) - mutex.Unlock() - waitgroup.Done() - } - }() + go func(i string) { + mutex.Lock() + fmt.Println("Iccid = ", i) + mutex.Unlock() + waitgroup.Done() + }(iccid) } waitgroup.Wait() From dd66d2cdcd2a8e6b8b8ce1acfa32a990e5cc7918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 11:17:11 +0100 Subject: [PATCH 111/309] Now works in concurrent mode, but isn1t very fast --- sim-administration/sim-batch-management/sim-batch-mgt.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 214aee6de..065ea7726 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -257,7 +257,11 @@ func main() { waitgroup.Add(1) go func(i string) { mutex.Lock() - fmt.Println("Iccid = ", i) + result, err := es2plus.ActivateIccid(client, i) + if err != nil { + panic(err) + } + fmt.Println("Iccid = ", i, "activation code = ", result.ACToken) mutex.Unlock() waitgroup.Done() }(iccid) From 2b8f1af74ee80d6c357438be40a8cf60776931a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 11:26:51 +0100 Subject: [PATCH 112/309] Now works with synchronization --- sim-administration/sim-batch-management/sim-batch-mgt.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 065ea7726..707e6e0e9 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -241,7 +241,6 @@ func main() { fmt.Printf("%s, %s\n", iccid, result.ACToken) case "bulk-activate-iccids": - fmt.Printf("Ready to bulk activate some iccids\n") file, err := os.Open(iccid) if err != nil { @@ -256,12 +255,13 @@ func main() { iccid := scanner.Text() waitgroup.Add(1) go func(i string) { - mutex.Lock() + result, err := es2plus.ActivateIccid(client, i) if err != nil { panic(err) } - fmt.Println("Iccid = ", i, "activation code = ", result.ACToken) + mutex.Lock() + fmt.Printf("%s, %s\n", i, result.ACToken) mutex.Unlock() waitgroup.Done() }(iccid) @@ -272,7 +272,6 @@ func main() { if err := scanner.Err(); err != nil { log.Fatal(err) } - fmt.Println("Done") case "cancel-profile": checkEs2TargetState(es2Target) From 08d8db7e516ac01c4159c1890391f5afd2fb8b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 13:57:37 +0100 Subject: [PATCH 113/309] Smoketest going, this now works. --- .../sim-batch-management/sim-batch-mgt.go | 94 ------------------- 1 file changed, 94 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 707e6e0e9..fe0d41f7f 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -19,15 +19,6 @@ var ( // TODO: Enable, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - // - // Smoketest of the ES2Plus interface - // - smoketest = kingpin.Command("es2plus-smoketest", "Smoketest the ES2+ protocol.") - smoketestCertFilePath = smoketest.Flag("cert", "Certificate pem file.").Required().String() - smoketestKeyFilePath = smoketest.Flag("key", "Certificate key file.").Required().String() - smoketestHostport = smoketest.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() - smoketestRequesterId = smoketest.Flag("requesterid", "ES2+ requester ID.").Required().String() - smoketestIccidInput = smoketest.Flag("iccid", "Iccid of profile to manipulate").Required().String() es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() @@ -156,8 +147,6 @@ func main() { cmd := kingpin.Parse() switch cmd { - case "es2plus-smoketest": - es2PlusSmoketest(smoketestCertFilePath, smoketestKeyFilePath, smoketestHostport, smoketestRequesterId, smoketestIccidInput) case "sim-profile-upload": outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "declare-batch": @@ -296,86 +285,3 @@ func checkEs2TargetState(target *string) { } } -// TODO: This is just smoketest-code, delete it after -// the smoketest-script has been rewritten a bit to do the same thing. -func es2PlusSmoketest(certFilePath *string, keyFilePath *string, hostport *string, requesterId *string, iccidInput *string) { - - fmt.Printf("certFilePath = '%s'\n", *certFilePath) - fmt.Printf("keyFilePath = '%s'\n", *keyFilePath) - fmt.Printf("hostport = '%s'\n", *hostport) - fmt.Printf("requesterId = '%s'\n", *requesterId) - fmt.Printf("iccidInput = '%s'\n", *iccidInput) - - iccid := *iccidInput - - client := es2plus.Client(*certFilePath, *keyFilePath, *hostport, *requesterId) - - result, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - fmt.Println("result1 -> ", result.State) - - result2, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") - if err != nil { - panic(err) - } - - fmt.Println("result2 -> ", result2) - - result, err = es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - - fmt.Println("result3 -> ", result.State) - - // XXX Should be CancelProfile - result4, err := es2plus.RecoverProfile(client, iccid, "AVAILABLE") - if err != nil { - panic(err) - } - - fmt.Println("result4 -> ", result4) - - result, err = es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - - fmt.Println("result5 -> ", result.State) - - if result.State != "AVAILABLE" { - panic("Couldn't convert state of iccid into AVAILABLE") - } - - result6, err := es2plus.DownloadOrder(client, iccid) - fmt.Println("result6 -> ", result6) - if err != nil { - panic(err) - } - - result7, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - fmt.Println("result7 -> ", result7) - - result8, err := es2plus.ConfirmOrder(client, iccid) - fmt.Println("result8 -> ", result8) - if err != nil { - panic(err) - } - - result9, err := es2plus.GetStatus(client, iccid) - if err != nil { - panic(err) - } - fmt.Println("result9 -> ", result9) - - if result.State != "RELEASED" { - panic("Couldn't convert state of sim profile into RELEASED") - } - - fmt.Println("Success") -} From 7e2ade9c9a5ac6f0ff6c77f3f0f7f2dba150cbac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 14:15:46 +0100 Subject: [PATCH 114/309] Rewrite es2plus client to have nicer syntax (dotish) --- .../sim-batch-management/es2plus/es2plus.go | 52 +++++++++++++------ .../sim-batch-management/sim-batch-mgt.go | 14 ++--- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index ed484874a..d825affd2 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -13,6 +13,20 @@ import ( "github.com/google/uuid" ) +/// +/// External interface +/// +type Es2PlusClient interface { + GetStatus(iccid string) (*ProfileStatus, error) + RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) + CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) + DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) + ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) + ActivateIccid(iccid string) (*ProfileStatus, error) + RequesterId() (string) +} + + /// /// Generic headers for invocations and responses /// @@ -144,7 +158,7 @@ type ES2PlusConfirmOrderResponse struct { // -type Es2PlusClient struct { +type Es2PlusClientState struct { httpClient *http.Client hostport string requesterId string @@ -152,8 +166,8 @@ type Es2PlusClient struct { printHeaders bool } -func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClient) { - return &Es2PlusClient { +func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClientState) { + return &Es2PlusClientState { httpClient: newHttpClient(certFilePath, keyFilePath), hostport: hostport, requesterId: requesterId, @@ -223,16 +237,16 @@ func newUuid() (string, error) { return uuid.URN(), nil } -func newEs2plusHeader(client *Es2PlusClient) (*ES2PlusHeader) { +func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader) { functionCallIdentifier, err := newUuid() if err != nil { panic(err) } - return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.requesterId} + return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()} } -func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClient, es2plusCommand string, payload interface{}, result interface{}) error { +func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClientState, es2plusCommand string, payload interface{}, result interface{}) error { // Serialize payload as json. jsonStrB, err := json.Marshal(payload) @@ -300,9 +314,9 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma /// -func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { +func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error) { - result := new(ES2ProfileStatusResponse) + result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" header := newEs2plusHeader(client) payload := &ES2PlusGetProfileStatusRequest{ @@ -326,7 +340,7 @@ func GetStatus(client *Es2PlusClient, iccid string) (*ProfileStatus, error) { } -func RecoverProfile(client *Es2PlusClient, iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { +func(client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { result := new(ES2PlusRecoverProfileResponse) es2plusCommand := "recoverProfile" header := newEs2plusHeader(client) @@ -339,7 +353,7 @@ func RecoverProfile(client *Es2PlusClient, iccid string, targetState string) (*E } -func CancelOrder(client *Es2PlusClient, iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { +func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { result := new(ES2PlusCancelOrderResponse) es2plusCommand := "cancelOrder" header := newEs2plusHeader(client) @@ -351,7 +365,7 @@ func CancelOrder(client *Es2PlusClient, iccid string, targetState string) (*ES2P return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) } -func DownloadOrder(client *Es2PlusClient, iccid string) (*ES2PlusDownloadOrderResponse, error) { +func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { result := new(ES2PlusDownloadOrderResponse) es2plusCommand := "downloadOrder" header := newEs2plusHeader(client) @@ -375,7 +389,7 @@ func DownloadOrder(client *Es2PlusClient, iccid string) (*ES2PlusDownloadOrderRe } -func ConfirmOrder(client *Es2PlusClient, iccid string) (*ES2PlusConfirmOrderResponse, error) { +func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { result := new(ES2PlusConfirmOrderResponse) es2plusCommand := "confirmOrder" header := newEs2plusHeader(client) @@ -402,24 +416,28 @@ func ConfirmOrder(client *Es2PlusClient, iccid string) (*ES2PlusConfirmOrderResp } } -func ActivateIccid(client *Es2PlusClient, iccid string) (*ProfileStatus, error){ +func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error){ - result, err := GetStatus(client, iccid) + result, err := client.GetStatus(iccid) if err != nil { panic(err) } if result.ACToken == "" { - _, err := DownloadOrder(client, iccid) + _, err := client.DownloadOrder(iccid) if err != nil { return nil, err } - _, err = ConfirmOrder(client, iccid) + _, err = client.ConfirmOrder(iccid) if err != nil { return nil, err } } - result, err = GetStatus(client, iccid) + result, err = client.GetStatus(iccid) return result, err } + +func (clientState *Es2PlusClientState) RequesterId() (string){ + return clientState.requesterId + } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index fe0d41f7f..a3044808b 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -196,7 +196,7 @@ func main() { iccid := *es2iccid switch *es2cmd { case "get-status": - result, err := es2plus.GetStatus(client, iccid) + result, err := client.GetStatus(iccid) if err != nil { panic(err) } @@ -204,25 +204,25 @@ func main() { fmt.Printf("iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) case "recover-profile": checkEs2TargetState(es2Target) - result, err := es2plus.RecoverProfile(client, iccid, *es2Target) + result, err := client.RecoverProfile(iccid, *es2Target) if err != nil { panic(err) } fmt.Println("result -> ", result) case "download-order": - result, err := es2plus.DownloadOrder(client, iccid) + result, err := client.DownloadOrder(iccid) if err != nil { panic(err) } fmt.Println("result -> ", result) case "confirm-order": - result, err := es2plus.ConfirmOrder(client, iccid) + result, err := client.ConfirmOrder(iccid) if err != nil { panic(err) } fmt.Println("result -> ", result) case "activate-iccid": - result, err := es2plus.ActivateIccid(client, iccid) + result, err := client.ActivateIccid(iccid) if err != nil { panic(err) @@ -245,7 +245,7 @@ func main() { waitgroup.Add(1) go func(i string) { - result, err := es2plus.ActivateIccid(client, i) + result, err := client.ActivateIccid(i) if err != nil { panic(err) } @@ -264,7 +264,7 @@ func main() { case "cancel-profile": checkEs2TargetState(es2Target) - _, err := es2plus.CancelOrder(client, iccid, *es2Target) + _, err := client.CancelOrder(iccid, *es2Target) if err != nil { panic(err) } From e8435dcd6752d34e0649f0c936cbf5e50b34cfe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 14:16:42 +0100 Subject: [PATCH 115/309] Add comment about blemish --- sim-administration/sim-batch-management/es2plus/es2plus.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index d825affd2..5a104bc0a 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -438,6 +438,7 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, return result, err } +// TODO: This shouldn't have to be public, but how can it be avoided? func (clientState *Es2PlusClientState) RequesterId() (string){ return clientState.requesterId } From 8863ed1b84acbd1ef7496390429e03a8c5e6dfb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 14:17:51 +0100 Subject: [PATCH 116/309] Remove comment that doesn't help much --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index a3044808b..3cc94dcff 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -170,8 +170,6 @@ func main() { *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) case "prime-batch-upload": - // TODO: Combine these two into something inside uploadtoprime. - // It's unecessary to break the batch thingy open in this way. var batch = uploadtoprime.OutputBatchFromCommandLineParameters( pbuFirstIccid, pbuLastIccid, From 5d9700ead276b071c8e2fe455fea33456db16872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 16:47:33 +0100 Subject: [PATCH 117/309] Able to print a nice (ish) desdcription of batch. Moving on to json-ifying it for even more beauty. --- .../sim-batch-management/TODO.md | 14 ++++++- .../sim-batch-management/sim-batch-mgt.go | 37 ++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index 325dbe193..1a862228d 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,10 +1,14 @@ An informal TODO list for the sim batch management tool == +1. Ingest input files into a corresponding batch, read all the sim + profiles into a sim profile table. This table can then be enriched + with access codes, MSISDNs etc, and then written to both HSSes and + be transformed into various types of upload files. 1. Rewrite upload-sim-batch-lib-test.go to be part of sim-batch-mgt.go, during that process: * Persist the batch data [Done] - * List available batches, show the status (use json). + * List persisted batches, show the status (use json). * Persist access parameters for Prime(s) (prod, and pre-prod) (indirectly). * Persist access parameters for sim vendors and HSSes * Check referential integrity in data model so that batches don't refer to @@ -23,3 +27,11 @@ An informal TODO list for the sim batch management tool directly from the script later. In either case it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. + + + +Notes +== + + simmgr_inventory=> select count(*) from sim_entries where profile = 'OYA_M1_STANDARD_ACB' and smdpplusstate = 'RELEASED'; + \ No newline at end of file diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 3cc94dcff..7e6cd73ca 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -19,7 +19,6 @@ var ( // TODO: Enable, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() @@ -104,6 +103,13 @@ var ( // TODO ??? batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") + listBatches = kingpin.Command("list-batches", "List all known batches.") + describeBatch = kingpin.Command("describe-batch", "List all known batches.") + + describeBatchBatch = describeBatch.Arg("batch", "List all known batches.").String() + + // describeBatch = kingpin.Command("describe-batch", "List details about named batch") + // describeBatchName = describeBatch.Arg("name").Required().String() // // Declare a new batch // @@ -149,6 +155,34 @@ func main() { switch cmd { case "sim-profile-upload": outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) + + case "list-batches": + + allBatches, err := db.GetAllBatches() + if err != nil { + panic(err) + } + + fmt.Println("Names of current batches: ") + for _, batch := range allBatches { + fmt.Printf(" %s\n", batch.Name) + } + + case "describe-batch": + + fmt.Print("aaa") + batch, err := db.GetBatchByName(*describeBatchBatch) + if err != nil { + panic(err) + } + + fmt.Print("yho") + if batch == nil { + fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) + } else { + fmt.Printf("Batch = '%s'\n", batch) + } + case "declare-batch": fmt.Println("Declare batch") db.DeclareBatch( @@ -282,4 +316,3 @@ func checkEs2TargetState(target *string) { panic("Target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") } } - From 53aa53a94c0fd39639ea7857cb55cea8d599a253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 16:55:50 +0100 Subject: [PATCH 118/309] Pretty print json using nice indentation. --- .../sim-batch-management/sim-batch-mgt.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 7e6cd73ca..6764b8c10 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -3,6 +3,7 @@ package main import ( "bufio" + "encoding/json" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" @@ -170,17 +171,20 @@ func main() { case "describe-batch": - fmt.Print("aaa") batch, err := db.GetBatchByName(*describeBatchBatch) if err != nil { panic(err) } - fmt.Print("yho") if batch == nil { fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) } else { - fmt.Printf("Batch = '%s'\n", batch) + bytes, err := json.MarshalIndent(batch, "", " ") + if err != nil { + fmt.Println("Can't serialize", batch) + } + + fmt.Printf("%v'\n", string(bytes)) } case "declare-batch": From 58b5d0c2d4752eb460ab7a9e6a6e6704bc73f055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 16:56:20 +0100 Subject: [PATCH 119/309] Add a little more initial indentation. --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 6764b8c10..b92cba58d 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -179,7 +179,7 @@ func main() { if batch == nil { fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) } else { - bytes, err := json.MarshalIndent(batch, "", " ") + bytes, err := json.MarshalIndent(batch, " ", " ") if err != nil { fmt.Println("Can't serialize", batch) } From feae5c3b256a5342a81a7b8ca68a781a7e5f9f8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 5 Nov 2019 16:58:31 +0100 Subject: [PATCH 120/309] No trailing single quote. --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index b92cba58d..f87a57480 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -184,7 +184,7 @@ func main() { fmt.Println("Can't serialize", batch) } - fmt.Printf("%v'\n", string(bytes)) + fmt.Printf("%v\n", string(bytes)) } case "declare-batch": From e067ecce54092df5f743d2bfd20c7ae6df39101e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 08:42:56 +0100 Subject: [PATCH 121/309] Adding a new t4est fixture --- .../sim-batch-management/sample-out-2.out | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 sim-administration/sim-batch-management/sample-out-2.out diff --git a/sim-administration/sim-batch-management/sample-out-2.out b/sim-administration/sim-batch-management/sample-out-2.out new file mode 100644 index 000000000..9ffe317ea --- /dev/null +++ b/sim-administration/sim-batch-management/sample-out-2.out @@ -0,0 +1,35 @@ +*************************************** +* HEADER DESCRIPTION +*************************************** +Customer: FooTel Telecommunications Inc. +Quantity: 3 +Type: FOOTEL_STANDARD +Profile: 1.0 +Batch: 01074809 +* +Transport_key: 11 +* +Address1: FooTel Telecommunications Inc. +Address2: Suite 3, floor 99 +Address3: Shebang Hi-Tech Industrial Park +Address4: 4711 +Address5: Deep trench plaza +Address6: Atlantis +* +Graph_ref: 1.0 +* +PO_ref_number: 010748 +*************************************** +* INPUT VARIABLES DESCRIPTION +*************************************** +Var_In_List: +IMSI:471161085310000 +Ser_nb:89601619103100049794 + +OUTPUT VARIABLES +var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4 +47111611031000501294 471161085310000 1234 12345678 1234 12345678 12345678 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 0001 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 11 8123 1 1 +47111611031000501294 471161085310000 1234 12345678 1234 12345678 12345678 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 0001 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 11 8123 1 1 +47111611031000501294 471161085310000 1234 12345678 1234 12345678 12345678 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 0001 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 11 8123 1 1 + + From fa127c77531c4ed6fb131cc74c15504eecd2e335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 08:51:07 +0100 Subject: [PATCH 122/309] Adding beginning of new test to start work on complex outfiles --- ...outfile_to_hss_input_converter_lib_test.go | 8 ++++- .../sim-batch-management/sample-out-2.out | 35 ------------------- .../sample_out_file_for_testing.out | 21 ----------- 3 files changed, 7 insertions(+), 57 deletions(-) delete mode 100644 sim-administration/sim-batch-management/sample-out-2.out delete mode 100644 sim-administration/sim-batch-management/sample_out_file_for_testing.out diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go index 0d6a9bda7..6621bae36 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go @@ -12,7 +12,7 @@ func testKeywordValueParser(t *testing.T) { assert.Equal(t, "BAR_FOOTEL_STD", theMap["ProfileType"]) } -func testReadOutputFile(t *testing.T) { +func testReadingSimpleOutputFile(t *testing.T) { sample_output_file_name := "sample_out_file_for_testing.out" record := ReadOutputFile(sample_output_file_name) @@ -34,3 +34,9 @@ func testReadOutputFile(t *testing.T) { assert.Equal(t, 3, len(record.Entries)) assert.Equal(t, 3, record.noOfEntries) } + + +func testReadingComplexOutputFile(t *testing.T) { + sample_output_file_name := "sample_out_file_for_testing.out" + record := ReadOutputFile(sample_output_file_name) +} diff --git a/sim-administration/sim-batch-management/sample-out-2.out b/sim-administration/sim-batch-management/sample-out-2.out deleted file mode 100644 index 9ffe317ea..000000000 --- a/sim-administration/sim-batch-management/sample-out-2.out +++ /dev/null @@ -1,35 +0,0 @@ -*************************************** -* HEADER DESCRIPTION -*************************************** -Customer: FooTel Telecommunications Inc. -Quantity: 3 -Type: FOOTEL_STANDARD -Profile: 1.0 -Batch: 01074809 -* -Transport_key: 11 -* -Address1: FooTel Telecommunications Inc. -Address2: Suite 3, floor 99 -Address3: Shebang Hi-Tech Industrial Park -Address4: 4711 -Address5: Deep trench plaza -Address6: Atlantis -* -Graph_ref: 1.0 -* -PO_ref_number: 010748 -*************************************** -* INPUT VARIABLES DESCRIPTION -*************************************** -Var_In_List: -IMSI:471161085310000 -Ser_nb:89601619103100049794 - -OUTPUT VARIABLES -var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4 -47111611031000501294 471161085310000 1234 12345678 1234 12345678 12345678 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 0001 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 11 8123 1 1 -47111611031000501294 471161085310000 1234 12345678 1234 12345678 12345678 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 0001 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 11 8123 1 1 -47111611031000501294 471161085310000 1234 12345678 1234 12345678 12345678 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 0001 ABBCCCAABBCCAABBCCAABBCCAABBCCAB 11 8123 1 1 - - diff --git a/sim-administration/sim-batch-management/sample_out_file_for_testing.out b/sim-administration/sim-batch-management/sample_out_file_for_testing.out deleted file mode 100644 index 28f23d15e..000000000 --- a/sim-administration/sim-batch-management/sample_out_file_for_testing.out +++ /dev/null @@ -1,21 +0,0 @@ -*HEADER DESCRIPTION -*************************************** -Customer : Footel -ProfileType : BAR_FOOTEL_STD -Order Date : 2019092901 -Batch No : 2019092901 -Quantity : 3 -*************************************** -*INPUT VARIABLES -*************************************** -var_In: - ICCID: 8947000000000012141 -IMSI: 242017100011213 -*************************************** -*OUTPUT VARIABLES -*************************************** -var_Out: ICCID/IMSI/KI -8947000000000012140F 242017100011213 AC63D37F5AF4EA77E3539B2A37F51211 -8947000000000012157F 242017100011214 E397EF06A41B78D7F9E0302E763E103C -8947000000000012165F 242017100011215 F8946C451E0E87DC2D5AA4846F0C1336 - From 0404ed810bedb7a09a9b2fb7752ac4e5a51492fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 08:56:18 +0100 Subject: [PATCH 123/309] Updating tests --- .../outfile_to_hss_input_converter_lib.go | 4 ++-- ...outfile_to_hss_input_converter_lib_test.go | 21 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go index af6c1f6bc..043c2d985 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go @@ -58,7 +58,7 @@ type ParserState struct { entries []model.SimEntry } -func ReadOutputFile(filename string) model.OutputFileRecord { +func ParseOutputFile(filename string) model.OutputFileRecord { _, err := os.Stat(filename) @@ -277,7 +277,7 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { // func ConvertInputfileToOutputfile(inputFile string, outputFilePrefix string) { - outRecord := ReadOutputFile(inputFile) + outRecord := ParseOutputFile(inputFile) outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" fmt.Println("outputFile = ", outputFile) diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go index 6621bae36..f1a8791d0 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go @@ -14,29 +14,28 @@ func testKeywordValueParser(t *testing.T) { func testReadingSimpleOutputFile(t *testing.T) { sample_output_file_name := "sample_out_file_for_testing.out" - record := ReadOutputFile(sample_output_file_name) + record := ParseOutputFile(sample_output_file_name) // First parameter to check assert.Equal(t, sample_output_file_name, record.Filename) // Check that all the header variables are there - assert.Equal(t, record.headerDescription["Customer"], "Footel") - assert.Equal(t, record.headerDescription["ProfileType"], "BAR_FOOTEL_STD") - assert.Equal(t, record.headerDescription["Order Date"], "2019092901") - assert.Equal(t, record.headerDescription["Batch No"], "2019092901") - assert.Equal(t, record.headerDescription["Quantity"], "3") + assert.Equal(t, record.HeaderDescription["Customer"], "Footel") + assert.Equal(t, record.HeaderDescription["ProfileType"], "BAR_FOOTEL_STD") + assert.Equal(t, record.HeaderDescription["Order Date"], "2019092901") + assert.Equal(t, record.HeaderDescription["Batch No"], "2019092901") + assert.Equal(t, record.HeaderDescription["Quantity"], "3") // Check all the input variables are there - assert.Equal(t, record.inputVariables["ICCID"], "8947000000000012141") - assert.Equal(t, record.inputVariables["IMSI"], "242017100011213") + assert.Equal(t, record.InputVariables["ICCID"], "8947000000000012141") + assert.Equal(t, record.InputVariables["IMSI"], "242017100011213") // Check that the output entry set looks legit. assert.Equal(t, 3, len(record.Entries)) - assert.Equal(t, 3, record.noOfEntries) + assert.Equal(t, 3, record.NoOfEntries) } - func testReadingComplexOutputFile(t *testing.T) { sample_output_file_name := "sample_out_file_for_testing.out" - record := ReadOutputFile(sample_output_file_name) + record := ParseOutputFile(sample_output_file_name) } From fcc4d1ef345762ac2f04acb556e389afb9bdb477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 09:39:14 +0100 Subject: [PATCH 124/309] Make tests public, so that they run --- .../outfile_to_hss_input_converter_lib.go | 2 ++ .../outfile_to_hss_input_converter_lib_test.go | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go index 043c2d985..840984e65 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go @@ -1,5 +1,7 @@ package outfileconversion +// TODO: Rename to oufileparser + import ( "bufio" "flag" diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go index f1a8791d0..8e354dd5a 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go +++ b/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go @@ -3,16 +3,18 @@ package outfileconversion import ( "gotest.tools/assert" "testing" + "fmt" ) -func testKeywordValueParser(t *testing.T) { + +func TestKeywordValueParser(t *testing.T) { theMap := make(map[string]string) ParseLineIntoKeyValueMap("ProfileType : BAR_FOOTEL_STD", theMap) assert.Equal(t, "BAR_FOOTEL_STD", theMap["ProfileType"]) } -func testReadingSimpleOutputFile(t *testing.T) { +func TestReadingSimpleOutputFile(t *testing.T) { sample_output_file_name := "sample_out_file_for_testing.out" record := ParseOutputFile(sample_output_file_name) @@ -35,7 +37,8 @@ func testReadingSimpleOutputFile(t *testing.T) { assert.Equal(t, 3, record.NoOfEntries) } -func testReadingComplexOutputFile(t *testing.T) { +func TestReadingComplexOutputFile(t *testing.T) { sample_output_file_name := "sample_out_file_for_testing.out" record := ParseOutputFile(sample_output_file_name) + fmt.Println("Record = ",record) } From 62744e117987c1b8042a094ed05f42b6fa47730d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 09:46:26 +0100 Subject: [PATCH 125/309] Changing package name to something more reasonable --- .../outfileparser.go} | 2 +- .../outfileparser_test.go} | 6 +++--- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename sim-administration/sim-batch-management/{outfileconversion/outfile_to_hss_input_converter_lib.go => outfileparser/outfileparser.go} (99%) rename sim-administration/sim-batch-management/{outfileconversion/outfile_to_hss_input_converter_lib_test.go => outfileparser/outfileparser_test.go} (93%) diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go similarity index 99% rename from sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go rename to sim-administration/sim-batch-management/outfileparser/outfileparser.go index 840984e65..27d4b93ef 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -1,4 +1,4 @@ -package outfileconversion +package outfileparser // TODO: Rename to oufileparser diff --git a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go similarity index 93% rename from sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go rename to sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 8e354dd5a..88190eb7b 100644 --- a/sim-administration/sim-batch-management/outfileconversion/outfile_to_hss_input_converter_lib_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -1,9 +1,9 @@ -package outfileconversion +package outfileparser import ( + "fmt" "gotest.tools/assert" "testing" - "fmt" ) @@ -38,7 +38,7 @@ func TestReadingSimpleOutputFile(t *testing.T) { } func TestReadingComplexOutputFile(t *testing.T) { - sample_output_file_name := "sample_out_file_for_testing.out" + sample_output_file_name := "sample-out-2.out" record := ParseOutputFile(sample_output_file_name) fmt.Println("Record = ",record) } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index f87a57480..1072bc106 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -6,7 +6,7 @@ import ( "encoding/json" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileconversion" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" kingpin "gopkg.in/alecthomas/kingpin.v2" From f847c204692cf5afd5be4b1a0c4526b25583161a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 09:50:48 +0100 Subject: [PATCH 126/309] Adding todos --- .../sim-batch-management/outfileparser/outfileparser.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 27d4b93ef..7924ed5c6 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -244,6 +244,8 @@ func fileExists(filename string) bool { return !info.IsDir() } + +// TODO: Move this into some other package. "hssoutput" or something. // TODO: Consider rewriting using https://golang.org/pkg/encoding/csv/ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { @@ -276,6 +278,7 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { // // Entrypoint +// TODO: Move this entrypoint into the the command processor // func ConvertInputfileToOutputfile(inputFile string, outputFilePrefix string) { From 52ccf383e47f1f3e3388de143ebfee6990ff5953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 11:02:18 +0100 Subject: [PATCH 127/309] Now we have a parser for the var_out line --- .../outfileparser/outfileparser_test.go | 55 ++++++++++++++++++- .../sim-batch-management/sim-batch-mgt.go | 1 - 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 88190eb7b..284d7d104 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -1,12 +1,14 @@ package outfileparser import ( + "errors" "fmt" "gotest.tools/assert" + "strings" "testing" ) - +/* func TestKeywordValueParser(t *testing.T) { theMap := make(map[string]string) ParseLineIntoKeyValueMap("ProfileType : BAR_FOOTEL_STD", theMap) @@ -36,9 +38,58 @@ func TestReadingSimpleOutputFile(t *testing.T) { assert.Equal(t, 3, len(record.Entries)) assert.Equal(t, 3, record.NoOfEntries) } +*/ + +/* func TestReadingComplexOutputFile(t *testing.T) { sample_output_file_name := "sample-out-2.out" record := ParseOutputFile(sample_output_file_name) - fmt.Println("Record = ",record) + fmt.Println("Record = ", record) +} +*/ + +func ParseVarOutLine(varOutLine string) (map[string]int, error) { + varOutSplit := strings.Split(varOutLine, ":") + + if (len(varOutSplit) != 2) { + fmt.Println("Length = ", len(varOutSplit)) + return nil, errors.New("syntax error in var_out line. More than two colon separated fields.") + } + + if (string(varOutSplit[0]) != "var_out") { + return nil, errors.New("syntax error in var_out line. Does not start with'var_out:'") + } + + slashedFields := strings.Split(varOutSplit[1], "/") + var result = map[string]int{} + for index, columnName := range slashedFields { + result[columnName] = index + } + return result, nil +} + +func TestParseOutputVariablesLine(t *testing.T) { + varOutLine := "var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4" + + m, err := ParseVarOutLine(varOutLine) + if err != nil { + t.Error("Couldn't parse var_out line:", err) + t.Fail() + } + + assert.Equal(t, m["ICCID"], 0) + assert.Equal(t, m["IMSI"], 1) + assert.Equal(t, m["PIN1"], 2) + assert.Equal(t, m["PUK1"], 3) + assert.Equal(t, m["PIN2"], 4) + assert.Equal(t, m["PUK2"], 5) + assert.Equal(t, m["ADM1"], 6) + assert.Equal(t, m["KI"], 7) + assert.Equal(t, m["Access_Control"], 8) + assert.Equal(t, m["Code Retailer"], 9) + assert.Equal(t, m["Code ADM"], 10) + assert.Equal(t, m["ADM2"], 11) + assert.Equal(t, m["ADM3"], 12) + assert.Equal(t, m["ADM4"], 13) } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 1072bc106..80c8f0e1e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" kingpin "gopkg.in/alecthomas/kingpin.v2" From 9f75947c8b07e5eb7e47a49633e8ee38947bc38c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 11:07:45 +0100 Subject: [PATCH 128/309] Passing tests --- .../outfileparser/outfileparser.go | 24 +++++++++++++- .../outfileparser/outfileparser_test.go | 32 +++---------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 7924ed5c6..62cc04711 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -13,6 +13,7 @@ import ( "regexp" "strconv" "strings" + "errors" ) const ( @@ -60,6 +61,27 @@ type ParserState struct { entries []model.SimEntry } +func ParseVarOutLine(varOutLine string) (map[string]int, error) { + varOutSplit := strings.Split(varOutLine, ":") + + if (len(varOutSplit) != 2) { + fmt.Println("Length = ", len(varOutSplit)) + return nil, errors.New("syntax error in var_out line. More than two colon separated fields.") + } + + if (string(varOutSplit[0]) != "var_out") { + return nil, errors.New("syntax error in var_out line. Does not start with'var_out:'") + } + + slashedFields := strings.Split(varOutSplit[1], "/") + var result = map[string]int{} + for index, columnName := range slashedFields { + result[columnName] = index + } + return result, nil +} + + func ParseOutputFile(filename string) model.OutputFileRecord { _, err := os.Stat(filename) @@ -161,7 +183,7 @@ func ParseOutputFile(filename string) model.OutputFileRecord { declaredNoOfEntities, err := strconv.Atoi(state.headerDescription["Quantity"]) if err != nil { - log.Fatal("Could not find declared quantity of entities") + log.Fatal("Could not find 'Quantity' field while parsing file '",filename, "'") } if countedNoOfEntries != declaredNoOfEntities { diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 284d7d104..6216e9963 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -1,14 +1,11 @@ package outfileparser import ( - "errors" - "fmt" "gotest.tools/assert" - "strings" "testing" ) -/* + func TestKeywordValueParser(t *testing.T) { theMap := make(map[string]string) ParseLineIntoKeyValueMap("ProfileType : BAR_FOOTEL_STD", theMap) @@ -38,36 +35,17 @@ func TestReadingSimpleOutputFile(t *testing.T) { assert.Equal(t, 3, len(record.Entries)) assert.Equal(t, 3, record.NoOfEntries) } -*/ -/* + +/* TODO: Uncomment this test, and start on making it pass. func TestReadingComplexOutputFile(t *testing.T) { sample_output_file_name := "sample-out-2.out" record := ParseOutputFile(sample_output_file_name) fmt.Println("Record = ", record) } -*/ - -func ParseVarOutLine(varOutLine string) (map[string]int, error) { - varOutSplit := strings.Split(varOutLine, ":") - - if (len(varOutSplit) != 2) { - fmt.Println("Length = ", len(varOutSplit)) - return nil, errors.New("syntax error in var_out line. More than two colon separated fields.") - } + */ - if (string(varOutSplit[0]) != "var_out") { - return nil, errors.New("syntax error in var_out line. Does not start with'var_out:'") - } - - slashedFields := strings.Split(varOutSplit[1], "/") - var result = map[string]int{} - for index, columnName := range slashedFields { - result[columnName] = index - } - return result, nil -} func TestParseOutputVariablesLine(t *testing.T) { varOutLine := "var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4" @@ -77,7 +55,7 @@ func TestParseOutputVariablesLine(t *testing.T) { t.Error("Couldn't parse var_out line:", err) t.Fail() } - + assert.Equal(t, m["ICCID"], 0) assert.Equal(t, m["IMSI"], 1) assert.Equal(t, m["PIN1"], 2) From 4378a8879937d8fdcdd76904a849582d56d6885e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 11:33:49 +0100 Subject: [PATCH 129/309] Conservative extension into wildly more flexible parser of outputfiles --- .../outfileparser/outfileparser.go | 46 ++++++++++--------- .../outfileparser/outfileparser_test.go | 3 +- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 62cc04711..0c27fa444 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -4,6 +4,7 @@ package outfileparser import ( "bufio" + "errors" "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" @@ -13,7 +14,6 @@ import ( "regexp" "strconv" "strings" - "errors" ) const ( @@ -59,29 +59,28 @@ type ParserState struct { inputVariables map[string]string headerDescription map[string]string entries []model.SimEntry + csvFieldMap map[string]int } -func ParseVarOutLine(varOutLine string) (map[string]int, error) { +func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { varOutSplit := strings.Split(varOutLine, ":") - if (len(varOutSplit) != 2) { + if len(varOutSplit) != 2 { fmt.Println("Length = ", len(varOutSplit)) - return nil, errors.New("syntax error in var_out line. More than two colon separated fields.") + return nerrors.New("syntax error in var_out line. More than two colon separated fields.") } - if (string(varOutSplit[0]) != "var_out") { - return nil, errors.New("syntax error in var_out line. Does not start with'var_out:'") + if string(varOutSplit[0]) != "var_out" { + return errors.New("syntax error in var_out line. Does not start with 'var_out:'") } slashedFields := strings.Split(varOutSplit[1], "/") - var result = map[string]int{} for index, columnName := range slashedFields { - result[columnName] = index + (*result)[columnName] = index } - return result, nil + return nil } - func ParseOutputFile(filename string) model.OutputFileRecord { _, err := os.Stat(filename) @@ -104,6 +103,7 @@ func ParseOutputFile(filename string) model.OutputFileRecord { currentState: INITIAL, inputVariables: make(map[string]string), headerDescription: make(map[string]string), + csvFieldMap: make(map[string]int), } scanner := bufio.NewScanner(file) @@ -135,22 +135,27 @@ func ParseOutputFile(filename string) model.OutputFileRecord { } ParseLineIntoKeyValueMap(line, state.inputVariables) case OUTPUT_VARIABLES: - if line == "var_Out: ICCID/IMSI/KI" { + + if (strings.HasPrefix(line, "var_Out: ")) { + if (len(state.csvFieldMap) != 0) { + log.Fatal("Parsing multiple 'var_out' lines can't be right") + } + if ParseVarOutLine(line, &(state.csvFieldMap)) != nil { + log.Fatalf("Couldn't parse output variable declaration '%s'\n", err) + } continue } - // We won't handle all variations, only the most common one - // if more fancy variations are necessary (with pin codes etc), then - // we'll add them as needed. - if strings.HasPrefix(line, "var_Out: ") { - log.Fatalf("Unknown output format, only know how to handle ICCID/IMSI/KI, but was '%s'\n", line) + if (len(state.csvFieldMap) == 0) { + log.Fatal("Cannot parse CSV part of input file without having first parsed a CSV header.") } line = strings.TrimSpace(line) if line == "" { continue } - rawIccid, imsi, ki := parseOutputLine(line) + + rawIccid, imsi, ki := parseOutputLine(state, line) iccidWithChecksum := rawIccid if strings.HasSuffix(rawIccid, "F") { @@ -183,7 +188,7 @@ func ParseOutputFile(filename string) model.OutputFileRecord { declaredNoOfEntities, err := strconv.Atoi(state.headerDescription["Quantity"]) if err != nil { - log.Fatal("Could not find 'Quantity' field while parsing file '",filename, "'") + log.Fatal("Could not find 'Quantity' field while parsing file '", filename, "'") } if countedNoOfEntries != declaredNoOfEntities { @@ -222,9 +227,9 @@ func getCustomer(state ParserState) string { return state.headerDescription["Customer"] } -func parseOutputLine(s string) (string, string, string) { +func parseOutputLine(state ParserState, s string) (string, string, string) { parsedString := strings.Split(s, " ") - return parsedString[0], parsedString[1], parsedString[2] + return parsedString[state.csvFieldMap["ICCID"]], parsedString[state.csvFieldMap["IMSI"]], parsedString[state.csvFieldMap["KI"]] } func transitionMode(state *ParserState, targetState string) { @@ -266,7 +271,6 @@ func fileExists(filename string) bool { return !info.IsDir() } - // TODO: Move this into some other package. "hssoutput" or something. // TODO: Consider rewriting using https://golang.org/pkg/encoding/csv/ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 6216e9963..48d90741a 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -50,7 +50,8 @@ func TestReadingComplexOutputFile(t *testing.T) { func TestParseOutputVariablesLine(t *testing.T) { varOutLine := "var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4" - m, err := ParseVarOutLine(varOutLine) + m := make(map[string]int) + err := ParseVarOutLine(varOutLine, &m) if err != nil { t.Error("Couldn't parse var_out line:", err) t.Fail() From fbdcd091ed56ba13867111ad188ea7a78963a194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 11:53:12 +0100 Subject: [PATCH 130/309] Pass test fo refactored csv file parsing --- .../outfileparser/outfileparser.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 0c27fa444..6913d58fd 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -66,12 +66,12 @@ func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { varOutSplit := strings.Split(varOutLine, ":") if len(varOutSplit) != 2 { - fmt.Println("Length = ", len(varOutSplit)) - return nerrors.New("syntax error in var_out line. More than two colon separated fields.") + return errors.New("syntax error in var_out line. More than two colon separated fields.") } - if string(varOutSplit[0]) != "var_out" { - return errors.New("syntax error in var_out line. Does not start with 'var_out:'") + varOutToken := strings.TrimSpace(string(varOutSplit[0])) + if strings.ToLower(varOutToken) != "var_out" { + return errors.New(fmt.Sprintf("syntax error in var_out line. Does not start with 'var_out', was '%s'", varOutToken)) } slashedFields := strings.Split(varOutSplit[1], "/") @@ -136,17 +136,22 @@ func ParseOutputFile(filename string) model.OutputFileRecord { ParseLineIntoKeyValueMap(line, state.inputVariables) case OUTPUT_VARIABLES: - if (strings.HasPrefix(line, "var_Out: ")) { + line = strings.TrimSpace(line) + lowercaseLine := strings.ToLower(line) + + if (strings.HasPrefix(lowercaseLine, "var_out:")) { if (len(state.csvFieldMap) != 0) { log.Fatal("Parsing multiple 'var_out' lines can't be right") } - if ParseVarOutLine(line, &(state.csvFieldMap)) != nil { + err := ParseVarOutLine(line, &(state.csvFieldMap)) + if err != nil { log.Fatalf("Couldn't parse output variable declaration '%s'\n", err) } continue } if (len(state.csvFieldMap) == 0) { + fmt.Println("Line = ", line) log.Fatal("Cannot parse CSV part of input file without having first parsed a CSV header.") } From 911fcf08544c987080ce8824d3293c59e88353d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 13:55:05 +0100 Subject: [PATCH 131/309] fix bug: Use the right package name --- sim-administration/sim-batch-management/sim-batch-mgt.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 80c8f0e1e..8ebda8b91 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -8,6 +8,7 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" kingpin "gopkg.in/alecthomas/kingpin.v2" "log" "os" @@ -154,7 +155,7 @@ func main() { cmd := kingpin.Parse() switch cmd { case "sim-profile-upload": - outfileconversion.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) + outfileparser.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "list-batches": From 9b1b16d1c3d613ed7a014814e8755893367c52d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 6 Nov 2019 14:34:30 +0100 Subject: [PATCH 132/309] Ready to generate input file from batch stored in db --- .../sim-batch-management/model/model.go | 6 ++ .../sim-batch-management/sim-batch-mgt.go | 55 ++++++++++++++++--- .../sim-batch-management/store/store.go | 2 + .../upload-sim-batch-lib-test.go | 30 ---------- .../uploadtoprime/upload-sim-batch-lib.go | 38 ------------- 5 files changed, 55 insertions(+), 76 deletions(-) delete mode 100644 sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 7ab907184..d52c858bf 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -68,6 +68,12 @@ type SimEntry struct { type Batch struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` + + // TODO: Customer is a misnomer: This is the customer name used when + // ordering a sim batch, used in the input file. So a very + // specific use, not in any way the generic thing the word + // as it is used now points to. + Customer string `db:"customer" json:"customer"` ProfileType string `db:"profileType" json:"profileType"` OrderDate string `db:"orderDate" json:"orderDate"` diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 8ebda8b91..bfea3ff4e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -6,9 +6,10 @@ import ( "encoding/json" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" kingpin "gopkg.in/alecthomas/kingpin.v2" "log" "os" @@ -105,15 +106,13 @@ var ( batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") listBatches = kingpin.Command("list-batches", "List all known batches.") - describeBatch = kingpin.Command("describe-batch", "List all known batches.") - describeBatchBatch = describeBatch.Arg("batch", "List all known batches.").String() + describeBatch = kingpin.Command("describe-batch", "Describe a batch with a particular name.") + describeBatchBatch = describeBatch.Arg("batch", "The batch to describe").String() + + generateInputFile = kingpin.Command("generate-input-file", "Generate input file for a named batch using stored parameters") + generateInputFileBatchname = generateInputFile.Arg("batchname", "The batch to generate the input file for.").String() - // describeBatch = kingpin.Command("describe-batch", "List details about named batch") - // describeBatchName = describeBatch.Arg("name").Required().String() - // - // Declare a new batch - // db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") dbName = db.Flag("name", "Unique name of this batch").Required().String() dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() @@ -187,6 +186,19 @@ func main() { fmt.Printf("%v\n", string(bytes)) } + case "generate-input-file": + batch, err := db.GetBatchByName(*generateInputFileBatchname) + if err != nil { + panic(err) + } + + if batch == nil { + fmt.Printf("No batch found with name '%s'\n", *generateInputFileBatchname) + } else { + var result = GenerateInputFile(batch) + fmt.Println(result) + } + case "declare-batch": fmt.Println("Declare batch") db.DeclareBatch( @@ -320,3 +332,30 @@ func checkEs2TargetState(target *string) { panic("Target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") } } + + +/// +/// Input batch management +/// + + +func GenerateInputFile(batch *model.Batch) string { + result := "*HEADER DESCRIPTION\n" + + "***************************************\n" + + fmt.Sprintf("Customer : %s\n", (*batch).Customer) + + fmt.Sprintf("ProfileType : %s\n", (*batch).ProfileType) + + fmt.Sprintf("Order Date : %s\n", (*batch).OrderDate) + + fmt.Sprintf("Batch No : %s\n", (*batch).BatchNo) + + fmt.Sprintf("Quantity : %d\n", (*batch).Quantity) + + "***************************************\n" + + "*INPUT VARIABLES\n" + + "***************************************\n" + + "var_In:\n" + + fmt.Sprintf(" ICCID: %s\n", (*batch).FirstIccid) + + fmt.Sprintf("IMSI: %s\n", (*batch).FirstImsi) + + "***************************************\n" + + "*OUTPUT VARIABLES\n" + + "***************************************\n" + + "var_Out: ICCID/IMSI/KI\n" + return result +} \ No newline at end of file diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 8fc8f25e0..2d4bba0a2 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -157,6 +157,7 @@ func (sdb SimBatchDB) DeclareBatch( profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { + // TODO: // 1. Check all the arguments (methods already written). // 2. Check that the name isn't already registred. @@ -246,5 +247,6 @@ func (sdb SimBatchDB) DeclareBatch( // Create the new batch (checking for consistency not done!) err = sdb.Create(&myBatch) + return &myBatch, err } diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go deleted file mode 100644 index 96bbc1967..000000000 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib-test.go +++ /dev/null @@ -1,30 +0,0 @@ -package uploadtoprime - -import ( - "fmt" - "gotest.tools/assert" - "testing" -) - -func TestParseInputFileGeneratorCommmandline(t *testing.T) { - - parsedBatch := ParseInputFileGeneratorCommmandline() - assert.Equal(t, "Footel", parsedBatch.Customer) - assert.Equal(t, "BAR_FOOTEL_STD", parsedBatch.ProfileType) - assert.Equal(t, "20191007", parsedBatch.OrderDate) - assert.Equal(t, "2019100701", parsedBatch.BatchNo) - assert.Equal(t, 10, parsedBatch.Quantity) - assert.Equal(t, 894700000000002214, parsedBatch.FirstIccid) - assert.Equal(t, 242017100012213, parsedBatch.FirstImsi) -} - -// TODO: Make a test that checks that the correct number of things are made, -// and also that the right things are made. Already had one fencepost error -// on this stuff. - -func TestGenerateInputFile(t *testing.T) { - parsedBatch := ParseInputFileGeneratorCommmandline() - var result = GenerateInputFile(parsedBatch) - - fmt.Println(result) -} diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index 608edd9e5..ebae65edc 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -208,42 +208,4 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, } } -/// -/// Input batch management -/// - -func ParseInputFileGeneratorCommmandline() model.InputBatch { - - // TODO: This function should be rewritten to parse a string array and send it to flags. - // we need to up our Go-Fu before we can make flag.Parse(arguments) work - - return model.InputBatch{ - Customer: "Footel", - ProfileType: "BAR_FOOTEL_STD", - OrderDate: "20191007", - BatchNo: "2019100701", - Quantity: 10, - FirstIccid: "894700000000002214", - FirstImsi: "242017100012213"} -} -func GenerateInputFile(batch model.InputBatch) string { - result := "*HEADER DESCRIPTION\n" + - "***************************************\n" + - fmt.Sprintf("Customer :%s\n", batch.Customer) + - fmt.Sprintf("ProfileType : %s\n", batch.ProfileType) + - fmt.Sprintf("Order Date : %s\n", batch.OrderDate) + - fmt.Sprintf("Batch No : %s\n", batch.BatchNo) + - fmt.Sprintf("Quantity : %d\n", batch.Quantity) + - "***************************************\n" + - "*INPUT VARIABLES\n" + - "***************************************\n" + - "var_In:\n" + - fmt.Sprintf(" ICCID: %s\n", batch.FirstIccid) + - fmt.Sprintf("IMSI: %s\n", batch.FirstImsi) + - "***************************************\n" + - "*OUTPUT VARIABLES\n" + - "***************************************\n" + - "var_Out: ICCID/IMSI/KI\n" - return result -} From f9392c68b9774779ac0ffd688b1f6cb96bbf009e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 11:10:38 +0100 Subject: [PATCH 133/309] Be able to generate output files from standard batch struct --- .../sim-batch-management/sim-batch-mgt.go | 16 +++++++++ .../uploadtoprime/upload-sim-batch-lib.go | 35 +++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index bfea3ff4e..c20b7852e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -113,6 +113,9 @@ var ( generateInputFile = kingpin.Command("generate-input-file", "Generate input file for a named batch using stored parameters") generateInputFileBatchname = generateInputFile.Arg("batchname", "The batch to generate the input file for.").String() + generateUploadBatch = kingpin.Command("generate-batch-upload-script", "Generate a batch upload script") + generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to generate upload script from").String() + db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") dbName = db.Flag("name", "Unique name of this batch").Required().String() dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() @@ -186,6 +189,19 @@ func main() { fmt.Printf("%v\n", string(bytes)) } + case "generate-batch-upload-script": + batch, err := db.GetBatchByName(*generateUploadBatchBatch) + if err != nil { + panic(err) + } + + if batch == nil { + fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) + } else { + var csvPayload = uploadtoprime.GenerateCsvPayload2(*batch) + uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) + } + case "generate-input-file": batch, err := db.GetBatchByName(*generateInputFileBatchname) if err != nil { diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index ebae65edc..7024099e7 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -51,6 +51,39 @@ func GenerateCsvPayload(batch model.OutputBatch) string { return sb.String() } +func GenerateCsvPayload2(batch model.Batch) string { + var sb strings.Builder + sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") + + iccidWithoutLuhnChecksum, err := strconv.Atoi(batch.FirstIccid) + if err != nil { + panic(err) + } + + imsi, err := strconv.Atoi(batch.FirstImsi) + if err != nil { + panic(err) + } + + var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) + if err2 != nil { + panic(err) + } + + for i := 0; i < batch.Quantity; i++ { + iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) + line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.ProfileType) + sb.WriteString(line) + + iccidWithoutLuhnChecksum += batch.IccidIncrement + imsi += batch.ImsiIncrement + msisdn += batch.MsisdnIncrement + } + + return sb.String() +} + +// TODO: Delete this function. func ParseUploadFileGeneratorCommmandline() model.OutputBatch { // @@ -207,5 +240,3 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, MsisdnIncrement: msisdnIncrement, } } - - From 2d0ddcbc6a7ef228782cda06cce410db0391f9f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 11:16:56 +0100 Subject: [PATCH 134/309] A little more consistent logging --- sim-administration/sim-batch-management/store/store.go | 1 + .../sim-batch-management/uploadtoprime/upload-sim-batch-lib.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 2d4bba0a2..f76a8127b 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -219,6 +219,7 @@ func (sdb SimBatchDB) DeclareBatch( log.Printf("msisdnLen = %10d\n", msisdnLen) log.Printf("iccidLen = %10d\n", iccidlen) log.Printf("imsiLen = %10d\n", imsiLen) + log.Printf("batchLength = %10d\n", batchLength) log.Fatal("FATAL: msisdnLen, iccidLen and imsiLen are not identical.") } diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go index 7024099e7..13478489d 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go +++ b/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go @@ -219,7 +219,7 @@ func OutputBatchFromCommandLineParameters(firstIccid *string, log.Printf("msisdnLen = %10d\n", msisdnLen) log.Printf("iccidLen = %10d\n", iccidlen) log.Printf("imsiLen = %10d\n", imsiLen) - log.Fatal("FATAL: msisdnLen, iccidLen and imsiLen are not identical.") + log.Fatal("FATAL (outfile generation): msisdnLen, iccidLen and imsiLen are not identical.") } tail := flag.Args() From 6faf1bf189127f32eaa8cf32005f619bab9e1df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:33:51 +0100 Subject: [PATCH 135/309] Adding a build-all scdript --- .../sim-batch-management/build-all.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 sim-administration/sim-batch-management/build-all.sh diff --git a/sim-administration/sim-batch-management/build-all.sh b/sim-administration/sim-batch-management/build-all.sh new file mode 100755 index 000000000..dc3b10bc4 --- /dev/null +++ b/sim-administration/sim-batch-management/build-all.sh @@ -0,0 +1,21 @@ +#!/bin/bash + + +go build + +if [ "$?" -ne "0" ]; then + echo "Sorry compilation failed aborting build." + exit 1 +fi + + + + +go test ./... + +if [ "$?" -ne "0" ]; then + echo "Sorry, one or more tests failed, aborting build." + exit 1 +fi + + From 356055d9834c52411193e5dff9d7247e56d8b191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:34:02 +0100 Subject: [PATCH 136/309] Renaming to simplr name --- .../uploadtoprime/{upload-sim-batch-lib.go => uploadtoprime.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sim-administration/sim-batch-management/uploadtoprime/{upload-sim-batch-lib.go => uploadtoprime.go} (100%) diff --git a/sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go similarity index 100% rename from sim-administration/sim-batch-management/uploadtoprime/upload-sim-batch-lib.go rename to sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go From 0e2e5fc984f7d5807a70d36c96f1c0ad405157af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:41:28 +0100 Subject: [PATCH 137/309] Remove OutputBatch, and deal with fallout --- .../sim-batch-management/model/model.go | 11 -- .../sim-batch-management/sim-batch-mgt.go | 45 ----- .../uploadtoprime/uploadtoprime.go | 183 ------------------ 3 files changed, 239 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index d52c858bf..ff7c8df01 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -23,17 +23,6 @@ type InputBatch struct { FirstImsi string `db:"firstImsi" json:"firstImsi"` } -type OutputBatch struct { - ProfileType string `db:"profileType" json:"profileType"` - Url string `db:"url" json:"url"` - Length int - FirstMsisdn int - MsisdnIncrement int - FirstIccid int - IccidIncrement int - FirstImsi int - ImsiIncrement int -} type OutputFileRecord struct { Filename string diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index c20b7852e..3f5ba95f4 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -73,34 +73,6 @@ var ( // TODO: Check if this can be used for the key files. // postImage = post.Flag("image", "image to post").ExistingFile() - // - // Upload of batch data to Prime - // - pbu = kingpin.Command("prime-batch-upload", "Generate a shellscript that will upload a batch of profiles to Prime.") - pbuFirstIccid = pbu.Flag("first-rawIccid", - "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - pbuLastIccid = pbu.Flag("last-rawIccid", - "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - pbuFirstIMSI = pbu.Flag("first-imsi", "First IMSI in batch").Required().String() - pbuLastIMSI = pbu.Flag("last-imsi", "Last IMSI in batch").Required().String() - pbuFirstMsisdn = pbu.Flag("first-msisdn", "First MSISDN in batch").Required().String() - pbuLastMsisdn = pbu.Flag("last-msisdn", "Last MSISDN in batch").Required().String() - pbuProfileType = pbu.Flag("profile-type", "SIM profile type").Required().String() - pbuBatchLengthString = pbu.Flag( - "batch-Quantity", - "Number of sim cards in batch").Required().String() - - // XXX Legal values are Loltel and M1 at this time, how to configure that - // flexibly? Eventually by putting them in a database and consulting it during - // command execution, but for now, just by squinting. - - pbuHssVendor = pbu.Flag("hss-vendor", "The HSS vendor").Default("M1").String() - pbuUploadHostname = pbu.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() - pbuUploadPortnumber = pbu.Flag("upload-portnumber", "port to upload to").Default("8080").String() - pbuProfileVendor = pbu.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() - pbuInitialHlrActivationStatusOfProfiles = pbu.Flag( - "initial-hlr-activation-status-of-profiles", - "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() // TODO ??? batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") @@ -235,23 +207,6 @@ func main() { *dbUploadPortnumber, *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) - case "prime-batch-upload": - var batch = uploadtoprime.OutputBatchFromCommandLineParameters( - pbuFirstIccid, - pbuLastIccid, - pbuLastIMSI, - pbuFirstIMSI, - pbuLastMsisdn, - pbuFirstMsisdn, - pbuUploadHostname, - pbuUploadPortnumber, - pbuHssVendor, - pbuInitialHlrActivationStatusOfProfiles, - pbuProfileType, - pbuProfileVendor, - pbuBatchLengthString) - var csvPayload = uploadtoprime.GenerateCsvPayload(batch) - uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) case "es2": diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index 13478489d..c26954561 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -11,12 +11,9 @@ package uploadtoprime import ( - "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" - "log" "strconv" "strings" ) @@ -29,28 +26,6 @@ func GeneratePostingCurlscript(url string, payload string) { fmt.Print("EOF\n") } -func GenerateCsvPayload(batch model.OutputBatch) string { - var sb strings.Builder - sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") - - var iccidWithoutLuhnChecksum = batch.FirstIccid - - var imsi = batch.FirstImsi - var msisdn = batch.FirstMsisdn - for i := 0; i < batch.Length; i++ { - - iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) - line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.ProfileType) - sb.WriteString(line) - - iccidWithoutLuhnChecksum += batch.IccidIncrement - imsi += batch.ImsiIncrement - msisdn += batch.MsisdnIncrement - } - - return sb.String() -} - func GenerateCsvPayload2(batch model.Batch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") @@ -82,161 +57,3 @@ func GenerateCsvPayload2(batch model.Batch) string { return sb.String() } - -// TODO: Delete this function. -func ParseUploadFileGeneratorCommmandline() model.OutputBatch { - - // - // Set up command line parsing - // - firstIccid := flag.String("first-rawIccid", - "not a valid rawIccid", - "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present") - lastIccid := flag.String("last-rawIccid", - "not a valid rawIccid", - "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present") - firstIMSI := flag.String("first-imsi", "Not a valid IMSI", "First IMSI in batch") - lastIMSI := flag.String("last-imsi", "Not a valid IMSI", "Last IMSI in batch") - firstMsisdn := flag.String("first-msisdn", "Not a valid MSISDN", "First MSISDN in batch") - lastMsisdn := flag.String("last-msisdn", "Not a valid MSISDN", "Last MSISDN in batch") - profileType := flag.String("profile-type", "Not a valid sim profile type", "SIM profile type") - batchLengthString := flag.String( - "batch-Quantity", - "Not a valid batch-Quantity, must be an integer", - "Number of sim cards in batch") - - // XXX Legal values are Loltel and M1 at this time, how to configure that - // flexibly? Eventually by puttig them in a database and consulting it during - // command execution, but for now, just by squinting. - - hssVendor := flag.String("hss-vendor", "M1", "The HSS vendor") - uploadHostname := - flag.String("upload-hostname", "localhost", "host to upload batch to") - uploadPortnumber := - flag.String("upload-portnumber", "8080", "port to upload to") - - profileVendor := - flag.String("profile-vendor", "Idemia", "Vendor of SIM profiles") - - initialHlrActivationStatusOfProfiles := - flag.String( - "initial-hlr-activation-status-of-profiles", - "ACTIVATED", - "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.") - - // - // Parse input according to spec above - // - flag.Parse() - - return OutputBatchFromCommandLineParameters( - firstIccid, - lastIccid, - lastIMSI, - firstIMSI, - lastMsisdn, - firstMsisdn, - uploadHostname, - uploadPortnumber, - hssVendor, - initialHlrActivationStatusOfProfiles, - profileType, - profileVendor, - batchLengthString) -} - -func OutputBatchFromCommandLineParameters(firstIccid *string, - lastIccid *string, - lastIMSI *string, - firstIMSI *string, - lastMsisdn *string, - firstMsisdn *string, - uploadHostname *string, - uploadPortnumber *string, - hssVendor *string, - initialHlrActivationStatusOfProfiles *string, - profileType *string, - profileVendor *string, - batchLengthString *string) model.OutputBatch { - - // - // Check parameters for syntactic correctness and - // semantic sanity. - // - - fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", *firstIccid) - fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", *lastIccid) - fieldsyntaxchecks.CheckIMSISyntax("last-imsi", *lastIMSI) - fieldsyntaxchecks.CheckIMSISyntax("first-imsi", *firstIMSI) - fieldsyntaxchecks.CheckMSISDNSyntax("last-msisdn", *lastMsisdn) - fieldsyntaxchecks.CheckMSISDNSyntax("first-msisdn", *firstMsisdn) - - batchLength, err := strconv.Atoi(*batchLengthString) - if err != nil { - log.Fatalf("Not a valid batch Quantity string '%s'.\n", *batchLengthString) - } - - if batchLength <= 0 { - log.Fatalf("OutputBatch Quantity must be positive, but was '%d'", batchLength) - } - - uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", - *uploadHostname, *uploadPortnumber, *hssVendor, *profileVendor, *initialHlrActivationStatusOfProfiles) - - fieldsyntaxchecks.CheckURLSyntax("uploadUrl", uploadUrl) - fieldsyntaxchecks.CheckProfileType("profile-type", *profileType) - - // Convert to integers, and get lengths - msisdnIncrement := -1 - if *firstMsisdn <= *lastMsisdn { - msisdnIncrement = 1 - } - - log.Println("firstmsisdn = ", *firstMsisdn) - log.Println("lastmsisdn = ", *lastMsisdn) - log.Println("MsisdnIncrement = ", msisdnIncrement) - - var firstMsisdnInt, _ = strconv.Atoi(*firstMsisdn) - var lastMsisdnInt, _ = strconv.Atoi(*lastMsisdn) - var msisdnLen = lastMsisdnInt - firstMsisdnInt + 1 - if msisdnLen < 0 { - msisdnLen = -msisdnLen - } - - var firstImsiInt, _ = strconv.Atoi(*firstIMSI) - var lastImsiInt, _ = strconv.Atoi(*lastIMSI) - var imsiLen = lastImsiInt - firstImsiInt + 1 - - var firstIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(*firstIccid)) - var lastIccidInt, _ = strconv.Atoi(fieldsyntaxchecks.IccidWithoutLuhnChecksum(*lastIccid)) - var iccidlen = lastIccidInt - firstIccidInt + 1 - - // Validate that lengths of sequences are equal in absolute - // values. - // TODO: Perhaps use some varargs trick of some sort here? - if loltelutils.Abs(msisdnLen) != loltelutils.Abs(iccidlen) || loltelutils.Abs(msisdnLen) != loltelutils.Abs(imsiLen) || batchLength != loltelutils.Abs(imsiLen) { - log.Printf("batchLength = %10d\n", batchLength) - log.Printf("msisdnLen = %10d\n", msisdnLen) - log.Printf("iccidLen = %10d\n", iccidlen) - log.Printf("imsiLen = %10d\n", imsiLen) - log.Fatal("FATAL (outfile generation): msisdnLen, iccidLen and imsiLen are not identical.") - } - - tail := flag.Args() - if len(tail) != 0 { - log.Printf("Unknown parameters: %s", flag.Args()) - } - - // Return a correctly parsed batch - return model.OutputBatch{ - ProfileType: *profileType, - Url: uploadUrl, - Length: loltelutils.Abs(iccidlen), - FirstIccid: firstIccidInt, - IccidIncrement: loltelutils.Sign(iccidlen), - FirstImsi: firstImsiInt, - ImsiIncrement: loltelutils.Sign(imsiLen), - FirstMsisdn: firstMsisdnInt, - MsisdnIncrement: msisdnIncrement, - } -} From 94e75f6cc9e61be171223b75aec428ffa084dccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:42:32 +0100 Subject: [PATCH 138/309] Fix formatting error in es2+ parser --- sim-administration/sim-batch-management/es2plus/es2plus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 5a104bc0a..f4a9fd446 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -284,7 +284,7 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma if printHeaders { - fmt.Println("Request -> %s\n", formatRequest(req)) + fmt.Printf("Request -> %s\n", formatRequest(req)) } resp, err := httpClient.Do(req) From a17a2a6dcac76a0d574566f30755d5370e597a91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:46:38 +0100 Subject: [PATCH 139/309] Move OutputFileRecord into the outfileparser where which is the only place it's used --- .../sim-batch-management/model/model.go | 16 +------------- .../outfileparser/outfileparser.go | 22 +++++++++++++++++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index ff7c8df01..e4704fa70 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -24,21 +24,7 @@ type InputBatch struct { } -type OutputFileRecord struct { - Filename string - InputVariables map[string]string - HeaderDescription map[string]string - Entries []SimEntry - // TODO: As it is today, the noOfEntries is just the number of Entries, - // but I may want to change that to be the declared number of Entries, - // and then later, dynamically, read in the individual Entries - // in a channel that is just piped to the goroutine that writes - // them to file, and fails if the number of declared Entries - // differs from the actual number of Entries. .... but that is - // for another day. - NoOfEntries int - OutputFileName string -} + type SimEntry struct { RawIccid string diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 6913d58fd..4634001fa 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -24,6 +24,24 @@ const ( UNKNOWN_HEADER = "unknown" ) + +type OutputFileRecord struct { + Filename string + InputVariables map[string]string + HeaderDescription map[string]string + Entries []model.SimEntry + // TODO: As it is today, the noOfEntries is just the number of Entries, + // but I may want to change that to be the declared number of Entries, + // and then later, dynamically, read in the individual Entries + // in a channel that is just piped to the goroutine that writes + // them to file, and fails if the number of declared Entries + // differs from the actual number of Entries. .... but that is + // for another day. + NoOfEntries int + OutputFileName string +} + + /// /// Functions /// @@ -81,7 +99,7 @@ func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { return nil } -func ParseOutputFile(filename string) model.OutputFileRecord { +func ParseOutputFile(filename string) OutputFileRecord { _, err := os.Stat(filename) @@ -202,7 +220,7 @@ func ParseOutputFile(filename string) model.OutputFileRecord { countedNoOfEntries) } - result := model.OutputFileRecord{ + result := OutputFileRecord{ Filename: filename, InputVariables: state.inputVariables, HeaderDescription: state.headerDescription, From 3742d8d3adc6f000688e74989ba9712c1a8f60a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:48:12 +0100 Subject: [PATCH 140/309] Remove struct that has now been subsumed by the Batch struct --- .../sim-batch-management/model/model.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index e4704fa70..4dc9ff4b9 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -8,21 +8,6 @@ package model // in a common persisted record that is then used for the bulk of the // processing. -// -// Batches as read from the input files -// -type InputBatch struct { - Id int64 `db:"id" json:"id"` - Name string `db:"name" json:"name"` - Customer string `db:"customer" json:"customer"` - ProfileType string `db:"profileType" json:"profileType"` - OrderDate string `db:"orderDate" json:"orderDate"` - BatchNo string `db:"batchNo" json:"batchNo"` - Quantity int `db:"quantity" json:"quantity"` - FirstIccid string `db:"firstIccid" json:"firstIccid"` - FirstImsi string `db:"firstImsi" json:"firstImsi"` -} - From 8f70ad897b356b934192ff06924f9fa203f3a4b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 13:52:31 +0100 Subject: [PATCH 141/309] Add dummy annotations, to be amended later --- .../sim-batch-management/model/model.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 4dc9ff4b9..8ec602f84 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -10,14 +10,15 @@ package model +// TODO: This struct isn't fully baked. type SimEntry struct { - RawIccid string - IccidWithChecksum string - IccidWithoutChecksum string - Imsi string - Ki string // XXX Toxic. Should never be stored persistently!! - OutputFileName string + RawIccid string `db:"rawIccid" json:"rawIccid"` + IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` + IccidWithoutChecksum string `db:"iccidWithoutChecksum" json:"iccidWithoutChecksum"` + Imsi string `db:"imsi" json:"imsi"` + Ki string `db:"ki" json:"ki"` + OutputFileName string `db:"outputFileName" json:"outputFileName"` } // @@ -25,6 +26,11 @@ type SimEntry struct { // nothing below this line should be left. // + +// TODO: Add a filename base which is e.g. Footel201910110102, functional +// dependencies on other fields, but we'll not worry about that right +// now. + type Batch struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` From cfba784a1293d6071a247c0c4ae52615cb01ba87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 14:55:53 +0100 Subject: [PATCH 142/309] Adding a filename base --- .../sim-batch-management/model/model.go | 1 + .../sim-batch-management/store/store.go | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 8ec602f84..725f5379e 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -40,6 +40,7 @@ type Batch struct { // specific use, not in any way the generic thing the word // as it is used now points to. + FilenameBase string `db:"filenameBase" json:"filenameBase"` Customer string `db:"customer" json:"customer"` ProfileType string `db:"profileType" json:"profileType"` OrderDate string `db:"orderDate" json:"orderDate"` diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index f76a8127b..f0acecf88 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -83,8 +83,9 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) Create(theBatch *model.Batch) error { - res := sdb.Db.MustExec("INSERT INTO BATCH (name, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", + res := sdb.Db.MustExec("INSERT INTO BATCH (name, filenameBase, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, + (*theBatch).FilenameBase, (*theBatch).Customer, (*theBatch).OrderDate, (*theBatch).Customer, @@ -112,6 +113,7 @@ func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, + filenameBase VARCHAR NOT NULL, customer VARCHAR NOT NULL, profileType VARCHAR NOT NULL, orderDate VARCHAR NOT NULL, @@ -228,11 +230,13 @@ func (sdb SimBatchDB) DeclareBatch( log.Printf("Unknown parameters: %s", flag.Args()) } - // Return a correctly parsed batch - // TODO: Batch name missing! + filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) + fmt.Printf("Filename base = '%s'\n", filenameBase) + myBatch := model.Batch{ OrderDate: orderDate, Customer: customer, + FilenameBase: filenameBase, Name: name, BatchNo: batchNo, ProfileType: profileType, @@ -246,7 +250,7 @@ func (sdb SimBatchDB) DeclareBatch( MsisdnIncrement: msisdnIncrement, } - // Create the new batch (checking for consistency not done!) + // Persist the newly created batch, and return it err = sdb.Create(&myBatch) return &myBatch, err From 4506eebf43cc6706250f7c61c3751182a416ae2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 15:05:26 +0100 Subject: [PATCH 143/309] Adding more to the data model being generated --- .../sim-batch-management/model/model.go | 10 ++++++---- .../sim-batch-management/store/store.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 725f5379e..d1a49ac4f 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -13,12 +13,14 @@ package model // TODO: This struct isn't fully baked. type SimEntry struct { - RawIccid string `db:"rawIccid" json:"rawIccid"` + BatchID string `db:"batchId" json:"batchId"` + RawIccid string `db:"rawIccid" json:"rawIccid"` IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` IccidWithoutChecksum string `db:"iccidWithoutChecksum" json:"iccidWithoutChecksum"` - Imsi string `db:"imsi" json:"imsi"` - Ki string `db:"ki" json:"ki"` - OutputFileName string `db:"outputFileName" json:"outputFileName"` + Iccid string `db:"iccid" json:"iccid"` + Imsi string `db:"imsi" json:"imsi"` + Msisdn string `db:"msisdn" json:"msisdn"` + Ki string `db:"ki" json:"ki"` } // diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index f0acecf88..c17519f9d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -129,8 +129,23 @@ func (sdb *SimBatchDB) GenerateTables() error { )` _, err := sdb.Db.Exec(foo) return err + + foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + batch INTEGER NOT NULL, + imsi VARCHAR NOT NULL, + iccidWithChecksum VARCHAR NOT NULL, + iccidWithoutChecksum VARCHAR NOT NULL, + iccid VARCHAR NOT NULL, + ki VARCHAR NOT NULL, + msisdn VARCHAR NOT NULL, + )` + _, err = sdb.Db.Exec(foo) + return err } + + func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` _, err := sdb.Db.Exec(foo) From d67e2e11dd2e9708dabb4e0c774af042f07e4486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 15:16:36 +0100 Subject: [PATCH 144/309] Adding a writer of sim entries --- .../sim-batch-management/model/model.go | 2 ++ .../sim-batch-management/store/store.go | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index d1a49ac4f..f60d44894 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -13,6 +13,7 @@ package model // TODO: This struct isn't fully baked. type SimEntry struct { + Id int64 `db:"id" json:"id"` BatchID string `db:"batchId" json:"batchId"` RawIccid string `db:"rawIccid" json:"rawIccid"` IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` @@ -23,6 +24,7 @@ type SimEntry struct { Ki string `db:"ki" json:"ki"` } + // // Below this line we grow the final persistence model. Eventually // nothing below this line should be left. diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index c17519f9d..e7fa402de 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -38,6 +38,9 @@ type Store interface { uploadPortnumber string, profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) + + + CreateSimEntry(simEntry *model.SimEntry) error } type SimBatchDB struct { @@ -109,6 +112,28 @@ func (sdb SimBatchDB) Create(theBatch *model.Batch) error { return err } +func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { + + res := sdb.Db.MustExec("INSERT INTO SIM_ENTRY (batchId, rawIccid, iccidWithChrecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?)", + (*theEntry).BatchID, + (*theEntry).RawIccid, + (*theEntry).IccidWithChecksum, + (*theEntry).IccidWithoutChecksum, + (*theEntry).Iccid, + (*theEntry).Imsi, + (*theEntry).Msisdn, + (*theEntry).Ki, + ) + + id, err := res.LastInsertId() + if err != nil { + fmt.Errorf("Getting last inserted id failed '%s'", err) + } + theEntry.Id = id + return err +} + + func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, From 73c5a2b7808e87ab82d02e5bd8958fb82b7eb478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 15:23:19 +0100 Subject: [PATCH 145/309] Made reader function, so far no tests. --- .../sim-batch-management/store/store.go | 57 +++++++++++-------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index e7fa402de..ad2f1c2ca 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -112,27 +112,6 @@ func (sdb SimBatchDB) Create(theBatch *model.Batch) error { return err } -func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { - - res := sdb.Db.MustExec("INSERT INTO SIM_ENTRY (batchId, rawIccid, iccidWithChrecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?)", - (*theEntry).BatchID, - (*theEntry).RawIccid, - (*theEntry).IccidWithChecksum, - (*theEntry).IccidWithoutChecksum, - (*theEntry).Iccid, - (*theEntry).Imsi, - (*theEntry).Msisdn, - (*theEntry).Ki, - ) - - id, err := res.LastInsertId() - if err != nil { - fmt.Errorf("Getting last inserted id failed '%s'", err) - } - theEntry.Id = id - return err -} - func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( @@ -156,8 +135,8 @@ func (sdb *SimBatchDB) GenerateTables() error { return err foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - batch INTEGER NOT NULL, + simId INTEGER PRIMARY KEY AUTOINCREMENT, + batchId INTEGER NOT NULL, imsi VARCHAR NOT NULL, iccidWithChecksum VARCHAR NOT NULL, iccidWithoutChecksum VARCHAR NOT NULL, @@ -170,6 +149,38 @@ func (sdb *SimBatchDB) GenerateTables() error { } +func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { + + res := sdb.Db.MustExec("INSERT INTO SIM_ENTRY (batchId, rawIccid, iccidWithChrecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?)", + (*theEntry).BatchID, + (*theEntry).RawIccid, + (*theEntry).IccidWithChecksum, + (*theEntry).IccidWithoutChecksum, + (*theEntry).Iccid, + (*theEntry).Imsi, + (*theEntry).Msisdn, + (*theEntry).Ki, + ) + + id, err := res.LastInsertId() + if err != nil { + fmt.Errorf("Getting last inserted id failed '%s'", err) + } + theEntry.Id = id + return err +} + +func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { + var result model.SimEntry + return &result, sdb.Db.Get(&result, "select * from SIM_ENTRY where simId = ?", simId) +} + + +func (sdb SimBatchDB) GetAllSimEntriesForBarch(batchId int64) ([]model.SimEntry, error) { + result := []model.SimEntry{} + return result, sdb.Db.Select(&result, "SELECT * from SIM_ENTRY WHERE batchId = ?", batchId ) +} + func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` From a6df7c4377a2a96f34e47f931ae091a2c35c172a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 15:33:22 +0100 Subject: [PATCH 146/309] Refactor database test to have separate test batch declaration method --- .../sim-batch-management/store/store_test.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 766462f34..2fb365314 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -96,7 +96,9 @@ func TestGetAllBatches(t *testing.T) { } } -func TestDeclareBatch(t *testing.T) { + +func declareTestBatch(t *testing.T) *model.Batch { + theBatch, err := sdb.DeclareBatch( "Name", "Customer", @@ -115,10 +117,16 @@ func TestDeclareBatch(t *testing.T) { "8088", // uploadPortnumber string, "snuff", // profileVendor string, "ACTIVE") // initialHlrActivationStatusOfProfiles string + + if err != nil { - panic(err) + t.Fatal(err) } + return theBatch +} +func TestDeclareBatch(t *testing.T) { + theBatch := declareTestBatch(t) retrievedValue, _ := sdb.GetBatchById(theBatch.Id) if !reflect.DeepEqual(*retrievedValue, *theBatch) { fmt.Errorf("getBatchById failed") From 141df8d42f5d02d898003b65747640258e7a5c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 16:07:15 +0100 Subject: [PATCH 147/309] Roundtrip storing and sim entries works in test --- .../sim-batch-management/model/model.go | 4 +- .../sim-batch-management/store/store.go | 12 ++-- .../sim-batch-management/store/store_test.go | 55 ++++++++++++++++--- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index f60d44894..c5c491db2 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -13,8 +13,8 @@ package model // TODO: This struct isn't fully baked. type SimEntry struct { - Id int64 `db:"id" json:"id"` - BatchID string `db:"batchId" json:"batchId"` + SimId int64 `db:"simId" json:"simId"` + BatchID int64 `db:"batchId" json:"batchId"` RawIccid string `db:"rawIccid" json:"rawIccid"` IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` IccidWithoutChecksum string `db:"iccidWithoutChecksum" json:"iccidWithoutChecksum"` diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index ad2f1c2ca..887102867 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -132,17 +132,17 @@ func (sdb *SimBatchDB) GenerateTables() error { url VARCHAR )` _, err := sdb.Db.Exec(foo) - return err foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( simId INTEGER PRIMARY KEY AUTOINCREMENT, batchId INTEGER NOT NULL, imsi VARCHAR NOT NULL, + rawIccid VARCHAR NOT NULL, iccidWithChecksum VARCHAR NOT NULL, iccidWithoutChecksum VARCHAR NOT NULL, iccid VARCHAR NOT NULL, ki VARCHAR NOT NULL, - msisdn VARCHAR NOT NULL, + msisdn VARCHAR NOT NULL )` _, err = sdb.Db.Exec(foo) return err @@ -151,7 +151,7 @@ func (sdb *SimBatchDB) GenerateTables() error { func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { - res := sdb.Db.MustExec("INSERT INTO SIM_ENTRY (batchId, rawIccid, iccidWithChrecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?)", + res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?)", (*theEntry).BatchID, (*theEntry).RawIccid, (*theEntry).IccidWithChecksum, @@ -166,19 +166,19 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { if err != nil { fmt.Errorf("Getting last inserted id failed '%s'", err) } - theEntry.Id = id + theEntry.SimId = id return err } func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { var result model.SimEntry - return &result, sdb.Db.Get(&result, "select * from SIM_ENTRY where simId = ?", simId) + return &result, sdb.Db.Get(&result, "select * from SIM_PROFILE where simId = ?", simId) } func (sdb SimBatchDB) GetAllSimEntriesForBarch(batchId int64) ([]model.SimEntry, error) { result := []model.SimEntry{} - return result, sdb.Db.Select(&result, "SELECT * from SIM_ENTRY WHERE batchId = ?", batchId ) + return result, sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId ) } diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 2fb365314..dac1ab4db 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -4,14 +4,13 @@ import ( "fmt" _ "github.com/mattn/go-sqlite3" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "gotest.tools/assert" "os" "reflect" "testing" ) -var sdb *store.SimBatchDB +var sdb *SimBatchDB var err error func TestMain(m *testing.M) { @@ -23,7 +22,7 @@ func TestMain(m *testing.M) { func setup() { // sdb, err = store.OpenFileSqliteDatabase("bazunka.db") - sdb, err = store.NewInMemoryDatabase() + sdb, err = NewInMemoryDatabase() if err != nil { fmt.Errorf("Couldn't open new in memory database '%s", err) } @@ -96,7 +95,6 @@ func TestGetAllBatches(t *testing.T) { } } - func declareTestBatch(t *testing.T) *model.Batch { theBatch, err := sdb.DeclareBatch( @@ -116,8 +114,7 @@ func declareTestBatch(t *testing.T) *model.Batch { "localhost", // uploadHostname string, "8088", // uploadPortnumber string, "snuff", // profileVendor string, - "ACTIVE") // initialHlrActivationStatusOfProfiles string - + "ACTIVE") // initialHlrActivationStatusOfProfiles string if err != nil { t.Fatal(err) @@ -129,6 +126,50 @@ func TestDeclareBatch(t *testing.T) { theBatch := declareTestBatch(t) retrievedValue, _ := sdb.GetBatchById(theBatch.Id) if !reflect.DeepEqual(*retrievedValue, *theBatch) { - fmt.Errorf("getBatchById failed") + t.Fatal("getBatchById failed") + } +} + +func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { + return ((a.BatchID == b.BatchID) && (a.RawIccid == b.RawIccid) && a.IccidWithChecksum == b.IccidWithChecksum && a.IccidWithoutChecksum == b.IccidWithoutChecksum && a.Iccid == b.Iccid && a.Imsi == b.Imsi && a.Msisdn == b.Msisdn && a.Ki == b.Ki) +} + +func TestDeclareAndRetrieveSimEntries(t *testing.T) { + theBatch := declareTestBatch(t) + batchId := theBatch.Id + + entry := model.SimEntry{ + BatchID: batchId, + RawIccid: "1", + IccidWithChecksum: "2", + IccidWithoutChecksum: "3", + Iccid: "4", + Imsi: "5", + Msisdn: "6", + Ki: "7", + } + + // assert.Equal(t, 0, entry.SimId) + sdb.CreateSimEntry(&entry) + assert.Assert(t, entry.SimId != 0) + + retrivedEntry, err := sdb.GetSimEntryById(entry.SimId) + if err != nil { + t.Fatal(err) + } + if !entryEqual(retrivedEntry, &entry) { + t.Fatal("Retrieved and stored sim entry are different") + } + + retrievedEntries, err := sdb.GetAllSimEntriesForBarch(entry.BatchID) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 1, len(retrievedEntries)) + + retrivedEntry = &(retrievedEntries[0]) + + if !entryEqual(retrivedEntry, &entry) { + t.Fatal("Retrieved and stored sim entry are different") } } From 9181d72d792eeaa0409a28462f821575ea89c86d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 16:13:42 +0100 Subject: [PATCH 148/309] Refactor Create funciton to the more specific CreateBatch --- sim-administration/sim-batch-management/store/store.go | 6 +++--- sim-administration/sim-batch-management/store/store_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 887102867..9e225ca27 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -84,7 +84,7 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { return &result, sdb.Db.Get(&result, "select * from BATCH where name = ?", name) } -func (sdb SimBatchDB) Create(theBatch *model.Batch) error { +func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { res := sdb.Db.MustExec("INSERT INTO BATCH (name, filenameBase, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, @@ -189,7 +189,7 @@ func (sdb *SimBatchDB) DropTables() error { } /** - * Create a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. + * CreateBatch a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. */ func (sdb SimBatchDB) DeclareBatch( name string, @@ -302,7 +302,7 @@ func (sdb SimBatchDB) DeclareBatch( } // Persist the newly created batch, and return it - err = sdb.Create(&myBatch) + err = sdb.CreateBatch(&myBatch) return &myBatch, err } diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index dac1ab4db..574ce6d25 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -61,7 +61,7 @@ func injectTestBatch() *model.Batch { MsisdnIncrement: -1, } - err := sdb.Create(&theBatch) + err := sdb.CreateBatch(&theBatch) if err != nil { panic(err) } From bce0639eac5711c09462890f391fb1d58392a682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 16:33:28 +0100 Subject: [PATCH 149/309] Now generates sim profiles with MSISDNs. The MSISDN generation needs to be amended, and also activation codes needs to be added, but we're getting somewhere now --- .../sim-batch-management/store/store.go | 57 +++++++++++++++++-- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 9e225ca27..37ae1971f 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -8,7 +8,6 @@ import ( "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" - "log" "os" "strconv" @@ -284,7 +283,7 @@ func (sdb SimBatchDB) DeclareBatch( filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) fmt.Printf("Filename base = '%s'\n", filenameBase) - myBatch := model.Batch{ + batch := model.Batch{ OrderDate: orderDate, Customer: customer, FilenameBase: filenameBase, @@ -301,8 +300,56 @@ func (sdb SimBatchDB) DeclareBatch( MsisdnIncrement: msisdnIncrement, } - // Persist the newly created batch, and return it - err = sdb.CreateBatch(&myBatch) + // Persist the newly created batch, + err = sdb.CreateBatch(&batch) + + + // Now create all the sim profiles + + + iccidWithoutLuhnChecksum, err := strconv.Atoi(batch.FirstIccid) + if err != nil { + panic(err) + } + + imsi, err := strconv.Atoi(batch.FirstImsi) + if err != nil { + panic(err) + } + + // XXX !!! TODO THis is wrong, but I'm doing it now, just to get started! + var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) + if err2 != nil { + panic(err) + } + + for i := 0; i < batch.Quantity; i++ { + iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) + + fmt.Println(" iccid with checksum ", iccidWithLuhnChecksum) + + simEntry := &model.SimEntry{ + BatchID: batch.Id, + RawIccid: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), + IccidWithChecksum: iccidWithLuhnChecksum, + IccidWithoutChecksum: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), + Iccid: iccidWithLuhnChecksum, + Imsi: fmt.Sprintf("%d", imsi), + Msisdn: fmt.Sprintf("%d", msisdn), + Ki: "", // Should be null + } + + err = sdb.CreateSimEntry(simEntry) + if (err != nil) { + panic(err) + } + + iccidWithoutLuhnChecksum += batch.IccidIncrement + imsi += batch.ImsiIncrement + msisdn += batch.MsisdnIncrement + } + - return &myBatch, err + // Return the newly created batch + return &batch, err } From e4c307175c9882ad73fdc91d9b42bbe3014b9281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 16:35:03 +0100 Subject: [PATCH 150/309] remove test printout, add (so far inactive) activation code field --- sim-administration/sim-batch-management/model/model.go | 1 + sim-administration/sim-batch-management/store/store.go | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index c5c491db2..74e5c4ff4 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -22,6 +22,7 @@ type SimEntry struct { Imsi string `db:"imsi" json:"imsi"` Msisdn string `db:"msisdn" json:"msisdn"` Ki string `db:"ki" json:"ki"` + ActivationCode string `db:"activationCode" json:"activationCode"` } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 37ae1971f..5d389e671 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -324,9 +324,7 @@ func (sdb SimBatchDB) DeclareBatch( } for i := 0; i < batch.Quantity; i++ { - iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) - - fmt.Println(" iccid with checksum ", iccidWithLuhnChecksum) + iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum))s simEntry := &model.SimEntry{ BatchID: batch.Id, From af0c745405aff73b8e3846eeb34bc7e7d377adc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 7 Nov 2019 21:25:32 +0100 Subject: [PATCH 151/309] Fix bugs so we're now able to generate the 10K profiles in the internal db --- sim-administration/sim-batch-management/store/store.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 5d389e671..316b4b772 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -306,8 +306,7 @@ func (sdb SimBatchDB) DeclareBatch( // Now create all the sim profiles - - iccidWithoutLuhnChecksum, err := strconv.Atoi(batch.FirstIccid) + iccidWithoutLuhnChecksum := firstIccidInt if err != nil { panic(err) } @@ -324,7 +323,7 @@ func (sdb SimBatchDB) DeclareBatch( } for i := 0; i < batch.Quantity; i++ { - iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum))s + iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) simEntry := &model.SimEntry{ BatchID: batch.Id, From 2101e4c0092e7971a404720a739e57be8c7c32e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 09:44:12 +0100 Subject: [PATCH 152/309] Quick establishment of batch content using transaction --- go.mod | 2 +- .../sim-batch-management/TODO.md | 4 +++ .../sim-batch-management/store/store.go | 27 ++++++++++++++----- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index e14f2307f..b233c635e 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.13 require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/go-ozzo/ozzo-dbx v1.0.15 // indirect + github.com/go-ozzo/ozzo-dbx v1.0.15 github.com/google/go-cmp v0.3.1 // indirect github.com/google/uuid v1.1.1 github.com/jmoiron/sqlx v1.2.0 diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index 1a862228d..d2ddf2bbd 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,6 +1,10 @@ An informal TODO list for the sim batch management tool == +1. Ingest 10K iccid/imsi/msisdn .csv file into existing batch. + +1. Get activation codes for all ICCIdes in an existing batch. + 1. Ingest input files into a corresponding batch, read all the sim profiles into a sim profile table. This table can then be enriched with access codes, MSISDNs etc, and then written to both HSSes and diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 316b4b772..b34d5fb3b 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -1,6 +1,7 @@ package store import ( + "database/sql" "flag" "fmt" "github.com/jmoiron/sqlx" @@ -38,8 +39,17 @@ type Store interface { profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) - CreateSimEntry(simEntry *model.SimEntry) error + + Begin() +} + +func (sdb *SimBatchDB) Begin() *sql.Tx { + tx, err := sdb.Db.Begin() + if err != nil { + panic(err) + } + return tx } type SimBatchDB struct { @@ -300,22 +310,25 @@ func (sdb SimBatchDB) DeclareBatch( MsisdnIncrement: msisdnIncrement, } + tx := sdb.Begin() + // Persist the newly created batch, err = sdb.CreateBatch(&batch) - - - // Now create all the sim profiles - - iccidWithoutLuhnChecksum := firstIccidInt if err != nil { + tx.Rollback() panic(err) } imsi, err := strconv.Atoi(batch.FirstImsi) if err != nil { + tx.Rollback() panic(err) } + // Now create all the sim profiles + + iccidWithoutLuhnChecksum := firstIccidInt + // XXX !!! TODO THis is wrong, but I'm doing it now, just to get started! var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) if err2 != nil { @@ -338,6 +351,7 @@ func (sdb SimBatchDB) DeclareBatch( err = sdb.CreateSimEntry(simEntry) if (err != nil) { + tx.Rollback() panic(err) } @@ -346,6 +360,7 @@ func (sdb SimBatchDB) DeclareBatch( msisdn += batch.MsisdnIncrement } + tx.Commit() // Return the newly created batch return &batch, err From 2bd31d51006898565915f368449e510e8d3c16af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 09:54:05 +0100 Subject: [PATCH 153/309] Concentrate commit/rollback into one defered function. --- .../sim-batch-management/store/store.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index b34d5fb3b..4e0099f78 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -312,16 +312,26 @@ func (sdb SimBatchDB) DeclareBatch( tx := sdb.Begin() + // This variable should be se to "true" if all transactions + // were successful, otherwise it will be rolled back. + weCool := false + + defer func() { + if (weCool) { + tx.Commit() + } else { + tx.Rollback() + } + }() + // Persist the newly created batch, err = sdb.CreateBatch(&batch) if err != nil { - tx.Rollback() panic(err) } imsi, err := strconv.Atoi(batch.FirstImsi) if err != nil { - tx.Rollback() panic(err) } @@ -351,7 +361,6 @@ func (sdb SimBatchDB) DeclareBatch( err = sdb.CreateSimEntry(simEntry) if (err != nil) { - tx.Rollback() panic(err) } @@ -360,7 +369,8 @@ func (sdb SimBatchDB) DeclareBatch( msisdn += batch.MsisdnIncrement } - tx.Commit() + // Signal to deferred function that we're ready to commit. + weCool = true // Return the newly created batch return &batch, err From f684b57435272a67d9be2363c18b70706789c056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 13:53:41 +0100 Subject: [PATCH 154/309] Phone number - updating script done (it seems) --- .../sim-batch-management/model/model.go | 4 +- .../sim-batch-management/sim-batch-mgt.go | 131 ++++++++++++++++-- .../sim-batch-management/store/store.go | 24 +++- .../sim-batch-management/store/store_test.go | 6 +- 4 files changed, 147 insertions(+), 18 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 74e5c4ff4..ad34455e1 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -37,8 +37,8 @@ type SimEntry struct { // now. type Batch struct { - Id int64 `db:"id" json:"id"` - Name string `db:"name" json:"name"` + BatchId int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` // TODO: Customer is a misnomer: This is the customer name used when // ordering a sim batch, used in the input file. So a very diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 3f5ba95f4..ac5d603e7 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -3,16 +3,19 @@ package main import ( "bufio" + "encoding/csv" "encoding/json" "fmt" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" kingpin "gopkg.in/alecthomas/kingpin.v2" + "io" "log" "os" + "strings" "sync" ) @@ -73,19 +76,22 @@ var ( // TODO: Check if this can be used for the key files. // postImage = post.Flag("image", "image to post").ExistingFile() - // TODO ??? batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") - listBatches = kingpin.Command("list-batches", "List all known batches.") + listBatches = kingpin.Command("list-batches", "List all known batches.") - describeBatch = kingpin.Command("describe-batch", "Describe a batch with a particular name.") + describeBatch = kingpin.Command("describe-batch", "Describe a batch with a particular name.") describeBatchBatch = describeBatch.Arg("batch", "The batch to describe").String() - generateInputFile = kingpin.Command("generate-input-file", "Generate input file for a named batch using stored parameters") + generateInputFile = kingpin.Command("generate-input-file", "Generate input file for a named batch using stored parameters") generateInputFileBatchname = generateInputFile.Arg("batchname", "The batch to generate the input file for.").String() - generateUploadBatch = kingpin.Command("generate-batch-upload-script", "Generate a batch upload script") + addMsisdnFromFile = kingpin.Command("add-msisdn-from-file", "Add MSISDN from CSV file containing at least ICCID/MSISDN, but also possibly IMSI.") + addMsisdnFromFileBatch = addMsisdnFromFile.Flag("batch", "The batch to augment").Required().String() + addMsisdnFromFileCsvfile = addMsisdnFromFile.Flag("csv-file", "The CSV file to read from").Required().ExistingFile() + + generateUploadBatch = kingpin.Command("generate-batch-upload-script", "Generate a batch upload script") generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to generate upload script from").String() db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") @@ -187,6 +193,113 @@ func main() { fmt.Println(result) } + case "add-msisdn-from-file": + batchName := *addMsisdnFromFileBatch + csvFilename := *addMsisdnFromFileCsvfile + + batch, err := db.GetBatchByName(batchName) + if err != nil { + panic(err) + } + + csvFile, _ := os.Open(csvFilename) + reader := csv.NewReader(bufio.NewReader(csvFile)) + + headerLine, error := reader.Read() + if error == io.EOF { + break + } else if error != nil { + log.Fatal(error) + } + + var columnMap map[string]int + columnMap = make(map[string]int) + + for index, fieldname := range headerLine { + columnMap[strings.ToLower(fieldname)] = index + } + + if _, hasIccid := columnMap["iccid"] ; !hasIccid { + panic("No ICCID column in CSV file") + } + + if _, hasMsisdn := columnMap["msisdn"]; !hasMsisdn { + panic("No MSISDN column in CSV file") + } + + if _, hasImsi := columnMap["imsi"]; !hasImsi { + panic("No IMSI column in CSV file") + } + + type csvRecord struct { + iccid string + imsi string + msisdn string + } + + var recordMap map[string]csvRecord + recordMap = make(map[string]csvRecord) + + // Read all the lines into the record map. + for { + line, error := reader.Read() + if error == io.EOF { + break + } else if error != nil { + log.Fatal(error) + } + + record := csvRecord{ + iccid: line[columnMap["iccid"]], + imsi: line[columnMap["imsi"]], + msisdn: line[columnMap["msisdn"]], + } + + if _, duplicateRecordExists := recordMap[record.iccid]; duplicateRecordExists { + panic(fmt.Sprintf("Duplicate ICCID record in map: %s", record.iccid)) + } + + recordMap[record.iccid] = record + } + + simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + panic(err) + } + + // Check for compatibility + tx := db.Begin() + noOfRecordsUpdated := 0 + for _, entry := range simEntries { + record, iccidRecordIsPresent := recordMap[entry.Iccid] + if !iccidRecordIsPresent { + tx.Rollback() + panic(fmt.Sprintf("ICCID not in batch: %s", entry.Iccid)) + } + + if entry.Imsi != record.imsi { + tx.Rollback() + panic(fmt.Sprintf("IMSI mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Imsi, record.iccid)) + } + + if entry.Msisdn != "" && record.msisdn != "" && record.msisdn != entry.Msisdn { + tx.Rollback() + panic(fmt.Sprintf("MSISDN mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Msisdn, record.msisdn)) + } + + if (entry.Msisdn == "" && record.msisdn != "") { + err = db.UpdateSimEntryMsisdn(entry.SimId, record.msisdn) + if err != nil { + tx.Rollback() + panic(err) + } + noOfRecordsUpdated += 1 + } + } + tx.Commit() + + fmt.Printf("Updated %d of a total of %d records in batch '%s'\n", noOfRecordsUpdated, len(simEntries), batchName) + case "declare-batch": fmt.Println("Declare batch") db.DeclareBatch( @@ -304,16 +417,14 @@ func checkEs2TargetState(target *string) { } } - /// /// Input batch management /// - func GenerateInputFile(batch *model.Batch) string { result := "*HEADER DESCRIPTION\n" + "***************************************\n" + - fmt.Sprintf("Customer : %s\n", (*batch).Customer) + + fmt.Sprintf("Customer : %s\n", (*batch).Customer) + fmt.Sprintf("ProfileType : %s\n", (*batch).ProfileType) + fmt.Sprintf("Order Date : %s\n", (*batch).OrderDate) + fmt.Sprintf("Batch No : %s\n", (*batch).BatchNo) + @@ -329,4 +440,4 @@ func GenerateInputFile(batch *model.Batch) string { "***************************************\n" + "var_Out: ICCID/IMSI/KI\n" return result -} \ No newline at end of file +} diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 4e0099f78..1dcabad77 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -40,6 +40,8 @@ type Store interface { initialHlrActivationStatusOfProfiles string) (*model.Batch, error) CreateSimEntry(simEntry *model.SimEntry) error + UpdateSimEntryMsisdn(simId int64, msisdn string) + GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) Begin() } @@ -117,7 +119,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { if err != nil { fmt.Errorf("Getting last inserted id failed '%s'", err) } - theBatch.Id = id + theBatch.BatchId = id return err } @@ -185,11 +187,19 @@ func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { } -func (sdb SimBatchDB) GetAllSimEntriesForBarch(batchId int64) ([]model.SimEntry, error) { +func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) { result := []model.SimEntry{} return result, sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId ) } +func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE simId = :simId", + map[string]interface{}{ + "simId": simId, + "msisdn": msisdn, + }) + return err +} func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` @@ -219,6 +229,14 @@ func (sdb SimBatchDB) DeclareBatch( profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { + alreadyExistingBatch, err := sdb.GetBatchByName(name) + if err != nil { + panic(err) + } + + if alreadyExistingBatch != nil { + panic(fmt.Sprintf("Batch already defined: '%s'", name)) + } // TODO: // 1. Check all the arguments (methods already written). @@ -349,7 +367,7 @@ func (sdb SimBatchDB) DeclareBatch( iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) simEntry := &model.SimEntry{ - BatchID: batch.Id, + BatchID: batch.BatchId, RawIccid: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), IccidWithChecksum: iccidWithLuhnChecksum, IccidWithoutChecksum: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 574ce6d25..47ffbcf4f 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -124,7 +124,7 @@ func declareTestBatch(t *testing.T) *model.Batch { func TestDeclareBatch(t *testing.T) { theBatch := declareTestBatch(t) - retrievedValue, _ := sdb.GetBatchById(theBatch.Id) + retrievedValue, _ := sdb.GetBatchById(theBatch.BatchId) if !reflect.DeepEqual(*retrievedValue, *theBatch) { t.Fatal("getBatchById failed") } @@ -136,7 +136,7 @@ func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { func TestDeclareAndRetrieveSimEntries(t *testing.T) { theBatch := declareTestBatch(t) - batchId := theBatch.Id + batchId := theBatch.BatchId entry := model.SimEntry{ BatchID: batchId, @@ -161,7 +161,7 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { t.Fatal("Retrieved and stored sim entry are different") } - retrievedEntries, err := sdb.GetAllSimEntriesForBarch(entry.BatchID) + retrievedEntries, err := sdb.GetAllSimEntriesForBatch(entry.BatchID) if err != nil { t.Fatal(err) } From f181f1eea5445e91b3bba17344a95f1b05454796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 14:38:26 +0100 Subject: [PATCH 155/309] Getting started on setting activation codes for batch in database --- .../sim-batch-management/sim-batch-mgt.go | 59 +++++++++++++++++-- .../sim-batch-management/store/store.go | 11 ++++ .../sim-batch-management/store/store_test.go | 3 +- 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index ac5d603e7..9408c2b29 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -6,8 +6,8 @@ import ( "encoding/csv" "encoding/json" "fmt" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/uploadtoprime" @@ -205,6 +205,8 @@ func main() { csvFile, _ := os.Open(csvFilename) reader := csv.NewReader(bufio.NewReader(csvFile)) + defer csvFile.Close() + headerLine, error := reader.Read() if error == io.EOF { break @@ -219,11 +221,11 @@ func main() { columnMap[strings.ToLower(fieldname)] = index } - if _, hasIccid := columnMap["iccid"] ; !hasIccid { + if _, hasIccid := columnMap["iccid"]; !hasIccid { panic("No ICCID column in CSV file") } - if _, hasMsisdn := columnMap["msisdn"]; !hasMsisdn { + if _, hasMsisdn := columnMap["msisdn"]; !hasMsisdn { panic("No MSISDN column in CSV file") } @@ -293,7 +295,7 @@ func main() { tx.Rollback() panic(err) } - noOfRecordsUpdated += 1 + noOfRecordsUpdated += 1 } } tx.Commit() @@ -327,6 +329,7 @@ func main() { client := es2plus.Client(*es2CertFilePath, *es2KeyFilePath, *es2Hostport, *es2RequesterId) iccid := *es2iccid switch *es2cmd { + case "get-status": result, err := client.GetStatus(iccid) if err != nil { @@ -361,6 +364,54 @@ func main() { } fmt.Printf("%s, %s\n", iccid, result.ACToken) + case "set-batch-activation-codes": + batchName := iccid + batch, err := db.GetBatchByName(batchName) + if err != nil { + fmt.Errorf("Unknown batch '%s'\n", batchName) + } + + entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + panic(err) + } + + // XXX Is this really necessary? I don't think so + // var mutex = &sync.Mutex{} + + fmt.Println("Starting to loop over entries") + // var waitgroup sync.WaitGroup + for _, entry := range entries { + + fmt.Printf("Processing iccid %s, activation code = '%s'\n", entry.Iccid, entry.ActivationCode) + // + // Only apply activation if not already noted in the + // database. + if entry.ActivationCode == "" { + fmt.Printf("Activating iccid = %s.", iccid) + iccid := entry.Iccid + // waitgroup.Add(1) + // go func(iccid string) { + + result, err := client.ActivateIccid(iccid) + if err != nil { + panic(err) + } + fmt.Printf("%s, %s", iccid, result.ACToken) + // mutex.Lock() + tx := db.Begin() + db.UpdateActivationCode(entry.SimId, result.ACToken) + tx.Commit() + // mutex.Unlock() + // waitgroup.Done() + // }(iccid) + } else { + fmt.Printf("Skipping iccid = %s empty activation code ('%s')", iccid, entry.ActivationCode) + } + } + + // waitgroup.Wait() + case "bulk-activate-iccids": file, err := os.Open(iccid) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 1dcabad77..dcfe94de9 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -41,6 +41,7 @@ type Store interface { CreateSimEntry(simEntry *model.SimEntry) error UpdateSimEntryMsisdn(simId int64, msisdn string) + UpdateActivationCode(simId int64, activationCode string) error GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) Begin() @@ -147,6 +148,7 @@ func (sdb *SimBatchDB) GenerateTables() error { foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( simId INTEGER PRIMARY KEY AUTOINCREMENT, batchId INTEGER NOT NULL, + activationCode VARCHAR, imsi VARCHAR NOT NULL, rawIccid VARCHAR NOT NULL, iccidWithChecksum VARCHAR NOT NULL, @@ -201,6 +203,15 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { return err } +func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE simId = :simId", + map[string]interface{}{ + "simId": simId, + "activationCode": activationCode, + }) + return err +} + func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` _, err := sdb.Db.Exec(foo) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 47ffbcf4f..7bf25f25e 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -131,7 +131,7 @@ func TestDeclareBatch(t *testing.T) { } func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { - return ((a.BatchID == b.BatchID) && (a.RawIccid == b.RawIccid) && a.IccidWithChecksum == b.IccidWithChecksum && a.IccidWithoutChecksum == b.IccidWithoutChecksum && a.Iccid == b.Iccid && a.Imsi == b.Imsi && a.Msisdn == b.Msisdn && a.Ki == b.Ki) + return ((a.ActivationCode == b.ActivationCode) && (a.BatchID == b.BatchID) && (a.RawIccid == b.RawIccid) && a.IccidWithChecksum == b.IccidWithChecksum && a.IccidWithoutChecksum == b.IccidWithoutChecksum && a.Iccid == b.Iccid && a.Imsi == b.Imsi && a.Msisdn == b.Msisdn && a.Ki == b.Ki) } func TestDeclareAndRetrieveSimEntries(t *testing.T) { @@ -147,6 +147,7 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { Imsi: "5", Msisdn: "6", Ki: "7", + ActivationCode: "8", } // assert.Equal(t, 0, entry.SimId) From dcdf82617ad43f66e9104bec35c55cca1e461374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 15:45:06 +0100 Subject: [PATCH 156/309] NOw many things works, unknown how many --- .../sim-batch-management/sim-batch-mgt.go | 3 ++ .../sim-batch-management/store/store.go | 41 ++++++++----------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 9408c2b29..cb1952d92 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -366,6 +366,9 @@ func main() { case "set-batch-activation-codes": batchName := iccid + + fmt.Printf("Getting batch named %s\n", batchName) + batch, err := db.GetBatchByName(batchName) if err != nil { fmt.Errorf("Unknown batch '%s'\n", batchName) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index dcfe94de9..24a82423f 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -42,12 +42,12 @@ type Store interface { CreateSimEntry(simEntry *model.SimEntry) error UpdateSimEntryMsisdn(simId int64, msisdn string) UpdateActivationCode(simId int64, activationCode string) error - GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) + GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) Begin() } -func (sdb *SimBatchDB) Begin() *sql.Tx { +func (sdb *SimBatchDB) Begin() *sql.Tx { tx, err := sdb.Db.Begin() if err != nil { panic(err) @@ -124,7 +124,6 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { return err } - func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, @@ -148,7 +147,7 @@ func (sdb *SimBatchDB) GenerateTables() error { foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( simId INTEGER PRIMARY KEY AUTOINCREMENT, batchId INTEGER NOT NULL, - activationCode VARCHAR, + activationCode VARCHAR NOT NULL, imsi VARCHAR NOT NULL, rawIccid VARCHAR NOT NULL, iccidWithChecksum VARCHAR NOT NULL, @@ -161,11 +160,11 @@ func (sdb *SimBatchDB) GenerateTables() error { return err } - func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { - res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?)", + res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", (*theEntry).BatchID, + (*theEntry).ActivationCode, (*theEntry).RawIccid, (*theEntry).IccidWithChecksum, (*theEntry).IccidWithoutChecksum, @@ -183,15 +182,14 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { return err } -func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { +func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { var result model.SimEntry return &result, sdb.Db.Get(&result, "select * from SIM_PROFILE where simId = ?", simId) } - -func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) { +func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) { result := []model.SimEntry{} - return result, sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId ) + return result, sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId) } func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { @@ -203,10 +201,10 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { return err } -func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { +func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE simId = :simId", map[string]interface{}{ - "simId": simId, + "simId": simId, "activationCode": activationCode, }) return err @@ -240,15 +238,6 @@ func (sdb SimBatchDB) DeclareBatch( profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { - alreadyExistingBatch, err := sdb.GetBatchByName(name) - if err != nil { - panic(err) - } - - if alreadyExistingBatch != nil { - panic(fmt.Sprintf("Batch already defined: '%s'", name)) - } - // TODO: // 1. Check all the arguments (methods already written). // 2. Check that the name isn't already registred. @@ -319,7 +308,7 @@ func (sdb SimBatchDB) DeclareBatch( log.Printf("Unknown parameters: %s", flag.Args()) } - filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) + filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) fmt.Printf("Filename base = '%s'\n", filenameBase) batch := model.Batch{ @@ -359,7 +348,7 @@ func (sdb SimBatchDB) DeclareBatch( panic(err) } - imsi, err := strconv.Atoi(batch.FirstImsi) + imsi, err := strconv.Atoi(batch.FirstImsi) if err != nil { panic(err) } @@ -369,16 +358,18 @@ func (sdb SimBatchDB) DeclareBatch( iccidWithoutLuhnChecksum := firstIccidInt // XXX !!! TODO THis is wrong, but I'm doing it now, just to get started! - var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) + var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) if err2 != nil { panic(err) } for i := 0; i < batch.Quantity; i++ { + iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) - simEntry := &model.SimEntry{ + simEntry := &model.SimEntry{ BatchID: batch.BatchId, + ActivationCode: "", RawIccid: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), IccidWithChecksum: iccidWithLuhnChecksum, IccidWithoutChecksum: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), From 470cf7d30b1adf0219645750848f7862f103131a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 16:13:53 +0100 Subject: [PATCH 157/309] Limit the number of concurrent goroutines accessing the sm-dp+ to avoid running out of file descriptors. --- .../sim-batch-management/es2plus/es2plus.go | 18 +++++-- .../sim-batch-management/sim-batch-mgt.go | 53 +++++++++++-------- .../sim-batch-management/store/store.go | 2 +- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index f4a9fd446..72b966b55 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -419,21 +419,29 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOr func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error){ result, err := client.GetStatus(iccid) + if err != nil { panic(err) } - if result.ACToken == "" { + if result.ACToken == "" { - _, err := client.DownloadOrder(iccid) - if err != nil { - return nil, err + if result.State == "AVAILABLE" { + _, err := client.DownloadOrder(iccid) + if err != nil { + return nil, err + } + result, err = client.GetStatus(iccid) } + + if result.State == "ALLOCATED" { _, err = client.ConfirmOrder(iccid) if err != nil { return nil, err } - } + } + + } result, err = client.GetStatus(iccid) return result, err } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index cb1952d92..c54f96a8c 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -380,40 +380,49 @@ func main() { } // XXX Is this really necessary? I don't think so - // var mutex = &sync.Mutex{} + var mutex = &sync.Mutex{} fmt.Println("Starting to loop over entries") - // var waitgroup sync.WaitGroup + var waitgroup sync.WaitGroup + concurrency := 80 + sem := make(chan bool, concurrency) + for _, entry := range entries { - fmt.Printf("Processing iccid %s, activation code = '%s'\n", entry.Iccid, entry.ActivationCode) // // Only apply activation if not already noted in the // database. if entry.ActivationCode == "" { - fmt.Printf("Activating iccid = %s.", iccid) - iccid := entry.Iccid - // waitgroup.Add(1) - // go func(iccid string) { - result, err := client.ActivateIccid(iccid) - if err != nil { - panic(err) - } - fmt.Printf("%s, %s", iccid, result.ACToken) - // mutex.Lock() - tx := db.Begin() - db.UpdateActivationCode(entry.SimId, result.ACToken) - tx.Commit() - // mutex.Unlock() - // waitgroup.Done() - // }(iccid) - } else { - fmt.Printf("Skipping iccid = %s empty activation code ('%s')", iccid, entry.ActivationCode) + sem <- true + + // fmt.Printf("Processing iccid %s, activation code = '%s'\n", entry.Iccid, entry.ActivationCode) + + iccid := entry.Iccid + waitgroup.Add(1) + go func(iccid string) { + + defer func() { <-sem }() + + result, err := client.ActivateIccid(iccid) + if err != nil { + panic(err) + } + fmt.Printf("%s, %s\n", iccid, result.ACToken) + mutex.Lock() + tx := db.Begin() + db.UpdateActivationCode(entry.SimId, result.ACToken) + tx.Commit() + mutex.Unlock() + waitgroup.Done() + }(iccid) } } - // waitgroup.Wait() + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } case "bulk-activate-iccids": diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 24a82423f..a0f1bf169 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -364,7 +364,7 @@ func (sdb SimBatchDB) DeclareBatch( } for i := 0; i < batch.Quantity; i++ { - + iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) simEntry := &model.SimEntry{ From 31332467654ed842df42e96b36a73c2c26e3c4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 18:16:43 +0100 Subject: [PATCH 158/309] Adding comments --- .../sim-batch-management/sim-batch-mgt.go | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index c54f96a8c..18ca82383 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -289,7 +289,7 @@ func main() { panic(fmt.Sprintf("MSISDN mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Msisdn, record.msisdn)) } - if (entry.Msisdn == "" && record.msisdn != "") { + if entry.Msisdn == "" && record.msisdn != "" { err = db.UpdateSimEntryMsisdn(entry.SimId, record.msisdn) if err != nil { tx.Rollback() @@ -384,9 +384,14 @@ func main() { fmt.Println("Starting to loop over entries") var waitgroup sync.WaitGroup - concurrency := 80 - sem := make(chan bool, concurrency) + // Limit concurency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. + concurrency := 160 + sem := make(chan bool, concurrency) + tx := db.Begin() for _, entry := range entries { // @@ -396,33 +401,31 @@ func main() { sem <- true - // fmt.Printf("Processing iccid %s, activation code = '%s'\n", entry.Iccid, entry.ActivationCode) - - iccid := entry.Iccid waitgroup.Add(1) - go func(iccid string) { + go func(entry model.SimEntry) { defer func() { <-sem }() - result, err := client.ActivateIccid(iccid) + result, err := client.ActivateIccid(entry.Iccid) if err != nil { panic(err) } - fmt.Printf("%s, %s\n", iccid, result.ACToken) + fmt.Printf("%s, %s\n", entry.Iccid, result.ACToken) mutex.Lock() - tx := db.Begin() + db.UpdateActivationCode(entry.SimId, result.ACToken) - tx.Commit() + mutex.Unlock() waitgroup.Done() - }(iccid) + }(entry) } } - waitgroup.Wait() + waitgroup.Wait() for i := 0; i < cap(sem); i++ { sem <- true } + tx.Commit() case "bulk-activate-iccids": From cdb3ebcba311ab7720afff84a125989296c0e322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 18:36:38 +0100 Subject: [PATCH 159/309] Generate sql code for updating activation codes. --- .../sim-batch-management/sim-batch-mgt.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 18ca82383..a6ce2ca63 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -94,6 +94,9 @@ var ( generateUploadBatch = kingpin.Command("generate-batch-upload-script", "Generate a batch upload script") generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to generate upload script from").String() + generateActivationCodeSql = kingpin.Command("generate-activation-code-updating-sql", "Generate SQL code to update access codes") + generateActivationCodeSqlBatch = generateActivationCodeSql.Arg("batch", "The batch to generate sql coce for").String() + db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") dbName = db.Flag("name", "Unique name of this batch").Required().String() dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() @@ -167,6 +170,22 @@ func main() { fmt.Printf("%v\n", string(bytes)) } + case "generate-activation-code-updating-sql": + fmt.Println("hahaha") + batch, err := db.GetBatchByName(*generateActivationCodeSqlBatch) + if err != nil { + panic(err) + } + + simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + panic(err) + } + + for _, b := range simEntries { + fmt.Printf("UPDATE INTO sim_entries (matchingid) VALUES ('%s') WHERE iccid='%s'\n;", b.ActivationCode, b.Iccid) + } + case "generate-batch-upload-script": batch, err := db.GetBatchByName(*generateUploadBatchBatch) if err != nil { From cd7e1435c9bd6449d57ebbc4a0d50d842a60d506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 19:18:35 +0100 Subject: [PATCH 160/309] Make the insertion script with the stored sim entries. --- .../sim-batch-management/sim-batch-mgt.go | 2 +- .../uploadtoprime/uploadtoprime.go | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index a6ce2ca63..5cffd0f8c 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -195,7 +195,7 @@ func main() { if batch == nil { fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) } else { - var csvPayload = uploadtoprime.GenerateCsvPayload2(*batch) + var csvPayload = uploadtoprime.GenerateCsvPayload3(db, *batch) uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) } diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index c26954561..76ecbfcf0 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -14,6 +14,7 @@ import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" "strconv" "strings" ) @@ -45,6 +46,9 @@ func GenerateCsvPayload2(batch model.Batch) string { panic(err) } + + + // TODO: Replace with a loop over actual entries for i := 0; i < batch.Quantity; i++ { iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.ProfileType) @@ -57,3 +61,20 @@ func GenerateCsvPayload2(batch model.Batch) string { return sb.String() } + +func GenerateCsvPayload3(db *store.SimBatchDB, batch model.Batch) string { + var sb strings.Builder + sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") + + entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + panic(err) + } + + for _ , entry:= range entries { + line := fmt.Sprintf("%s, %s, %s,,,,,%s\n", entry.Iccid, entry.Imsi, entry.Msisdn, batch.ProfileType) + sb.WriteString(line) + } + + return sb.String() +} From 0df2d72d99143e83e2b8312cac2c71febed14784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 19:19:05 +0100 Subject: [PATCH 161/309] Remove now obsolete comments --- .../uploadtoprime/uploadtoprime.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index 76ecbfcf0..1dcb5c57a 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -1,13 +1,3 @@ -//usr/bin/env go run "$0" "$@"; exit "$?" - -// XXX This is an utility script to feed the prime with sim profiles. -// it is actually a much better idea to extend the import functionality of -// prime to generate sequences and checksums, but that will require a major -// extension of a program that is soon going into production, so I'm keeping this -// complexity external for now. However, the existance of this program should be -// considered technical debt, and the debt can be paid back e.g. by -// internalizing the logic into prime. - package uploadtoprime import ( From b9ae33feb973fc28c9a22fbbb64262292b5deb6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 19:19:48 +0100 Subject: [PATCH 162/309] Nuke code no longer being used --- .../uploadtoprime/uploadtoprime.go | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index 1dcb5c57a..6c34ef0bb 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -2,10 +2,8 @@ package uploadtoprime import ( "fmt" - "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" - "strconv" "strings" ) @@ -17,41 +15,6 @@ func GeneratePostingCurlscript(url string, payload string) { fmt.Print("EOF\n") } -func GenerateCsvPayload2(batch model.Batch) string { - var sb strings.Builder - sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") - - iccidWithoutLuhnChecksum, err := strconv.Atoi(batch.FirstIccid) - if err != nil { - panic(err) - } - - imsi, err := strconv.Atoi(batch.FirstImsi) - if err != nil { - panic(err) - } - - var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) - if err2 != nil { - panic(err) - } - - - - // TODO: Replace with a loop over actual entries - for i := 0; i < batch.Quantity; i++ { - iccid := fmt.Sprintf("%d%1d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) - line := fmt.Sprintf("%s, %d, %d,,,,,%s\n", iccid, imsi, msisdn, batch.ProfileType) - sb.WriteString(line) - - iccidWithoutLuhnChecksum += batch.IccidIncrement - imsi += batch.ImsiIncrement - msisdn += batch.MsisdnIncrement - } - - return sb.String() -} - func GenerateCsvPayload3(db *store.SimBatchDB, batch model.Batch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") From 6c90f6a676ad7778d3949d099765788af8f538f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 19:22:14 +0100 Subject: [PATCH 163/309] Update todo --- .../sim-batch-management/TODO.md | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index d2ddf2bbd..de35fad19 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,26 +1,9 @@ -An informal TODO list for the sim batch management tool +TODO == - -1. Ingest 10K iccid/imsi/msisdn .csv file into existing batch. - -1. Get activation codes for all ICCIdes in an existing batch. - -1. Ingest input files into a corresponding batch, read all the sim - profiles into a sim profile table. This table can then be enriched - with access codes, MSISDNs etc, and then written to both HSSes and - be transformed into various types of upload files. -1. Rewrite upload-sim-batch-lib-test.go to be part of sim-batch-mgt.go, - during that process: - * Persist the batch data [Done] - * List persisted batches, show the status (use json). - * Persist access parameters for Prime(s) (prod, and pre-prod) (indirectly). - * Persist access parameters for sim vendors and HSSes - * Check referential integrity in data model so that batches don't refer to - HSSes or other types of entities. - * From the batch information, generate - - Input file to SIM vendor. - - Upload file for Prime instances. - * From output file, generate input file for HSS without storing Ki values. +1. Make the build-all script run without errors. +1. Clean up the code a lot +1. Take pending code review comments into account. +1. Create a very clean PR for future code review. 1. Add crypto resources so that the program can talk to external parties. 1. Figure out how to handle workflows. Be explicit! 1. Handle both parameterized lists of MSISDNs and list-based input. @@ -32,10 +15,3 @@ An informal TODO list for the sim batch management tool it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. - - -Notes -== - - simmgr_inventory=> select count(*) from sim_entries where profile = 'OYA_M1_STANDARD_ACB' and smdpplusstate = 'RELEASED'; - \ No newline at end of file From 12365c98d8e693e549c91af01417106e0c39ebcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 19:46:45 +0100 Subject: [PATCH 164/309] Adding more instrumentation, hoping for the best --- .../sim-batch-management/store/store.go | 5 ++++ .../sim-batch-management/store/store_test.go | 23 +++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index a0f1bf169..720154879 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -213,6 +213,11 @@ func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) e func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` _, err := sdb.Db.Exec(foo) + if err != nil { + return err + } + foo = `DROP TABLE SIM_PROFILE` + _, err = sdb.Db.Exec(foo) return err } diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 7bf25f25e..eff609032 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -24,18 +24,28 @@ func setup() { // sdb, err = store.OpenFileSqliteDatabase("bazunka.db") sdb, err = NewInMemoryDatabase() if err != nil { - fmt.Errorf("Couldn't open new in memory database '%s", err) + fmt.Sprintf("Couldn't open new in memory database '%s", err) } + + batches, err := sdb.GetAllBatches() + if err == nil { + panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) + } + + if len(batches) != 0 { + panic(fmt.Sprintf("batches already registred, test misconfigured")) + } + err = sdb.GenerateTables() if err != nil { - fmt.Errorf("Couldn't generate tables '%s'", err) + fmt.Sprintf("Couldn't generate tables '%s'", err) } } func shutdown() { - sdb.DropTables() + err := sdb.DropTables() if err != nil { - fmt.Errorf("Couldn't drop tables '%s'", err) + panic(fmt.Sprintf("Couldn't drop tables '%s'", err)) } } @@ -61,6 +71,11 @@ func injectTestBatch() *model.Batch { MsisdnIncrement: -1, } + batch, _ := sdb.GetBatchByName(theBatch.Name) + if batch != nil { + fmt.Errorf("Duplicate batch detected %s", theBatch.Name) + } + err := sdb.CreateBatch(&theBatch) if err != nil { panic(err) From e759931b3cd4f2f0b7c769739375570b46b78881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 22:51:39 +0100 Subject: [PATCH 165/309] Change name of batchid from id to batchid --- sim-administration/sim-batch-management/store/store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 720154879..f1a9874b0 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -88,7 +88,7 @@ func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { var result model.Batch - return &result, sdb.Db.Get(&result, "select * from BATCH where id = ?", id) + return &result, sdb.Db.Get(&result, "select * from BATCH where batchId = ?", id) } func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { @@ -126,7 +126,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( - id integer primary key autoincrement, + batchId integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, filenameBase VARCHAR NOT NULL, customer VARCHAR NOT NULL, From 29d31b3d09c6f5509fcf558dc72cfb2eb400b59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 23:34:59 +0100 Subject: [PATCH 166/309] Make the metaedata in sync with the realities of the rest of the program --- sim-administration/sim-batch-management/model/model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index ad34455e1..e6f71bad5 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -37,7 +37,7 @@ type SimEntry struct { // now. type Batch struct { - BatchId int64 `db:"id" json:"id"` + BatchId int64 `batchId:"id" json:"batchId"` Name string `db:"name" json:"name"` // TODO: Customer is a misnomer: This is the customer name used when From 4d573826cdcab159235b695fa65a057c75f18e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 8 Nov 2019 23:49:55 +0100 Subject: [PATCH 167/309] Fix braino/typo --- sim-administration/sim-batch-management/model/model.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index e6f71bad5..1d3e986d9 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -37,7 +37,7 @@ type SimEntry struct { // now. type Batch struct { - BatchId int64 `batchId:"id" json:"batchId"` + BatchId int64 `db:"batchId" json:"batchId"` Name string `db:"name" json:"name"` // TODO: Customer is a misnomer: This is the customer name used when From 4f3e168fb3acfb2719002471e75ae58c06532e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 08:56:52 +0100 Subject: [PATCH 168/309] Add static analysis to build-all.sh --- sim-administration/sim-batch-management/build-all.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/build-all.sh b/sim-administration/sim-batch-management/build-all.sh index dc3b10bc4..8860fc0b9 100755 --- a/sim-administration/sim-batch-management/build-all.sh +++ b/sim-administration/sim-batch-management/build-all.sh @@ -1,6 +1,9 @@ #!/bin/bash + + +# First we want the thing to compile go build if [ "$?" -ne "0" ]; then @@ -10,7 +13,7 @@ fi - +# THen to pass tests go test ./... if [ "$?" -ne "0" ]; then @@ -18,4 +21,10 @@ if [ "$?" -ne "0" ]; then exit 1 fi +# Then... +# somewhat nonportably ... run static analysis of the +# go code. + + ~/go/bin/staticcheck ./... + From 3a521f46b3f2c1e88f3309a52e893b552cf53d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 13:33:47 +0100 Subject: [PATCH 169/309] Stuff going on --- go.mod | 1 + go.sum | 23 ++++++++++ .../sim-batch-management/model/model.go | 43 ++++++------------- .../sim-batch-management/sim-batch-mgt.go | 36 ++++++++++------ .../sim-batch-management/store/store.go | 25 ++++++----- .../sim-batch-management/store/store_test.go | 38 +++++++++++++--- 6 files changed, 102 insertions(+), 64 deletions(-) diff --git a/go.mod b/go.mod index b233c635e..418248118 100644 --- a/go.mod +++ b/go.mod @@ -13,4 +13,5 @@ require ( github.com/pkg/errors v0.8.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gotest.tools v2.2.0+incompatible + honnef.co/go/tools v0.0.1-2019.2.3 // indirect ) diff --git a/go.sum b/go.sum index ffd017944..cf5fb4a8b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= @@ -9,10 +11,15 @@ github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -21,11 +28,27 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 1d3e986d9..44bfef410 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -1,43 +1,24 @@ package model -// TODO: There are now multiple structs that model batches. -// It's probably a good idea to harmonize these so that it's -// only one type of batch info that's being read, and then have -// various ways to combine the misc. sources of batch information -// that lets partial information from multiple records be harmonized -// in a common persisted record that is then used for the bulk of the -// processing. - - -// TODO: This struct isn't fully baked. +// TODO: This type SimEntry struct { struct isn't fully baked. type SimEntry struct { - SimId int64 `db:"simId" json:"simId"` - BatchID int64 `db:"batchId" json:"batchId"` - RawIccid string `db:"rawIccid" json:"rawIccid"` - IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` - IccidWithoutChecksum string `db:"iccidWithoutChecksum" json:"iccidWithoutChecksum"` - Iccid string `db:"iccid" json:"iccid"` - Imsi string `db:"imsi" json:"imsi"` - Msisdn string `db:"msisdn" json:"msisdn"` - Ki string `db:"ki" json:"ki"` - ActivationCode string `db:"activationCode" json:"activationCode"` + Id int64 `db:"id" json:"id"` + BatchID int64 `db:"batchId" json:"batchId"` + RawIccid string `db:"rawIccid" json:"rawIccid"` + IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` + IccidWithoutChecksum string `db:"iccidWithoutChecksum" json:"iccidWithoutChecksum"` + Iccid string `db:"iccid" json:"iccid"` + Imsi string `db:"imsi" json:"imsi"` + Msisdn string `db:"msisdn" json:"msisdn"` + Ki string `db:"ki" json:"ki"` + ActivationCode string `db:"activationCode" json:"activationCode"` } -// -// Below this line we grow the final persistence model. Eventually -// nothing below this line should be left. -// - - -// TODO: Add a filename base which is e.g. Footel201910110102, functional -// dependencies on other fields, but we'll not worry about that right -// now. - type Batch struct { - BatchId int64 `db:"batchId" json:"batchId"` + BatchId int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` // TODO: Customer is a misnomer: This is the customer name used when diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 5cffd0f8c..098f509aa 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -24,9 +24,11 @@ var ( // TODO: Enable, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") - es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() - es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").Required().String() + es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") + es2cmd = es2.Arg("cmd", + "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() + // .String() + es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").String() es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() es2KeyFilePath = es2.Flag("key", "Certificate key file.").Required().String() @@ -171,10 +173,10 @@ func main() { } case "generate-activation-code-updating-sql": - fmt.Println("hahaha") batch, err := db.GetBatchByName(*generateActivationCodeSqlBatch) if err != nil { - panic(err) + msg := fmt.Sprintf("Couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) + panic(msg) } simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) @@ -183,7 +185,10 @@ func main() { } for _, b := range simEntries { - fmt.Printf("UPDATE INTO sim_entries (matchingid) VALUES ('%s') WHERE iccid='%s'\n;", b.ActivationCode, b.Iccid) + fmt.Printf( + "UPDATE sim_entries SET matchingid = '%s', smdpplusstate = 'RELEASED', provisionstate = 'AVAILABLE' WHERE iccid = '%s' and smdpplusstate = 'AVAILABLE';\n", + b.ActivationCode, + b.Iccid) } case "generate-batch-upload-script": @@ -309,7 +314,7 @@ func main() { } if entry.Msisdn == "" && record.msisdn != "" { - err = db.UpdateSimEntryMsisdn(entry.SimId, record.msisdn) + err = db.UpdateSimEntryMsisdn(entry.Id, record.msisdn) if err != nil { tx.Rollback() panic(err) @@ -350,6 +355,7 @@ func main() { switch *es2cmd { case "get-status": + result, err := client.GetStatus(iccid) if err != nil { panic(err) @@ -398,16 +404,20 @@ func main() { panic(err) } + if len(entries) != batch.Quantity { + panic(fmt.Sprintf("Batch quantity retrieved from database (%d) different from batch quantity (%d)\n", len(entries), batch.Quantity)) + } + // XXX Is this really necessary? I don't think so var mutex = &sync.Mutex{} - fmt.Println("Starting to loop over entries") var waitgroup sync.WaitGroup - // Limit concurency of the for-loop below + // Limit concurrency of the for-loop below // to 160 goroutines. The reason is that if we get too // many we run out of file descriptors, and we don't seem to // get much speedup after hundred or so. + concurrency := 160 sem := make(chan bool, concurrency) tx := db.Begin() @@ -416,6 +426,7 @@ func main() { // // Only apply activation if not already noted in the // database. + if entry.ActivationCode == "" { sem <- true @@ -429,11 +440,10 @@ func main() { if err != nil { panic(err) } - fmt.Printf("%s, %s\n", entry.Iccid, result.ACToken) - mutex.Lock() - - db.UpdateActivationCode(entry.SimId, result.ACToken) + mutex.Lock() + fmt.Printf("%s, %s\n", entry.Iccid, result.ACToken) + db.UpdateActivationCode(entry.Id, result.ACToken) mutex.Unlock() waitgroup.Done() }(entry) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index f1a9874b0..2092ed8b1 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -60,7 +60,7 @@ type SimBatchDB struct { } func NewInMemoryDatabase() (*SimBatchDB, error) { - db, err := sqlx.Open("sqlite3", ":memory:") + db, err := sqlx.Connect("sqlite3", ":memory:") if err != nil { return nil, err } @@ -88,7 +88,7 @@ func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { var result model.Batch - return &result, sdb.Db.Get(&result, "select * from BATCH where batchId = ?", id) + return &result, sdb.Db.Get(&result, "select * from BATCH where id = ?", id) } func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { @@ -113,12 +113,12 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { (*theBatch).MsisdnIncrement, (*theBatch).IccidIncrement, (*theBatch).ImsiIncrement, - (*theBatch).Url, - ) + (*theBatch).Url) id, err := res.LastInsertId() if err != nil { - fmt.Errorf("Getting last inserted id failed '%s'", err) + // XXX Should be error logging + fmt.Printf("Getting last inserted id failed '%s'", err) } theBatch.BatchId = id return err @@ -126,7 +126,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( - batchId integer primary key autoincrement, + id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, filenameBase VARCHAR NOT NULL, customer VARCHAR NOT NULL, @@ -145,7 +145,7 @@ func (sdb *SimBatchDB) GenerateTables() error { _, err := sdb.Db.Exec(foo) foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( - simId INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT, batchId INTEGER NOT NULL, activationCode VARCHAR NOT NULL, imsi VARCHAR NOT NULL, @@ -178,13 +178,13 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { if err != nil { fmt.Errorf("Getting last inserted id failed '%s'", err) } - theEntry.SimId = id + theEntry.Id = id return err } func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { var result model.SimEntry - return &result, sdb.Db.Get(&result, "select * from SIM_PROFILE where simId = ?", simId) + return &result, sdb.Db.Get(&result, "select * from SIM_PROFILE where id = ?", simId) } func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) { @@ -193,7 +193,7 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, } func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { - _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE simId = :simId", + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", map[string]interface{}{ "simId": simId, "msisdn": msisdn, @@ -202,7 +202,7 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { } func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { - _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE simId = :simId", + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE id = :simId", map[string]interface{}{ "simId": simId, "activationCode": activationCode, @@ -222,7 +222,7 @@ func (sdb *SimBatchDB) DropTables() error { } /** - * CreateBatch a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. + * DeclareBatch a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. */ func (sdb SimBatchDB) DeclareBatch( name string, @@ -314,7 +314,6 @@ func (sdb SimBatchDB) DeclareBatch( } filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) - fmt.Printf("Filename base = '%s'\n", filenameBase) batch := model.Batch{ OrderDate: orderDate, diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index eff609032..fae0d13e7 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -21,7 +21,7 @@ func TestMain(m *testing.M) { } func setup() { - // sdb, err = store.OpenFileSqliteDatabase("bazunka.db") + // sdb, err = OpenFileSqliteDatabase("bazunka.db") sdb, err = NewInMemoryDatabase() if err != nil { fmt.Sprintf("Couldn't open new in memory database '%s", err) @@ -38,8 +38,23 @@ func setup() { err = sdb.GenerateTables() if err != nil { - fmt.Sprintf("Couldn't generate tables '%s'", err) + panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) + } + + + // The tables don't seem to be guaranteed to be empty + foo, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") + if err != nil { + panic(fmt.Sprintf("Couldn't delete SIM_PROFILE '%s'", err)) + } + bar, err := sdb.Db.Exec("DELETE FROM BATCH") + if err != nil { + panic(fmt.Sprintf("Couldn't delete BATCH '%s'", err)) } + + fooRows, _ := foo.RowsAffected() + barRows,_ := bar.RowsAffected() + fmt.Printf("foo = %d, bar=%d\n",fooRows, barRows) } func shutdown() { @@ -47,6 +62,7 @@ func shutdown() { if err != nil { panic(fmt.Sprintf("Couldn't drop tables '%s'", err)) } + sdb.Db.Close() } // ... just to know that everything is sane. @@ -87,7 +103,7 @@ func TestGetBatchById(t *testing.T) { theBatch := injectTestBatch() - firstInputBatch, _ := sdb.GetBatchById(1) + firstInputBatch, _ := sdb.GetBatchById(theBatch.BatchId) if !reflect.DeepEqual(*firstInputBatch, *theBatch) { fmt.Errorf("getBatchById failed") } @@ -95,13 +111,21 @@ func TestGetBatchById(t *testing.T) { func TestGetAllBatches(t *testing.T) { - theBatch := injectTestBatch() allBatches, err := sdb.GetAllBatches() if err != nil { fmt.Errorf("Reading query failed '%s'", err) } + assert.Equal(t, len(allBatches), 0) + + theBatch := injectTestBatch() + + allBatches, err = sdb.GetAllBatches() + if err != nil { + fmt.Errorf("Reading query failed '%s'", err) + } + assert.Equal(t, len(allBatches), 1) firstInputBatch := allBatches[0] @@ -165,11 +189,11 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { ActivationCode: "8", } - // assert.Equal(t, 0, entry.SimId) + // assert.Equal(t, 0, entry.Id) sdb.CreateSimEntry(&entry) - assert.Assert(t, entry.SimId != 0) + assert.Assert(t, entry.Id != 0) - retrivedEntry, err := sdb.GetSimEntryById(entry.SimId) + retrivedEntry, err := sdb.GetSimEntryById(entry.Id) if err != nil { t.Fatal(err) } From 019734909b33e9a10a5ff5535ae2e291e9d0730c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 15:20:31 +0100 Subject: [PATCH 170/309] Whitespace, and better at returnign when errors happen: --- .../sim-batch-management/store/store.go | 74 +++++++++++-------- .../sim-batch-management/store/store_test.go | 8 +- 2 files changed, 51 insertions(+), 31 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 2092ed8b1..ee00add4d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -64,6 +64,12 @@ func NewInMemoryDatabase() (*SimBatchDB, error) { if err != nil { return nil, err } + + err = db.Ping() + if err != nil { + return nil, err + } + return &SimBatchDB{Db: db}, nil } @@ -74,7 +80,7 @@ func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (* } func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { - db, err := sqlx.Open("sqlite3", "foobar.db") + db, err := sqlx.Open("sqlite3", path) if err != nil { return nil, err } @@ -98,7 +104,7 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { - res := sdb.Db.MustExec("INSERT INTO BATCH (name, filenameBase, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", + res, err := sdb.Db.Exec("INSERT INTO BATCH (name, filenameBase, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", (*theBatch).Name, (*theBatch).FilenameBase, (*theBatch).Customer, @@ -115,10 +121,19 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { (*theBatch).ImsiIncrement, (*theBatch).Url) + if err != nil { + // XXX Should be error logging + fmt.Printf("Failed to insert new batch '%s'", err) + return err + } + + + id, err := res.LastInsertId() if err != nil { // XXX Should be error logging fmt.Printf("Getting last inserted id failed '%s'", err) + return err } theBatch.BatchId = id return err @@ -126,36 +141,37 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( - id integer primary key autoincrement, - name VARCHAR NOT NULL UNIQUE, - filenameBase VARCHAR NOT NULL, - customer VARCHAR NOT NULL, - profileType VARCHAR NOT NULL, - orderDate VARCHAR NOT NULL, - batchNo VARCHAR NOT NULL, - quantity INTEGER NOT NULL, - firstIccid VARCHAR, - firstImsi VARCHAR, - firstMsisdn VARCHAR, - msisdnIncrement INTEGER, - imsiIncrement INTEGER, - iccidIncrement INTEGER, - url VARCHAR - )` + id integer primary key autoincrement, + name VARCHAR NOT NULL UNIQUE, + filenameBase VARCHAR NOT NULL, + customer VARCHAR NOT NULL, + profileType VARCHAR NOT NULL, + orderDate VARCHAR NOT NULL, + batchNo VARCHAR NOT NULL, + quantity INTEGER NOT NULL, + firstIccid VARCHAR, + firstImsi VARCHAR, + firstMsisdn VARCHAR, + msisdnIncrement INTEGER, + imsiIncrement INTEGER, + iccidIncrement INTEGER, + url VARCHAR)` _, err := sdb.Db.Exec(foo) + if err != nil { + return err + } foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - batchId INTEGER NOT NULL, - activationCode VARCHAR NOT NULL, - imsi VARCHAR NOT NULL, - rawIccid VARCHAR NOT NULL, - iccidWithChecksum VARCHAR NOT NULL, - iccidWithoutChecksum VARCHAR NOT NULL, - iccid VARCHAR NOT NULL, - ki VARCHAR NOT NULL, - msisdn VARCHAR NOT NULL - )` + id INTEGER PRIMARY KEY AUTOINCREMENT, + batchId INTEGER NOT NULL, + activationCode VARCHAR NOT NULL, + imsi VARCHAR NOT NULL, + rawIccid VARCHAR NOT NULL, + iccidWithChecksum VARCHAR NOT NULL, + iccidWithoutChecksum VARCHAR NOT NULL, + iccid VARCHAR NOT NULL, + ki VARCHAR NOT NULL, + msisdn VARCHAR NOT NULL)` _, err = sdb.Db.Exec(foo) return err } diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index fae0d13e7..32932b03f 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -26,7 +26,7 @@ func setup() { if err != nil { fmt.Sprintf("Couldn't open new in memory database '%s", err) } - +/* batches, err := sdb.GetAllBatches() if err == nil { panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) @@ -35,13 +35,15 @@ func setup() { if len(batches) != 0 { panic(fmt.Sprintf("batches already registred, test misconfigured")) } +*/ + err = sdb.GenerateTables() if err != nil { panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) } - +/* // The tables don't seem to be guaranteed to be empty foo, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") if err != nil { @@ -55,6 +57,8 @@ func setup() { fooRows, _ := foo.RowsAffected() barRows,_ := bar.RowsAffected() fmt.Printf("foo = %d, bar=%d\n",fooRows, barRows) + + */ } func shutdown() { From 63a6e6c6a08bc97a990fcbe8b2e72958f5943041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 15:25:07 +0100 Subject: [PATCH 171/309] Whitepace --- sim-administration/sim-batch-management/store/store_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 32932b03f..0a56b96d0 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -115,7 +115,6 @@ func TestGetBatchById(t *testing.T) { func TestGetAllBatches(t *testing.T) { - allBatches, err := sdb.GetAllBatches() if err != nil { fmt.Errorf("Reading query failed '%s'", err) From b60bc05e82bdbb9de4373899bb559bb7f7e6ef1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 19:06:28 +0100 Subject: [PATCH 172/309] Now tests pass, but ... there are some magical moves in there to make that happen. --- .../sim-batch-management/model/model.go | 2 +- .../sim-batch-management/store/store.go | 83 +++++++++++++------ .../sim-batch-management/store/store_test.go | 64 +++++++------- 3 files changed, 96 insertions(+), 53 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 44bfef410..2d146a619 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -18,7 +18,7 @@ type SimEntry struct { type Batch struct { - BatchId int64 `db:"id" json:"id"` + BatchId int64 `db:"id" json:"id"` // TODO: SHould this be called 'Id' Name string `db:"name" json:"name"` // TODO: Customer is a misnomer: This is the customer name used when diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index ee00add4d..b118cb942 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -103,23 +103,47 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { } func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { +// TODO: mutex +/* + res, err := sdb.Db.Exec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", + // "INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", +theBatch.Name ,theBatch.FilenameBase,theBatch.OrderDate, theBatch.Customer,theBatch.ProfileType, + theBatch.BatchNo, + theBatch.Quantity, + theBatch.FirstIccid, + theBatch.FirstImsi, + theBatch.FirstMsisdn, + theBatch.MsisdnIncrement, + theBatch.IccidIncrement, + theBatch.ImsiIncrement, + theBatch.Url, + ) +*/ + + +// TODO: a) Report it as a real error (minimal reproducable) +// b) Insert the object, then add extra fields, do it in a transaction, and don't break +/** foo := `CREATE TABLE IF NOT EXISTS BATCH ( + id integer primary key autoincrement, + name VARCHAR NOT NULL UNIQUE, + filenameBase VARCHAR NOT NULL, + customer VARCHAR NOT NULL, + profileType VARCHAR NOT NULL, + orderDate VARCHAR NOT NULL, + batchNo VARCHAR NOT NULL, + quantity INTEGER NOT NULL, + + */ +/* + res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", + theBatch, + ) + */ + + res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", + theBatch, + ) - res, err := sdb.Db.Exec("INSERT INTO BATCH (name, filenameBase, customer, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", - (*theBatch).Name, - (*theBatch).FilenameBase, - (*theBatch).Customer, - (*theBatch).OrderDate, - (*theBatch).Customer, - (*theBatch).ProfileType, - (*theBatch).BatchNo, - (*theBatch).Quantity, - (*theBatch).FirstIccid, - (*theBatch).FirstImsi, - (*theBatch).FirstMsisdn, - (*theBatch).MsisdnIncrement, - (*theBatch).IccidIncrement, - (*theBatch).ImsiIncrement, - (*theBatch).Url) if err != nil { // XXX Should be error logging @@ -128,7 +152,6 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { } - id, err := res.LastInsertId() if err != nil { // XXX Should be error logging @@ -136,6 +159,16 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { return err } theBatch.BatchId = id + + // "UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", + + res, err = sdb.Db.NamedExec("UPDATE BATCH SET firstIccid = :firstIccid, firstImsi = :firstImsi, firstMsisdn = :firstMsisdn, msisdnIncrement = :msisdnIncrement, iccidIncrement = :iccidIncrement, imsiIncrement = :imsiIncrement, url=:url WHERE id = :id", + theBatch) + // , :firstIccid, :firstImsi, :firstMsisdn, :msisdnIncrement, :iccidIncrement, :imsiIncrement, :url + + + + return err } @@ -143,12 +176,12 @@ func (sdb *SimBatchDB) GenerateTables() error { foo := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, - filenameBase VARCHAR NOT NULL, - customer VARCHAR NOT NULL, - profileType VARCHAR NOT NULL, - orderDate VARCHAR NOT NULL, - batchNo VARCHAR NOT NULL, - quantity INTEGER NOT NULL, + filenameBase VARCHAR, + customer VARCHAR, + profileType VARCHAR, + orderDate VARCHAR, + batchNo VARCHAR, + quantity INTEGER, firstIccid VARCHAR, firstImsi VARCHAR, firstMsisdn VARCHAR, @@ -179,7 +212,7 @@ func (sdb *SimBatchDB) GenerateTables() error { func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", - (*theEntry).BatchID, + (*theEntry).BatchID, // XXX Fix this! (*theEntry).ActivationCode, (*theEntry).RawIccid, (*theEntry).IccidWithChecksum, @@ -215,6 +248,7 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { "msisdn": msisdn, }) return err + } func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { @@ -348,6 +382,7 @@ func (sdb SimBatchDB) DeclareBatch( MsisdnIncrement: msisdnIncrement, } + tx := sdb.Begin() // This variable should be se to "true" if all transactions diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 0a56b96d0..048d87060 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -21,29 +21,39 @@ func TestMain(m *testing.M) { } func setup() { - // sdb, err = OpenFileSqliteDatabase("bazunka.db") - sdb, err = NewInMemoryDatabase() + + // In memory database fails, so we try this gonzo method of getting + // a fresh database + filename := "bazunka.db" + // delete file, ignore any errors + os.Remove(filename) + + sdb, err = OpenFileSqliteDatabase(filename) + // sdb, err = NewInMemoryDatabase() if err != nil { fmt.Sprintf("Couldn't open new in memory database '%s", err) } -/* - batches, err := sdb.GetAllBatches() - if err == nil { - panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) - } - - if len(batches) != 0 { - panic(fmt.Sprintf("batches already registred, test misconfigured")) - } -*/ + /* + batches, err := sdb.GetAllBatches() + if err == nil { + panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) + } + if len(batches) != 0 { + panic(fmt.Sprintf("batches already registred, test misconfigured")) + } + */ err = sdb.GenerateTables() if err != nil { panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) } -/* + cleanTables() +} + +func cleanTables() { + // The tables don't seem to be guaranteed to be empty foo, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") if err != nil { @@ -57,8 +67,6 @@ func setup() { fooRows, _ := foo.RowsAffected() barRows,_ := bar.RowsAffected() fmt.Printf("foo = %d, bar=%d\n",fooRows, barRows) - - */ } func shutdown() { @@ -115,6 +123,8 @@ func TestGetBatchById(t *testing.T) { func TestGetAllBatches(t *testing.T) { + cleanTables() + allBatches, err := sdb.GetAllBatches() if err != nil { fmt.Errorf("Reading query failed '%s'", err) @@ -168,8 +178,17 @@ func TestDeclareBatch(t *testing.T) { theBatch := declareTestBatch(t) retrievedValue, _ := sdb.GetBatchById(theBatch.BatchId) if !reflect.DeepEqual(*retrievedValue, *theBatch) { - t.Fatal("getBatchById failed") + t.Fatal("getBatchById failed, stored batch not equal to retrieved batch") + } + + retrievedEntries, err := sdb.GetAllSimEntriesForBatch(theBatch.BatchId) + if err != nil { + t.Fatal(err) } + assert.Equal(t, 1, len(retrievedEntries)) + + // TODO: Add check for content of retrieved entity + } func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { @@ -177,6 +196,7 @@ func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { } func TestDeclareAndRetrieveSimEntries(t *testing.T) { + cleanTables() theBatch := declareTestBatch(t) batchId := theBatch.BatchId @@ -203,16 +223,4 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { if !entryEqual(retrivedEntry, &entry) { t.Fatal("Retrieved and stored sim entry are different") } - - retrievedEntries, err := sdb.GetAllSimEntriesForBatch(entry.BatchID) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, 1, len(retrievedEntries)) - - retrivedEntry = &(retrievedEntries[0]) - - if !entryEqual(retrivedEntry, &entry) { - t.Fatal("Retrieved and stored sim entry are different") - } } From 813a43f4d91bea10811f3a6087b693c5fc94d340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 19:10:06 +0100 Subject: [PATCH 173/309] Clean up code a bit, pass all tests --- .../sim-batch-management/store/store_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 048d87060..1c6ed00ca 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -55,21 +55,22 @@ func setup() { func cleanTables() { // The tables don't seem to be guaranteed to be empty - foo, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") + simProfileDeletionResult, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") if err != nil { panic(fmt.Sprintf("Couldn't delete SIM_PROFILE '%s'", err)) } - bar, err := sdb.Db.Exec("DELETE FROM BATCH") + batchDeleteResult, err := sdb.Db.Exec("DELETE FROM BATCH") if err != nil { panic(fmt.Sprintf("Couldn't delete BATCH '%s'", err)) } - fooRows, _ := foo.RowsAffected() - barRows,_ := bar.RowsAffected() - fmt.Printf("foo = %d, bar=%d\n",fooRows, barRows) + simRows, _ := simProfileDeletionResult.RowsAffected() + batchRows,_ := batchDeleteResult.RowsAffected() + fmt.Printf("simProfileDeletionResult = %d, batchDeleteResult=%d\n",simRows, batchRows) } func shutdown() { + cleanTables() err := sdb.DropTables() if err != nil { panic(fmt.Sprintf("Couldn't drop tables '%s'", err)) From d27f158953377b8a97859301cac4c504f25e0907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 22:35:02 +0100 Subject: [PATCH 174/309] Start working on the syntax for the declaring profile vendors and other actors in the persistent storage, to be used in less verbose es2 and other commands. --- .../sim-batch-management/sim-batch-mgt.go | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 098f509aa..39aef5654 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -21,9 +21,28 @@ import ( // "gopkg.in/alecthomas/kingpin.v2" var ( - // TODO: Enable, but also make it have an effect. + // TODO: Global flags can be added to Kingpin, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() + + // Declare a profile-vendor with an SM-DP+ that can be referred to from + // batches. Referential integrity required, so it won't be possible to + // declare bathes with non-existing profile vendors. + dpv = kingpin.Command("declare-profile-vendor", "Declare a profile vendor with an SM-DP+ we can talk to") + dpvName = dpv.Flag("name", "Name of profile-vendor").Required().String() + dpvCertFilePath = dpv.Flag("cert", "Certificate pem file.").Required().String() + dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() + dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() + dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().String() + // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files + // TODO: Declare hss-es, that can be refered to in profiles. + // TODO: Declare legal hss/dpv combinations, batches must use legal combos. + // TODO: Declare contact methods for primes. It might be a good idea to + // impose referential integrity constraint on this too, so that + // profile/vendor/hss/prime combos are constrained. It should be possible + // to specify prod/dev primes. + + es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() @@ -142,6 +161,9 @@ func main() { case "sim-profile-upload": outfileparser.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) + case declare-profile-vendor": + fmt.Println("Declaration of profile-vendors not yet implemented.\n") + case "list-batches": allBatches, err := db.GetAllBatches() From b8095ae3c44858a245b7a469f3d79cb34aec0b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 11 Nov 2019 22:36:10 +0100 Subject: [PATCH 175/309] Add the beginning of a test for parsing complex output files. --- .../outfileparser/outfileparser_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 48d90741a..cd0b5ccdf 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -1,11 +1,11 @@ package outfileparser import ( + "fmt" "gotest.tools/assert" "testing" ) - func TestKeywordValueParser(t *testing.T) { theMap := make(map[string]string) ParseLineIntoKeyValueMap("ProfileType : BAR_FOOTEL_STD", theMap) @@ -36,16 +36,13 @@ func TestReadingSimpleOutputFile(t *testing.T) { assert.Equal(t, 3, record.NoOfEntries) } - - -/* TODO: Uncomment this test, and start on making it pass. func TestReadingComplexOutputFile(t *testing.T) { sample_output_file_name := "sample-out-2.out" record := ParseOutputFile(sample_output_file_name) fmt.Println("Record = ", record) -} - */ + // TODO: Check that we got all the fields +} func TestParseOutputVariablesLine(t *testing.T) { varOutLine := "var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4" From 2994ae288f8f94d4de7872c6d18c9a96a50c1435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 09:22:53 +0100 Subject: [PATCH 176/309] Added minimal test of failing things --- .../fieldsyntaxchecks/luhn_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go index 563c7250d..dae473bd7 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/luhn_test.go @@ -6,6 +6,7 @@ import ( func TestLuhn(t *testing.T) { validNumbers := []int{ + 8965030119110000013, 79927398713, 4929972884676289, 4532733309529845, @@ -44,6 +45,10 @@ func TestLuhn(t *testing.T) { 6387065788050980, 6388464094939979} + invalidNumbers := []int{ + 896503011911000001, + } + for _, number := range validNumbers { checksum := LuhnChecksum(number / 10) @@ -51,4 +56,12 @@ func TestLuhn(t *testing.T) { t.Errorf("%v's check number should be %v, but got %v", number, number%10, checksum) } } + + for _, number := range invalidNumbers { + + checksum := LuhnChecksum(number / 10) + if checksum == number%10 { + t.Errorf("%v's check number should _not_ be %v, but got %v", number, number%10, checksum) + } + } } From ad3b82eb6e628f05172ac079937835f91f4bc3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 09:50:35 +0100 Subject: [PATCH 177/309] Making sure the luhn checksums are legit --- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 5 +++++ .../sim-batch-management/sim-batch-mgt.go | 11 ++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go index 50cb93c5a..d7fce64e8 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -47,6 +47,11 @@ func LuhnChecksum(number int) int { return generateControlDigit(strconv.Itoa(number)) } +func AddLuhnChecksum(original string) string { + checksum := generateControlDigit(original) + return fmt.Sprintf("%s%1d", original, checksum) +} + func IsICCID(s string) bool { match, _ := regexp.MatchString("^\\d{18}\\d?\\d?$", s) return match diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 39aef5654..e6c8756b7 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -24,17 +24,17 @@ var ( // TODO: Global flags can be added to Kingpin, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - // Declare a profile-vendor with an SM-DP+ that can be referred to from // batches. Referential integrity required, so it won't be possible to // declare bathes with non-existing profile vendors. - dpv = kingpin.Command("declare-profile-vendor", "Declare a profile vendor with an SM-DP+ we can talk to") + dpv = kingpin.Command("declare-profile-vendor", "Declare a profile vendor with an SM-DP+ we can talk to") dpvName = dpv.Flag("name", "Name of profile-vendor").Required().String() dpvCertFilePath = dpv.Flag("cert", "Certificate pem file.").Required().String() dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().String() - // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files + // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... + // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files // TODO: Declare hss-es, that can be refered to in profiles. // TODO: Declare legal hss/dpv combinations, batches must use legal combos. // TODO: Declare contact methods for primes. It might be a good idea to @@ -42,7 +42,6 @@ var ( // profile/vendor/hss/prime combos are constrained. It should be possible // to specify prod/dev primes. - es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() @@ -120,6 +119,7 @@ var ( db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") dbName = db.Flag("name", "Unique name of this batch").Required().String() + dbAddLuhn = db.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() dbBatchNo = db.Flag("batch-no", "Unique number of this batch (with respect to the profile vendor)").Required().String() dbOrderDate = db.Flag("order-date", "Order date in format ddmmyyyy").Required().String() @@ -161,7 +161,7 @@ func main() { case "sim-profile-upload": outfileparser.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) - case declare-profile-vendor": + case "declare-profile-vendor": fmt.Println("Declaration of profile-vendors not yet implemented.\n") case "list-batches": @@ -352,6 +352,7 @@ func main() { fmt.Println("Declare batch") db.DeclareBatch( *dbName, + *dbAddLuhn, *dbCustomer, *dbBatchNo, *dbOrderDate, From 4d2c36a9c8fbd0fbe317fd08dbea04911c084d07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 13:53:27 +0100 Subject: [PATCH 178/309] Add generation of luhn cheksums, add code to pick out the state of all profiles in a batch --- .../sim-batch-management/TODO.md | 6 +- .../sim-batch-management/sim-batch-mgt.go | 84 +++++++++++++++++- .../sim-batch-management/store/store.go | 87 ++++++++++--------- .../sim-batch-management/store/store_test.go | 9 +- 4 files changed, 136 insertions(+), 50 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index de35fad19..29f3f1a4d 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,10 +1,14 @@ TODO == -1. Make the build-all script run without errors. +1. Make the build-all script run without errors (including linter errors) 1. Clean up the code a lot 1. Take pending code review comments into account. 1. Create a very clean PR for future code review. +1. Compress the utility scripts into very close to nothing, by adding their functionality to the .go code. 1. Add crypto resources so that the program can talk to external parties. +1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that + batches can be properly constrained, defaults set the right way and external + components accessed from gocode. 1. Figure out how to handle workflows. Be explicit! 1. Handle both parameterized lists of MSISDNs and list-based input. 1. The interfaces to external parties will be diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index e6c8756b7..1fafc4cca 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/es2plus" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/outfileparser" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" @@ -44,8 +45,7 @@ var ( es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", - "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid ").Required().String() - // .String() + "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid, get-profile-activation-statuses-for-batch").Required().String() es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").String() es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() @@ -110,6 +110,7 @@ var ( addMsisdnFromFile = kingpin.Command("add-msisdn-from-file", "Add MSISDN from CSV file containing at least ICCID/MSISDN, but also possibly IMSI.") addMsisdnFromFileBatch = addMsisdnFromFile.Flag("batch", "The batch to augment").Required().String() addMsisdnFromFileCsvfile = addMsisdnFromFile.Flag("csv-file", "The CSV file to read from").Required().ExistingFile() + addMsisdnFromFileAddLuhn = addMsisdnFromFile.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() generateUploadBatch = kingpin.Command("generate-batch-upload-script", "Generate a batch upload script") generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to generate upload script from").String() @@ -242,6 +243,7 @@ func main() { case "add-msisdn-from-file": batchName := *addMsisdnFromFileBatch csvFilename := *addMsisdnFromFileCsvfile + addLuhns := *addMsisdnFromFileAddLuhn batch, err := db.GetBatchByName(batchName) if err != nil { @@ -297,8 +299,14 @@ func main() { log.Fatal(error) } + iccid := line[columnMap["iccid"]] + + if addLuhns { + iccid = fieldsyntaxchecks.AddLuhnChecksum(iccid) + } + record := csvRecord{ - iccid: line[columnMap["iccid"]], + iccid: iccid, imsi: line[columnMap["imsi"]], msisdn: line[columnMap["msisdn"]], } @@ -412,6 +420,76 @@ func main() { } fmt.Printf("%s, %s\n", iccid, result.ACToken) + case "get-profile-activation-statuses-for-batch": + batchName := iccid + + fmt.Printf("Getting statuses for all profiles in batch named %s\n", batchName) + + batch, err := db.GetBatchByName(batchName) + if err != nil { + fmt.Errorf("Unknown batch '%s'\n", batchName) + } + + entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + panic(err) + } + + if len(entries) != batch.Quantity { + panic(fmt.Sprintf("Batch quantity retrieved from database (%d) different from batch quantity (%d)\n", len(entries), batch.Quantity)) + } + + fmt.Printf("Found %d profiles\n", len(entries)) + + // XXX Is this really necessary? I don't think so + var mutex = &sync.Mutex{} + + var waitgroup sync.WaitGroup + + // Limit concurrency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. + + concurrency := 160 + sem := make(chan bool, concurrency) + tx := db.Begin() + for _, entry := range entries { + + // + // Only apply activation if not already noted in the + // database. + // + + sem <- true + + waitgroup.Add(1) + go func(entry model.SimEntry) { + + defer func() { <-sem }() + + result, err := client.GetStatus(entry.Iccid) + if err != nil { + panic(err) + } + + if result == nil { + panic(fmt.Sprintf("Couldn't find any status for iccid='%s'\n", entry.Iccid)) + } + + mutex.Lock() + fmt.Printf("%s, %s\n", entry.Iccid, result.State) + mutex.Unlock() + waitgroup.Done() + }(entry) + } + + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } + tx.Commit() + case "set-batch-activation-codes": batchName := iccid diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index b118cb942..ade415981 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -103,55 +103,52 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { } func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { -// TODO: mutex -/* - res, err := sdb.Db.Exec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", - // "INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", -theBatch.Name ,theBatch.FilenameBase,theBatch.OrderDate, theBatch.Customer,theBatch.ProfileType, - theBatch.BatchNo, - theBatch.Quantity, - theBatch.FirstIccid, - theBatch.FirstImsi, - theBatch.FirstMsisdn, - theBatch.MsisdnIncrement, - theBatch.IccidIncrement, - theBatch.ImsiIncrement, - theBatch.Url, - ) -*/ - + // TODO: mutex + /* + res, err := sdb.Db.Exec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", + // "INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", + theBatch.Name ,theBatch.FilenameBase,theBatch.OrderDate, theBatch.Customer,theBatch.ProfileType, + theBatch.BatchNo, + theBatch.Quantity, + theBatch.FirstIccid, + theBatch.FirstImsi, + theBatch.FirstMsisdn, + theBatch.MsisdnIncrement, + theBatch.IccidIncrement, + theBatch.ImsiIncrement, + theBatch.Url, + ) + */ -// TODO: a) Report it as a real error (minimal reproducable) -// b) Insert the object, then add extra fields, do it in a transaction, and don't break -/** foo := `CREATE TABLE IF NOT EXISTS BATCH ( - id integer primary key autoincrement, - name VARCHAR NOT NULL UNIQUE, - filenameBase VARCHAR NOT NULL, - customer VARCHAR NOT NULL, - profileType VARCHAR NOT NULL, - orderDate VARCHAR NOT NULL, - batchNo VARCHAR NOT NULL, - quantity INTEGER NOT NULL, + // TODO: a) Report it as a real error (minimal reproducable) + // b) Insert the object, then add extra fields, do it in a transaction, and don't break + /** foo := `CREATE TABLE IF NOT EXISTS BATCH ( + id integer primary key autoincrement, + name VARCHAR NOT NULL UNIQUE, + filenameBase VARCHAR NOT NULL, + customer VARCHAR NOT NULL, + profileType VARCHAR NOT NULL, + orderDate VARCHAR NOT NULL, + batchNo VARCHAR NOT NULL, + quantity INTEGER NOT NULL, - */ -/* - res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", - theBatch, - ) + */ + /* + res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", + theBatch, + ) */ res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", theBatch, ) - if err != nil { // XXX Should be error logging fmt.Printf("Failed to insert new batch '%s'", err) return err } - id, err := res.LastInsertId() if err != nil { // XXX Should be error logging @@ -166,9 +163,6 @@ theBatch.Name ,theBatch.FilenameBase,theBatch.OrderDate, theBatch.Customer,theBa theBatch) // , :firstIccid, :firstImsi, :firstMsisdn, :msisdnIncrement, :iccidIncrement, :imsiIncrement, :url - - - return err } @@ -212,7 +206,7 @@ func (sdb *SimBatchDB) GenerateTables() error { func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", - (*theEntry).BatchID, // XXX Fix this! + (*theEntry).BatchID, // XXX Fix this! (*theEntry).ActivationCode, (*theEntry).RawIccid, (*theEntry).IccidWithChecksum, @@ -276,6 +270,7 @@ func (sdb *SimBatchDB) DropTables() error { */ func (sdb SimBatchDB) DeclareBatch( name string, + addLuhn bool, customer string, batchNo string, orderDate string, @@ -303,6 +298,15 @@ func (sdb SimBatchDB) DeclareBatch( // semantic sanity. // + fmt.Printf("Pre adding luhn ? %t. first='%s', last='%s'\n", addLuhn, firstIccid, lastIccid) + + if addLuhn { + firstIccid = fieldsyntaxchecks.AddLuhnChecksum(firstIccid) + lastIccid = fieldsyntaxchecks.AddLuhnChecksum(lastIccid) + } + + fmt.Printf("Post adding luhn ? %t. first='%s',last='%s'\n", addLuhn, firstIccid, lastIccid) + fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", firstIccid) fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", lastIccid) fieldsyntaxchecks.CheckIMSISyntax("last-imsi", lastIMSI) @@ -382,7 +386,6 @@ func (sdb SimBatchDB) DeclareBatch( MsisdnIncrement: msisdnIncrement, } - tx := sdb.Begin() // This variable should be se to "true" if all transactions @@ -390,7 +393,7 @@ func (sdb SimBatchDB) DeclareBatch( weCool := false defer func() { - if (weCool) { + if weCool { tx.Commit() } else { tx.Rollback() @@ -435,7 +438,7 @@ func (sdb SimBatchDB) DeclareBatch( } err = sdb.CreateSimEntry(simEntry) - if (err != nil) { + if err != nil { panic(err) } diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 1c6ed00ca..cf06e2f37 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -54,6 +54,7 @@ func setup() { func cleanTables() { + // return // The tables don't seem to be guaranteed to be empty simProfileDeletionResult, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") if err != nil { @@ -65,8 +66,8 @@ func cleanTables() { } simRows, _ := simProfileDeletionResult.RowsAffected() - batchRows,_ := batchDeleteResult.RowsAffected() - fmt.Printf("simProfileDeletionResult = %d, batchDeleteResult=%d\n",simRows, batchRows) + batchRows, _ := batchDeleteResult.RowsAffected() + fmt.Printf("simProfileDeletionResult = %d, batchDeleteResult=%d\n", simRows, batchRows) } func shutdown() { @@ -167,7 +168,7 @@ func declareTestBatch(t *testing.T) *model.Batch { "localhost", // uploadHostname string, "8088", // uploadPortnumber string, "snuff", // profileVendor string, - "ACTIVE") // initialHlrActivationStatusOfProfiles string + "ACTIVE") // initialHlrActivationStatusOfProfiles string if err != nil { t.Fatal(err) @@ -193,7 +194,7 @@ func TestDeclareBatch(t *testing.T) { } func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { - return ((a.ActivationCode == b.ActivationCode) && (a.BatchID == b.BatchID) && (a.RawIccid == b.RawIccid) && a.IccidWithChecksum == b.IccidWithChecksum && a.IccidWithoutChecksum == b.IccidWithoutChecksum && a.Iccid == b.Iccid && a.Imsi == b.Imsi && a.Msisdn == b.Msisdn && a.Ki == b.Ki) + return ((a.ActivationCode == b.ActivationCode) && (a.BatchID == b.BatchID) && (a.RawIccid == b.RawIccid) && a.IccidWithChecksum == b.IccidWithChecksum && a.IccidWithoutChecksum == b.IccidWithoutChecksum && a.Iccid == b.Iccid && a.Imsi == b.Imsi && a.Msisdn == b.Msisdn && a.Ki == b.Ki) } func TestDeclareAndRetrieveSimEntries(t *testing.T) { From acee12135913ea575de9f89a6c14971a3d0f2ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 14:43:36 +0100 Subject: [PATCH 179/309] Look up statuses of sim cards provided in csv file. --- .../sim-batch-management/sim-batch-mgt.go | 121 ++++++++++++++++-- 1 file changed, 111 insertions(+), 10 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 1fafc4cca..508a5f012 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -45,8 +45,8 @@ var ( es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", - "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-iccid, get-profile-activation-statuses-for-batch").Required().String() - es2iccid = es2.Arg("iccid", "Iccid of profile to manipulate").String() + "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-Iccid, get-profile-activation-statuses-for-batch, get-profile-activation-statuses-for-iccids-in-file").Required().String() + es2iccid = es2.Arg("Iccid", "Iccid of profile to manipulate").String() es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() es2KeyFilePath = es2.Flag("key", "Certificate key file.").Required().String() @@ -209,7 +209,7 @@ func main() { for _, b := range simEntries { fmt.Printf( - "UPDATE sim_entries SET matchingid = '%s', smdpplusstate = 'RELEASED', provisionstate = 'AVAILABLE' WHERE iccid = '%s' and smdpplusstate = 'AVAILABLE';\n", + "UPDATE sim_entries SET matchingid = '%s', smdpplusstate = 'RELEASED', provisionstate = 'AVAILABLE' WHERE Iccid = '%s' and smdpplusstate = 'AVAILABLE';\n", b.ActivationCode, b.Iccid) } @@ -269,7 +269,7 @@ func main() { columnMap[strings.ToLower(fieldname)] = index } - if _, hasIccid := columnMap["iccid"]; !hasIccid { + if _, hasIccid := columnMap["Iccid"]; !hasIccid { panic("No ICCID column in CSV file") } @@ -299,7 +299,7 @@ func main() { log.Fatal(error) } - iccid := line[columnMap["iccid"]] + iccid := line[columnMap["Iccid"]] if addLuhns { iccid = fieldsyntaxchecks.AddLuhnChecksum(iccid) @@ -392,7 +392,7 @@ func main() { panic(err) } - fmt.Printf("iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) + fmt.Printf("Iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) case "recover-profile": checkEs2TargetState(es2Target) result, err := client.RecoverProfile(iccid, *es2Target) @@ -412,7 +412,7 @@ func main() { panic(err) } fmt.Println("result -> ", result) - case "activate-iccid": + case "activate-Iccid": result, err := client.ActivateIccid(iccid) if err != nil { @@ -420,6 +420,109 @@ func main() { } fmt.Printf("%s, %s\n", iccid, result.ACToken) + case "get-profile-activation-statuses-for-iccids-in-file": + csvFilename := iccid + + csvFile, _ := os.Open(csvFilename) + reader := csv.NewReader(bufio.NewReader(csvFile)) + + defer csvFile.Close() + + headerLine, error := reader.Read() + if error == io.EOF { + break + } else if error != nil { + log.Fatal(error) + } + + var columnMap map[string]int + columnMap = make(map[string]int) + + for index, fieldname := range headerLine { + columnMap[strings.ToLower(fieldname)] = index + } + + if _, hasIccid := columnMap["iccid"]; !hasIccid { + panic("No ICCID column in CSV file") + } + + type csvRecord struct { + Iccid string + } + + var recordMap map[string]csvRecord + recordMap = make(map[string]csvRecord) + + // Read all the lines into the record map. + for { + line, error := reader.Read() + if error == io.EOF { + break + } else if error != nil { + log.Fatal(error) + } + + iccid := line[columnMap["Iccid"]] + + record := csvRecord{ + Iccid: iccid, + } + + if _, duplicateRecordExists := recordMap[record.Iccid]; duplicateRecordExists { + panic(fmt.Sprintf("Duplicate ICCID record in map: %s", record.Iccid)) + } + + recordMap[record.Iccid] = record + } + + // XXX Is this really necessary? I don't think so + var mutex = &sync.Mutex{} + + var waitgroup sync.WaitGroup + + // Limit concurrency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. + + concurrency := 160 + sem := make(chan bool, concurrency) + fmt.Printf("%s, %s\n", "ICCID", "STATE") + for _, entry := range recordMap { + + // + // Only apply activation if not already noted in the + // database. + // + + sem <- true + + waitgroup.Add(1) + go func(entry csvRecord) { + + defer func() { <-sem }() + + result, err := client.GetStatus(entry.Iccid) + if err != nil { + panic(err) + } + + if result == nil { + panic(fmt.Sprintf("Couldn't find any status for Iccid='%s'\n", entry.Iccid)) + } + + mutex.Lock() + fmt.Printf("%s, %s\n", entry.Iccid, result.State) + mutex.Unlock() + waitgroup.Done() + }(entry) + } + + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } + case "get-profile-activation-statuses-for-batch": batchName := iccid @@ -453,7 +556,6 @@ func main() { concurrency := 160 sem := make(chan bool, concurrency) - tx := db.Begin() for _, entry := range entries { // @@ -474,7 +576,7 @@ func main() { } if result == nil { - panic(fmt.Sprintf("Couldn't find any status for iccid='%s'\n", entry.Iccid)) + panic(fmt.Sprintf("Couldn't find any status for Iccid='%s'\n", entry.Iccid)) } mutex.Lock() @@ -488,7 +590,6 @@ func main() { for i := 0; i < cap(sem); i++ { sem <- true } - tx.Commit() case "set-batch-activation-codes": batchName := iccid From 17cec9a355210efd195c9e8b06add5e862e7909c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 15:42:35 +0100 Subject: [PATCH 180/309] All tests pass --- .../outfileparser/outfileparser.go | 10 ++++++++-- .../sim-batch-management/sim-batch-mgt.go | 2 +- .../sim-batch-management/store/store_test.go | 1 + 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 4634001fa..e5d5a9338 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -139,6 +139,9 @@ func ParseOutputFile(filename string) OutputFileRecord { nextMode := modeFromSectionHeader(line) transitionMode(&state, nextMode) continue + }else if line == "OUTPUT VARIABLES" { + transitionMode(&state, OUTPUT_VARIABLES) + continue } // ... or should we look closer at it and parse it @@ -148,9 +151,10 @@ func ParseOutputFile(filename string) OutputFileRecord { case HEADER_DESCRIPTION: ParseLineIntoKeyValueMap(line, state.headerDescription) case INPUT_VARIABLES: - if line == "var_In:" { + if line == "var_In:" || line == "Var_In_List:" || strings.TrimSpace(line) == "" { continue } + ParseLineIntoKeyValueMap(line, state.inputVariables) case OUTPUT_VARIABLES: @@ -261,12 +265,14 @@ func transitionMode(state *ParserState, targetState string) { // TODO: Consider replacing this thing with a map lookup. func modeFromSectionHeader(s string) string { - sectionName := s[1:] + sectionName := strings.Trim(s, "* ") switch sectionName { case "HEADER DESCRIPTION": return HEADER_DESCRIPTION case "INPUT VARIABLES": return INPUT_VARIABLES + case "INPUT VARIABLES DESCRIPTION": + return INPUT_VARIABLES case "OUTPUT VARIABLES": return OUTPUT_VARIABLES default: diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 508a5f012..ca66b6933 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -163,7 +163,7 @@ func main() { outfileparser.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) case "declare-profile-vendor": - fmt.Println("Declaration of profile-vendors not yet implemented.\n") + fmt.Println("Declaration of profile-vendors not yet implemented.") case "list-batches": diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index cf06e2f37..cc9aa15c4 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -153,6 +153,7 @@ func declareTestBatch(t *testing.T) *model.Batch { theBatch, err := sdb.DeclareBatch( "Name", + false, "Customer", "8778fsda", // batch number "20200101", // date string From 5b30fdce13238f5a660d7e94f294612d97b63ab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 16:47:57 +0100 Subject: [PATCH 181/309] Steps towards no failing tests, and no linter errors. Not quite there yet. --- .../sim-batch-management/es2plus/es2plus.go | 17 ++++++++------ .../outfileparser/outfileparser.go | 7 ++---- .../sim-batch-management/sim-batch-mgt.go | 11 +++++----- .../sim-batch-management/store/store.go | 5 +++-- .../sim-batch-management/store/store_test.go | 22 ++++++++++++------- 5 files changed, 34 insertions(+), 28 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 72b966b55..28c3b13ac 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -329,13 +329,13 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error } if (len(result.ProfileStatusList) == 0) { - fmt.Sprintf("No results found for iccid = '%s'", iccid) + fmt.Printf("No results found for iccid = '%s'", iccid) return nil, nil } else if (len(result.ProfileStatusList) == 1) { returnvalue := result.ProfileStatusList[0] return &returnvalue, nil } else { - return nil, errors.New("GetStatus returned more than one profile!") + return nil, errors.New("GetStatus returned more than one profile") } } @@ -381,10 +381,10 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownload } executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType - if ("Executed-Success" != executionStatus) { - return result, errors.New(fmt.Sprintf("ExecutionStatus was: ''%s'", executionStatus)) + if ( executionStatus == "Executed-Success") { + return result, nil } else { - return result, nil + return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) } } @@ -409,8 +409,8 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOr } executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType - if ("Executed-Success" != executionStatus) { - return result, errors.New(fmt.Sprintf("ExecutionStatus was: ''%s'", executionStatus)) + if (executionStatus != "Executed-Success") { + return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) } else { return result, nil } @@ -432,6 +432,9 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, return nil, err } result, err = client.GetStatus(iccid) + if err != nil { + return nil, err + } } if result.State == "ALLOCATED" { diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index e5d5a9338..34a0f2671 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -1,10 +1,7 @@ package outfileparser -// TODO: Rename to oufileparser - import ( "bufio" - "errors" "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" @@ -84,12 +81,12 @@ func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { varOutSplit := strings.Split(varOutLine, ":") if len(varOutSplit) != 2 { - return errors.New("syntax error in var_out line. More than two colon separated fields.") + return fmt.Errorf("syntax error in var_out line, more than two colon separated fields") } varOutToken := strings.TrimSpace(string(varOutSplit[0])) if strings.ToLower(varOutToken) != "var_out" { - return errors.New(fmt.Sprintf("syntax error in var_out line. Does not start with 'var_out', was '%s'", varOutToken)) + return fmt.Errorf("syntax error in var_out line. Does not start with 'var_out', was '%s'", varOutToken) } slashedFields := strings.Split(varOutSplit[1], "/") diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index ca66b6933..a7eb10578 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -28,12 +28,14 @@ var ( // Declare a profile-vendor with an SM-DP+ that can be referred to from // batches. Referential integrity required, so it won't be possible to // declare bathes with non-existing profile vendors. + /** TODO dpv = kingpin.Command("declare-profile-vendor", "Declare a profile vendor with an SM-DP+ we can talk to") dpvName = dpv.Flag("name", "Name of profile-vendor").Required().String() dpvCertFilePath = dpv.Flag("cert", "Certificate pem file.").Required().String() dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().String() + */ // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files // TODO: Declare hss-es, that can be refered to in profiles. @@ -96,10 +98,7 @@ var ( // TODO: Check if this can be used for the key files. // postImage = post.Flag("image", "image to post").ExistingFile() - // TODO ??? - batch = kingpin.Command("batch", "Utility for persisting and manipulating sim card batches.") - - listBatches = kingpin.Command("list-batches", "List all known batches.") + // TODO: listBatches = kingpin.Command("list-batches", "List all known batches.") describeBatch = kingpin.Command("describe-batch", "Describe a batch with a particular name.") describeBatchBatch = describeBatch.Arg("batch", "The batch to describe").String() @@ -530,7 +529,7 @@ func main() { batch, err := db.GetBatchByName(batchName) if err != nil { - fmt.Errorf("Unknown batch '%s'\n", batchName) + panic(fmt.Errorf("unknown batch '%s'", batchName)) } entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) @@ -598,7 +597,7 @@ func main() { batch, err := db.GetBatchByName(batchName) if err != nil { - fmt.Errorf("Unknown batch '%s'\n", batchName) + panic(fmt.Errorf("unknown batch '%s'", batchName)) } entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index ade415981..b1bfb3fdc 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -99,6 +99,7 @@ func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { var result model.Batch + result.BatchId = -1 return &result, sdb.Db.Get(&result, "select * from BATCH where name = ?", name) } @@ -159,7 +160,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { // "UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", - res, err = sdb.Db.NamedExec("UPDATE BATCH SET firstIccid = :firstIccid, firstImsi = :firstImsi, firstMsisdn = :firstMsisdn, msisdnIncrement = :msisdnIncrement, iccidIncrement = :iccidIncrement, imsiIncrement = :imsiIncrement, url=:url WHERE id = :id", + _, err = sdb.Db.NamedExec("UPDATE BATCH SET firstIccid = :firstIccid, firstImsi = :firstImsi, firstMsisdn = :firstMsisdn, msisdnIncrement = :msisdnIncrement, iccidIncrement = :iccidIncrement, imsiIncrement = :imsiIncrement, url=:url WHERE id = :id", theBatch) // , :firstIccid, :firstImsi, :firstMsisdn, :msisdnIncrement, :iccidIncrement, :imsiIncrement, :url @@ -219,7 +220,7 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { id, err := res.LastInsertId() if err != nil { - fmt.Errorf("Getting last inserted id failed '%s'", err) + return fmt.Errorf("getting last inserted id failed '%s'", err) } theEntry.Id = id return err diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index cc9aa15c4..c5e9d069f 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -31,7 +31,7 @@ func setup() { sdb, err = OpenFileSqliteDatabase(filename) // sdb, err = NewInMemoryDatabase() if err != nil { - fmt.Sprintf("Couldn't open new in memory database '%s", err) + panic(fmt.Errorf("Couldn't open new in memory database '%s", err)) } /* batches, err := sdb.GetAllBatches() @@ -83,7 +83,7 @@ func shutdown() { func TestMemoryDbPing(t *testing.T) { err = sdb.Db.Ping() if err != nil { - fmt.Errorf("Could not ping in-memory database. '%s'", err) + t.Errorf("Could not ping in-memory database. '%s'", err) } } @@ -102,8 +102,8 @@ func injectTestBatch() *model.Batch { } batch, _ := sdb.GetBatchByName(theBatch.Name) - if batch != nil { - fmt.Errorf("Duplicate batch detected %s", theBatch.Name) + if batch.BatchId != -1 { + panic(fmt.Errorf("Duplicate batch detected '%s'", theBatch.Name)) } err := sdb.CreateBatch(&theBatch) @@ -115,11 +115,17 @@ func injectTestBatch() *model.Batch { func TestGetBatchById(t *testing.T) { + cleanTables() + batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") + if batch.BatchId != -1 { + t.Errorf("Duplicate detected, error in test setup") + } + theBatch := injectTestBatch() firstInputBatch, _ := sdb.GetBatchById(theBatch.BatchId) if !reflect.DeepEqual(*firstInputBatch, *theBatch) { - fmt.Errorf("getBatchById failed") + t.Errorf("getBatchById failed") } } @@ -129,7 +135,7 @@ func TestGetAllBatches(t *testing.T) { allBatches, err := sdb.GetAllBatches() if err != nil { - fmt.Errorf("Reading query failed '%s'", err) + t.Errorf("Reading query failed '%s'", err) } assert.Equal(t, len(allBatches), 0) @@ -138,14 +144,14 @@ func TestGetAllBatches(t *testing.T) { allBatches, err = sdb.GetAllBatches() if err != nil { - fmt.Errorf("Reading query failed '%s'", err) + t.Errorf("Reading query failed '%s'", err) } assert.Equal(t, len(allBatches), 1) firstInputBatch := allBatches[0] if !reflect.DeepEqual(firstInputBatch, theBatch) { - fmt.Errorf("getBatchById failed") + t.Errorf("getBatchById failed, returned batch not equal to initial batch") } } From 20f38497c1b45173a971794e3b4111ecd1c7db13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 22:41:24 +0100 Subject: [PATCH 182/309] Remove printouts that are of little interest, fix type-o that caused deep equal to fail --- sim-administration/sim-batch-management/store/store.go | 5 ----- .../sim-batch-management/store/store_test.go | 7 +++---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index b1bfb3fdc..96ac72062 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -298,16 +298,11 @@ func (sdb SimBatchDB) DeclareBatch( // Check parameters for syntactic correctness and // semantic sanity. // - - fmt.Printf("Pre adding luhn ? %t. first='%s', last='%s'\n", addLuhn, firstIccid, lastIccid) - if addLuhn { firstIccid = fieldsyntaxchecks.AddLuhnChecksum(firstIccid) lastIccid = fieldsyntaxchecks.AddLuhnChecksum(lastIccid) } - fmt.Printf("Post adding luhn ? %t. first='%s',last='%s'\n", addLuhn, firstIccid, lastIccid) - fieldsyntaxchecks.CheckICCIDSyntax("first-rawIccid", firstIccid) fieldsyntaxchecks.CheckICCIDSyntax("last-rawIccid", lastIccid) fieldsyntaxchecks.CheckIMSISyntax("last-imsi", lastIMSI) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index c5e9d069f..49810d95d 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -65,9 +65,8 @@ func cleanTables() { panic(fmt.Sprintf("Couldn't delete BATCH '%s'", err)) } - simRows, _ := simProfileDeletionResult.RowsAffected() - batchRows, _ := batchDeleteResult.RowsAffected() - fmt.Printf("simProfileDeletionResult = %d, batchDeleteResult=%d\n", simRows, batchRows) + simProfileDeletionResult.RowsAffected() + batchDeleteResult.RowsAffected() } func shutdown() { @@ -150,7 +149,7 @@ func TestGetAllBatches(t *testing.T) { assert.Equal(t, len(allBatches), 1) firstInputBatch := allBatches[0] - if !reflect.DeepEqual(firstInputBatch, theBatch) { + if !reflect.DeepEqual(firstInputBatch, *theBatch) { t.Errorf("getBatchById failed, returned batch not equal to initial batch") } } From a7806444c47fb90b5ff90a60c02c89efe40b6ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 22:56:16 +0100 Subject: [PATCH 183/309] Remove dead code --- .../sim-batch-management/store/store_test.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 49810d95d..b9dde08d9 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -25,6 +25,7 @@ func setup() { // In memory database fails, so we try this gonzo method of getting // a fresh database filename := "bazunka.db" + // delete file, ignore any errors os.Remove(filename) @@ -33,16 +34,6 @@ func setup() { if err != nil { panic(fmt.Errorf("Couldn't open new in memory database '%s", err)) } - /* - batches, err := sdb.GetAllBatches() - if err == nil { - panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) - } - - if len(batches) != 0 { - panic(fmt.Sprintf("batches already registred, test misconfigured")) - } - */ err = sdb.GenerateTables() if err != nil { @@ -116,7 +107,7 @@ func TestGetBatchById(t *testing.T) { cleanTables() batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") - if batch.BatchId != -1 { + if batch.BatchId != -1 { t.Errorf("Duplicate detected, error in test setup") } @@ -158,7 +149,7 @@ func declareTestBatch(t *testing.T) *model.Batch { theBatch, err := sdb.DeclareBatch( "Name", - false, + false, "Customer", "8778fsda", // batch number "20200101", // date string From 6ab77d049123de173faddf562359610cfe4ae00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 22:58:39 +0100 Subject: [PATCH 184/309] Use reflection.DeepEqual instead of homegrown equality --- .../sim-batch-management/store/store_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index b9dde08d9..a7241ec9c 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -190,10 +190,6 @@ func TestDeclareBatch(t *testing.T) { } -func entryEqual(a *model.SimEntry, b *model.SimEntry) bool { - return ((a.ActivationCode == b.ActivationCode) && (a.BatchID == b.BatchID) && (a.RawIccid == b.RawIccid) && a.IccidWithChecksum == b.IccidWithChecksum && a.IccidWithoutChecksum == b.IccidWithoutChecksum && a.Iccid == b.Iccid && a.Imsi == b.Imsi && a.Msisdn == b.Msisdn && a.Ki == b.Ki) -} - func TestDeclareAndRetrieveSimEntries(t *testing.T) { cleanTables() theBatch := declareTestBatch(t) @@ -211,7 +207,7 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { ActivationCode: "8", } - // assert.Equal(t, 0, entry.Id) + sdb.CreateSimEntry(&entry) assert.Assert(t, entry.Id != 0) @@ -219,7 +215,8 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { if err != nil { t.Fatal(err) } - if !entryEqual(retrivedEntry, &entry) { + + if !reflect.DeepEqual(retrivedEntry, &entry) { t.Fatal("Retrieved and stored sim entry are different") } } From 83695cc8776f7ce625bbfe9be8760bc2c9a8a0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 23:01:03 +0100 Subject: [PATCH 185/309] Removing dead code, cleaning up a little --- .../sim-batch-management/store/store.go | 42 +------------------ 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 96ac72062..1be8fc831 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -105,40 +105,6 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { // TODO: mutex - /* - res, err := sdb.Db.Exec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", - // "INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, firstIccid, firstImsi, firstMsisdn, msisdnIncrement, iccidIncrement, imsiIncrement, url) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ", - theBatch.Name ,theBatch.FilenameBase,theBatch.OrderDate, theBatch.Customer,theBatch.ProfileType, - theBatch.BatchNo, - theBatch.Quantity, - theBatch.FirstIccid, - theBatch.FirstImsi, - theBatch.FirstMsisdn, - theBatch.MsisdnIncrement, - theBatch.IccidIncrement, - theBatch.ImsiIncrement, - theBatch.Url, - ) - */ - - // TODO: a) Report it as a real error (minimal reproducable) - // b) Insert the object, then add extra fields, do it in a transaction, and don't break - /** foo := `CREATE TABLE IF NOT EXISTS BATCH ( - id integer primary key autoincrement, - name VARCHAR NOT NULL UNIQUE, - filenameBase VARCHAR NOT NULL, - customer VARCHAR NOT NULL, - profileType VARCHAR NOT NULL, - orderDate VARCHAR NOT NULL, - batchNo VARCHAR NOT NULL, - quantity INTEGER NOT NULL, - - */ - /* - res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", - theBatch, - ) - */ res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", theBatch, @@ -146,23 +112,19 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { if err != nil { // XXX Should be error logging - fmt.Printf("Failed to insert new batch '%s'", err) - return err + return fmt.Errorf("failed to insert new batch '%s'", err) } id, err := res.LastInsertId() if err != nil { // XXX Should be error logging - fmt.Printf("Getting last inserted id failed '%s'", err) - return err + return fmt.Errorf("getting last inserted id failed '%s'", err) } theBatch.BatchId = id - // "UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", _, err = sdb.Db.NamedExec("UPDATE BATCH SET firstIccid = :firstIccid, firstImsi = :firstImsi, firstMsisdn = :firstMsisdn, msisdnIncrement = :msisdnIncrement, iccidIncrement = :iccidIncrement, imsiIncrement = :imsiIncrement, url=:url WHERE id = :id", theBatch) - // , :firstIccid, :firstImsi, :firstMsisdn, :msisdnIncrement, :iccidIncrement, :imsiIncrement, :url return err } From a2298dd045586214bcf71602761185294808df40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 12 Nov 2019 23:18:15 +0100 Subject: [PATCH 186/309] Opencoding sim-profile-upload functionality --- .../outfileparser/outfileparser.go | 35 +++---------------- .../sim-batch-management/sim-batch-mgt.go | 15 ++++++-- 2 files changed, 18 insertions(+), 32 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 34a0f2671..e18a9b1b6 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -21,24 +21,15 @@ const ( UNKNOWN_HEADER = "unknown" ) - type OutputFileRecord struct { Filename string InputVariables map[string]string HeaderDescription map[string]string Entries []model.SimEntry - // TODO: As it is today, the noOfEntries is just the number of Entries, - // but I may want to change that to be the declared number of Entries, - // and then later, dynamically, read in the individual Entries - // in a channel that is just piped to the goroutine that writes - // them to file, and fails if the number of declared Entries - // differs from the actual number of Entries. .... but that is - // for another day. - NoOfEntries int - OutputFileName string + NoOfEntries int + OutputFileName string } - /// /// Functions /// @@ -95,7 +86,7 @@ func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { } return nil } - +// Implement a state machine that parses an output file. func ParseOutputFile(filename string) OutputFileRecord { _, err := os.Stat(filename) @@ -136,7 +127,7 @@ func ParseOutputFile(filename string) OutputFileRecord { nextMode := modeFromSectionHeader(line) transitionMode(&state, nextMode) continue - }else if line == "OUTPUT VARIABLES" { + } else if line == "OUTPUT VARIABLES" { transitionMode(&state, OUTPUT_VARIABLES) continue } @@ -162,7 +153,7 @@ func ParseOutputFile(filename string) OutputFileRecord { if (len(state.csvFieldMap) != 0) { log.Fatal("Parsing multiple 'var_out' lines can't be right") } - err := ParseVarOutLine(line, &(state.csvFieldMap)) + err := ParseVarOutLine(line, &(state.csvFieldMap)) if err != nil { log.Fatalf("Couldn't parse output variable declaration '%s'\n", err) } @@ -327,19 +318,3 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { fmt.Println("Successfully written ", max, " sim card records.") return f.Close() } - -// -// Entrypoint -// TODO: Move this entrypoint into the the command processor -// - -func ConvertInputfileToOutputfile(inputFile string, outputFilePrefix string) { - outRecord := ParseOutputFile(inputFile) - outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" - fmt.Println("outputFile = ", outputFile) - - err := WriteHssCsvFile(outputFile, outRecord.Entries) - if err != nil { - log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") - } -} diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index a7eb10578..0d60e55b7 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -35,7 +35,7 @@ var ( dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().String() - */ + */ // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files // TODO: Declare hss-es, that can be refered to in profiles. @@ -159,7 +159,18 @@ func main() { cmd := kingpin.Parse() switch cmd { case "sim-profile-upload": - outfileparser.ConvertInputfileToOutputfile(*spUploadInputFile, *spUploadOutputFilePrefix) + + inputFile := *spUploadInputFile + outputFilePrefix := *spUploadOutputFilePrefix + + outRecord := outfileparser.ParseOutputFile(inputFile) + outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" + fmt.Println("outputFile = ", outputFile) + + err := outfileparser.WriteHssCsvFile(outputFile, outRecord.Entries) + if err != nil { + log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") + } case "declare-profile-vendor": fmt.Println("Declaration of profile-vendors not yet implemented.") From 612ffc09eb6e0a79370127597e610af43b15d9d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 00:18:05 +0100 Subject: [PATCH 187/309] Instead of panicing, return and handle error --- .../sim-batch-management/es2plus/es2plus.go | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 5464e0d37..2e313bd6a 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -239,14 +239,14 @@ func newUuid() (string, error) { } -func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader) { +func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { functionCallIdentifier, err := newUuid() if err != nil { - panic(err) + return nil, err } - return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()} + return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } @@ -321,12 +321,15 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - header := newEs2plusHeader(client) + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } payload := &ES2PlusGetProfileStatusRequest{ Header: *header, IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) if err != nil { return nil, err } @@ -348,7 +351,10 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error func(client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { result := new(ES2PlusRecoverProfileResponse) es2plusCommand := "recoverProfile" - header := newEs2plusHeader(client) + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } payload := &ES2PlusRecoverProfileRequest{ Header: *header, Iccid: iccid, @@ -361,7 +367,10 @@ func(client *Es2PlusClientState) RecoverProfile(iccid string, targetState strin func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { result := new(ES2PlusCancelOrderResponse) es2plusCommand := "cancelOrder" - header := newEs2plusHeader(client) + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } payload := &ES2PlusCancelOrderRequest { Header: *header, Iccid: iccid, @@ -374,14 +383,17 @@ func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { result := new(ES2PlusDownloadOrderResponse) es2plusCommand := "downloadOrder" - header := newEs2plusHeader(client) + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } payload := &ES2PlusDownloadOrderRequest { Header: *header, Iccid: iccid, Eid: "", Profiletype: "", } - err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) if err != nil { return nil, err } @@ -399,7 +411,10 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownload func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { result := new(ES2PlusConfirmOrderResponse) es2plusCommand := "confirmOrder" - header := newEs2plusHeader(client) + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } payload := &ES2PlusConfirmOrderRequest { Header: *header, Iccid: iccid, @@ -410,7 +425,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOr ReleaseFlag: true, } - err := marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) if err != nil { return nil, err } From 3b950ae3ea53297998acff096f96b5759dd674a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 00:19:45 +0100 Subject: [PATCH 188/309] Instead of panicing, return and handle error --- sim-administration/sim-batch-management/es2plus/es2plus.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 2e313bd6a..e54fe6164 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -282,7 +282,10 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma // byte array jsonStrB in a POST request. Set up the required // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) - req, _ := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + if err != nil { + return nil, err + } req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") From b7936916c1637fdefab2f69ccc7ee4efbdd6ec86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 00:26:26 +0100 Subject: [PATCH 189/309] Fix a couple of things suggested in review comments --- .../sim-batch-management/es2plus/es2plus.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index e54fe6164..4ac26520d 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -9,7 +9,6 @@ import ( "log" "net/http" "strings" - "errors" "github.com/google/uuid" ) @@ -295,6 +294,7 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma } resp, err := httpClient.Do(req) + defer resp.Body.Close() // On http protocol failure return quickly if err != nil { @@ -338,14 +338,12 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error } if (len(result.ProfileStatusList) == 0) { - fmt.Printf("No results found for iccid = '%s'", iccid) return nil, nil } else if (len(result.ProfileStatusList) == 1) { - returnvalue := result.ProfileStatusList[0] - return &returnvalue, nil + return &result.ProfileStatusList[0], nil } else { - return nil, errors.New("GetStatus returned more than one profile") + return nil, fmt.Errorf("GetStatus returned more than one profile") } } From 61b966a3ac00af6136563da468206848cc286f7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 00:28:44 +0100 Subject: [PATCH 190/309] Whitespace --- .../sim-batch-management/es2plus/es2plus.go | 462 +++++++++--------- 1 file changed, 218 insertions(+), 244 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 4ac26520d..cd8ced5c3 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -5,27 +5,26 @@ import ( "crypto/tls" "encoding/json" "fmt" + "github.com/google/uuid" "io/ioutil" "log" "net/http" "strings" - "github.com/google/uuid" ) /// /// External interface /// type Es2PlusClient interface { - GetStatus(iccid string) (*ProfileStatus, error) - RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) - CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) - DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) - ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) - ActivateIccid(iccid string) (*ProfileStatus, error) - RequesterId() (string) + GetStatus(iccid string) (*ProfileStatus, error) + RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) + CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) + DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) + ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) + ActivateIccid(iccid string) (*ProfileStatus, error) + RequesterId() string } - /// /// Generic headers for invocations and responses /// @@ -49,13 +48,10 @@ type FunctionExecutionStatus struct { StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` } - type ES2PlusResponseHeader struct { FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` } - - // // Status code invocation. // @@ -82,35 +78,32 @@ type ProfileStatus struct { LockFlag bool `json:"lockFlag"` } - // // Profile reset invocation // type ES2PlusRecoverProfileRequest struct { - Header ES2PlusHeader `json:"header"` - Iccid string `json:"iccid"` - ProfileStatus string `json:"profileStatus"` - } + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + ProfileStatus string `json:"profileStatus"` +} type ES2PlusRecoverProfileResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header ES2PlusResponseHeader `json:"header"` } - - // // Cancel order invocation // type ES2PlusCancelOrderRequest struct { - Header ES2PlusHeader `json:"header"` - Iccid string `json:"iccid"` - FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` - } + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` +} type ES2PlusCancelOrderResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header ES2PlusResponseHeader `json:"header"` } // @@ -118,15 +111,15 @@ type ES2PlusCancelOrderResponse struct { // type ES2PlusDownloadOrderRequest struct { - Header ES2PlusHeader `json:"header"` - Iccid string `json:"iccid"` - Eid string `json:"eid,omitempty"` - Profiletype string `json:"profiletype,omitempty"` + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + Profiletype string `json:"profiletype,omitempty"` } type ES2PlusDownloadOrderResponse struct { - Header ES2PlusResponseHeader `json:"header"` - Iccid string `json:"iccid"` + Header ES2PlusResponseHeader `json:"header"` + Iccid string `json:"iccid"` } // @@ -134,46 +127,43 @@ type ES2PlusDownloadOrderResponse struct { // type ES2PlusConfirmOrderRequest struct { - Header ES2PlusHeader `json:"header"` - Iccid string `json:"iccid"` - Eid string `json:"eid,omitempty"` - MatchingId string `json:"matchingId,omitempty"` - ConfirmationCode string `json:"confirmationCode,omitempty"` - SmdpAddress string `json:"smdpAddress,omitempty"` - ReleaseFlag bool `json:"releaseFlag"` + Header ES2PlusHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + MatchingId string `json:"matchingId,omitempty"` + ConfirmationCode string `json:"confirmationCode,omitempty"` + SmdpAddress string `json:"smdpAddress,omitempty"` + ReleaseFlag bool `json:"releaseFlag"` } type ES2PlusConfirmOrderResponse struct { - Header ES2PlusResponseHeader `json:"header"` - Iccid string `json:"iccid"` - Eid string `json:"eid,omitempty"` - MatchingId string `json:"matchingId,omitempty"` - SmdpAddress string `json:"smdpAddress,omitempty"` + Header ES2PlusResponseHeader `json:"header"` + Iccid string `json:"iccid"` + Eid string `json:"eid,omitempty"` + MatchingId string `json:"matchingId,omitempty"` + SmdpAddress string `json:"smdpAddress,omitempty"` } - // // Generating new ES2Plus clients // - type Es2PlusClientState struct { - httpClient *http.Client - hostport string - requesterId string + httpClient *http.Client + hostport string + requesterId string printPayload bool printHeaders bool } - -func Client (certFilePath string, keyFilePath string, hostport string, requesterId string) (*Es2PlusClientState) { - return &Es2PlusClientState { - httpClient: newHttpClient(certFilePath, keyFilePath), - hostport: hostport, - requesterId: requesterId, - printPayload: false, - printHeaders: false, - } +func Client(certFilePath string, keyFilePath string, hostport string, requesterId string) *Es2PlusClientState { + return &Es2PlusClientState{ + httpClient: newHttpClient(certFilePath, keyFilePath), + hostport: hostport, + requesterId: requesterId, + printPayload: false, + printHeaders: false, + } } func newHttpClient(certFilePath string, keyFilePath string) *http.Client { @@ -192,12 +182,10 @@ func newHttpClient(certFilePath string, keyFilePath string) *http.Client { return client } - /// /// Generic protocol code /// - // // Function used during debugging to print requests before they are // sent over the wire. Very useful, should not be deleted even though it @@ -230,73 +218,69 @@ func formatRequest(r *http.Request) string { } func newUuid() (string, error) { - uuid, err := uuid.NewRandom() - if err != nil { - return "", err - } - return uuid.URN(), nil + uuid, err := uuid.NewRandom() + if err != nil { + return "", err + } + return uuid.URN(), nil } - func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { - functionCallIdentifier, err := newUuid() - if err != nil { - return nil, err - } + functionCallIdentifier, err := newUuid() + if err != nil { + return nil, err + } - return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil + return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } +func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClientState, es2plusCommand string, payload interface{}, result interface{}) error { -func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClientState, es2plusCommand string, payload interface{}, result interface{}) error { - - // Serialize payload as json. + // Serialize payload as json. jsonStrB, err := json.Marshal(payload) if err != nil { - return err + return err } - if client.printPayload { - fmt.Print("Payload ->", string(jsonStrB)) - } + if client.printPayload { + fmt.Print("Payload ->", string(jsonStrB)) + } - // Get the result of the HTTP POST as a byte array - // that can be deserialized into json. Fail fast - // an error has been detected. - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient, client.printHeaders) + // Get the result of the HTTP POST as a byte array + // that can be deserialized into json. Fail fast + // an error has been detected. + responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient, client.printHeaders) if err != nil { - return err + return err } - // Return error code from deserialisation, result is put into - // result via referenced object. + // Return error code from deserialisation, result is put into + // result via referenced object. return json.Unmarshal(responseBytes, result) } - func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client, printHeaders bool) ([]byte, error) { - // Build and execute an ES2+ protocol request by using the serialised JSON in the - // byte array jsonStrB in a POST request. Set up the required - // headers for ES2+ and content type. + // Build and execute an ES2+ protocol request by using the serialised JSON in the + // byte array jsonStrB in a POST request. Set up the required + // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) if err != nil { - return nil, err + return nil, err } req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") - - if printHeaders { - fmt.Printf("Request -> %s\n", formatRequest(req)) + if printHeaders { + fmt.Printf("Request -> %s\n", formatRequest(req)) } resp, err := httpClient.Do(req) defer resp.Body.Close() - // On http protocol failure return quickly + // On http protocol failure return quickly if err != nil { return nil, err } @@ -304,7 +288,6 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma // TODO Should check response headers here! // (in particular X-admin-protocol) and fail if not OK. - // Get payload bytes from response body and return them. // Return an error if the bytes can't be retrieved. response, err := ioutil.ReadAll(resp.Body) @@ -315,165 +298,156 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma return responseBytes, nil } - /// /// Externally visible API for Es2Plus protocol /// - func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error) { - result := new(ES2ProfileStatusResponse) - es2plusCommand := "getProfileStatus" - header, err := newEs2plusHeader(client) - if err != nil { - return nil, err - } - payload := &ES2PlusGetProfileStatusRequest{ - Header: *header, - IccidList: [] ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, - } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { - return nil, err - } - - if (len(result.ProfileStatusList) == 0) { - return nil, nil - } else if (len(result.ProfileStatusList) == 1) { - return &result.ProfileStatusList[0], nil - } else { - - return nil, fmt.Errorf("GetStatus returned more than one profile") - } -} - - - -func(client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { - result := new(ES2PlusRecoverProfileResponse) - es2plusCommand := "recoverProfile" - header, err := newEs2plusHeader(client) - if err != nil { - return nil, err - } - payload := &ES2PlusRecoverProfileRequest{ - Header: *header, - Iccid: iccid, - ProfileStatus: targetState, - } - return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) -} - - -func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { - result := new(ES2PlusCancelOrderResponse) - es2plusCommand := "cancelOrder" - header, err := newEs2plusHeader(client) - if err != nil { - return nil, err - } - payload := &ES2PlusCancelOrderRequest { - Header: *header, - Iccid: iccid, - FinalProfileStatusIndicator: targetState, - } - return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) -} - - -func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { - result := new(ES2PlusDownloadOrderResponse) - es2plusCommand := "downloadOrder" - header, err := newEs2plusHeader(client) - if err != nil { - return nil, err - } - payload := &ES2PlusDownloadOrderRequest { - Header: *header, - Iccid: iccid, - Eid: "", - Profiletype: "", - } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { - return nil, err - } - - executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType - if ( executionStatus == "Executed-Success") { - return result, nil - } else { - return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) - } -} - - - -func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { - result := new(ES2PlusConfirmOrderResponse) - es2plusCommand := "confirmOrder" - header, err := newEs2plusHeader(client) - if err != nil { - return nil, err - } - payload := &ES2PlusConfirmOrderRequest { - Header: *header, - Iccid: iccid, - Eid: "", - ConfirmationCode: "", - MatchingId: "", - SmdpAddress: "", - ReleaseFlag: true, - } - - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { - return nil, err - } - - executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType - if (executionStatus != "Executed-Success") { - return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) - } else { - return result, nil - } -} - -func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error){ - - result, err := client.GetStatus(iccid) + result := new(ES2ProfileStatusResponse) + es2plusCommand := "getProfileStatus" + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } + payload := &ES2PlusGetProfileStatusRequest{ + Header: *header, + IccidList: []ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + } + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + if len(result.ProfileStatusList) == 0 { + return nil, nil + } else if len(result.ProfileStatusList) == 1 { + return &result.ProfileStatusList[0], nil + } else { + + return nil, fmt.Errorf("GetStatus returned more than one profile") + } +} + +func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { + result := new(ES2PlusRecoverProfileResponse) + es2plusCommand := "recoverProfile" + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } + payload := &ES2PlusRecoverProfileRequest{ + Header: *header, + Iccid: iccid, + ProfileStatus: targetState, + } + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) +} +func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { + result := new(ES2PlusCancelOrderResponse) + es2plusCommand := "cancelOrder" + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } + payload := &ES2PlusCancelOrderRequest{ + Header: *header, + Iccid: iccid, + FinalProfileStatusIndicator: targetState, + } + return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) +} + +func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { + result := new(ES2PlusDownloadOrderResponse) + es2plusCommand := "downloadOrder" + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } + payload := &ES2PlusDownloadOrderRequest{ + Header: *header, + Iccid: iccid, + Eid: "", + Profiletype: "", + } + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + if executionStatus == "Executed-Success" { + return result, nil + } else { + return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) + } +} + +func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { + result := new(ES2PlusConfirmOrderResponse) + es2plusCommand := "confirmOrder" + header, err := newEs2plusHeader(client) + if err != nil { + return nil, err + } + payload := &ES2PlusConfirmOrderRequest{ + Header: *header, + Iccid: iccid, + Eid: "", + ConfirmationCode: "", + MatchingId: "", + SmdpAddress: "", + ReleaseFlag: true, + } + + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { + return nil, err + } + + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + if executionStatus != "Executed-Success" { + return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) + } else { + return result, nil + } +} + +func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { + + result, err := client.GetStatus(iccid) + + if err != nil { + panic(err) + } + + if result.ACToken == "" { + + if result.State == "AVAILABLE" { + _, err := client.DownloadOrder(iccid) if err != nil { - panic(err) + return nil, err } - - if result.ACToken == "" { - - if result.State == "AVAILABLE" { - _, err := client.DownloadOrder(iccid) - if err != nil { - return nil, err - } - result, err = client.GetStatus(iccid) - if err != nil { - return nil, err - } - } - - if result.State == "ALLOCATED" { - _, err = client.ConfirmOrder(iccid) - if err != nil { - return nil, err - } - } - - } result, err = client.GetStatus(iccid) - return result, err + if err != nil { + return nil, err + } + } + + if result.State == "ALLOCATED" { + _, err = client.ConfirmOrder(iccid) + if err != nil { + return nil, err + } + } + + } + result, err = client.GetStatus(iccid) + return result, err } // TODO: This shouldn't have to be public, but how can it be avoided? -func (clientState *Es2PlusClientState) RequesterId() (string){ - return clientState.requesterId - } - +func (clientState *Es2PlusClientState) RequesterId() string { + return clientState.requesterId +} From 69512f7aa3ae4582104d932e8d85d184fe555236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 00:38:14 +0100 Subject: [PATCH 191/309] explain why we're running unsafe. --- sim-administration/sim-batch-management/es2plus/es2plus.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index cd8ced5c3..f293d5a7f 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -173,6 +173,10 @@ func newHttpClient(certFilePath string, keyFilePath string) *http.Client { if err != nil { log.Fatalf("server: loadkeys: %s", err) } + + // TODO: The certificate used to sign the other end of the TLS connection + // is privately signed, and at this time we don't require the full + // certificate chain to be available. config := tls.Config{Certificates: []tls.Certificate{cert}, InsecureSkipVerify: true} client := &http.Client{ Transport: &http.Transport{ From 829ef6a6001a408b2846b6afe757175152e43c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 08:32:45 +0100 Subject: [PATCH 192/309] Refactor to use logger instead of direct print to stdout --- .../sim-batch-management/es2plus/es2plus.go | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index f293d5a7f..21b99519b 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -152,8 +152,8 @@ type Es2PlusClientState struct { httpClient *http.Client hostport string requesterId string - printPayload bool - printHeaders bool + logPayload bool + logHeaders bool } func Client(certFilePath string, keyFilePath string, hostport string, requesterId string) *Es2PlusClientState { @@ -161,8 +161,8 @@ func Client(certFilePath string, keyFilePath string, hostport string, requesterI httpClient: newHttpClient(certFilePath, keyFilePath), hostport: hostport, requesterId: requesterId, - printPayload: false, - printHeaders: false, + logPayload: false, + logHeaders: false, } } @@ -239,7 +239,10 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } -func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClientState, es2plusCommand string, payload interface{}, result interface{}) error { +func marshalUnmarshalGenericEs2plusCommand( + client *Es2PlusClientState, + es2plusCommand string, + payload interface{}, result interface{}) error { // Serialize payload as json. jsonStrB, err := json.Marshal(payload) @@ -247,14 +250,19 @@ func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClientState, es2plusCo return err } - if client.printPayload { - fmt.Print("Payload ->", string(jsonStrB)) + if client.logPayload { + log.Print("Payload ->", string(jsonStrB)) } // Get the result of the HTTP POST as a byte array // that can be deserialized into json. Fail fast // an error has been detected. - responseBytes, err := executeGenericEs2plusCommand(jsonStrB, client.hostport, es2plusCommand, client.httpClient, client.printHeaders) + responseBytes, err := executeGenericEs2plusCommand( + jsonStrB, + client.hostport, + es2plusCommand, + client.httpClient, + client.logHeaders) if err != nil { return err } @@ -264,7 +272,7 @@ func marshalUnmarshalGenericEs2plusCommand(client *Es2PlusClientState, es2plusCo return json.Unmarshal(responseBytes, result) } -func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client, printHeaders bool) ([]byte, error) { +func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) ([]byte, error) { // Build and execute an ES2+ protocol request by using the serialised JSON in the // byte array jsonStrB in a POST request. Set up the required @@ -277,8 +285,8 @@ func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusComma req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") - if printHeaders { - fmt.Printf("Request -> %s\n", formatRequest(req)) + if logHeaders { + log.Printf("Request -> %s\n", formatRequest(req)) } resp, err := httpClient.Do(req) From 2fd9fd62f215240410d71e7715cdb854fb40d77d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 08:37:15 +0100 Subject: [PATCH 193/309] whitespace --- sim-administration/sim-batch-management/es2plus/es2plus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 21b99519b..65cca9230 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -335,7 +335,6 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error } else if len(result.ProfileStatusList) == 1 { return &result.ProfileStatusList[0], nil } else { - return nil, fmt.Errorf("GetStatus returned more than one profile") } } From 0602e496d531996b4c90eb1b32c14a3294121ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 08:48:20 +0100 Subject: [PATCH 194/309] Intermediate stage, working json encoding, but but not consistently implemented --- .../sim-batch-management/es2plus/es2plus.go | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 65cca9230..e757f1f4b 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -239,7 +239,8 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } -func marshalUnmarshalGenericEs2plusCommand( +/* +func originalMarshalUnmarshalGenericEs2plusCommand( client *Es2PlusClientState, es2plusCommand string, payload interface{}, result interface{}) error { @@ -271,14 +272,50 @@ func marshalUnmarshalGenericEs2plusCommand( // result via referenced object. return json.Unmarshal(responseBytes, result) } +*/ + +func marshalUnmarshalGenericEs2plusCommand( + client *Es2PlusClientState, + es2plusCommand string, + payload interface{}, result interface{}) error { + + // Serialize payload as json. + jsonStrB := new(bytes.Buffer) + err := json.NewEncoder(jsonStrB).Encode(payload) + + if err != nil { + return err + } + + if client.logPayload { + log.Print("Payload ->", jsonStrB.String()) + } + + // Get the result of the HTTP POST as a byte array + // that can be deserialized into json. Fail fast + // an error has been detected. + responseBytes, err := executeGenericEs2plusCommand( + jsonStrB, + client.hostport, + es2plusCommand, + client.httpClient, + client.logHeaders) + if err != nil { + return err + } + + // Return error code from deserialisation, result is put into + // result via referenced object. + return json.Unmarshal(responseBytes, result) +} -func executeGenericEs2plusCommand(jsonStrB []byte, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) ([]byte, error) { +func executeGenericEs2plusCommand(jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) ([]byte, error) { // Build and execute an ES2+ protocol request by using the serialised JSON in the // byte array jsonStrB in a POST request. Set up the required // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStrB)) + req, err := http.NewRequest("POST", url, jsonStrB) if err != nil { return nil, err } From 601da1758552061e0d920ce969f4d0766c0df0a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 08:59:32 +0100 Subject: [PATCH 195/309] Functional, not esthetic, refactoring of marshalling into more modern json serialization/deseralisation --- .../sim-batch-management/es2plus/es2plus.go | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index e757f1f4b..12f1fd351 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "github.com/google/uuid" - "io/ioutil" "log" "net/http" "strings" @@ -294,22 +293,34 @@ func marshalUnmarshalGenericEs2plusCommand( // Get the result of the HTTP POST as a byte array // that can be deserialized into json. Fail fast // an error has been detected. - responseBytes, err := executeGenericEs2plusCommand( + return executeGenericEs2plusCommand( + result, jsonStrB, client.hostport, es2plusCommand, client.httpClient, client.logHeaders) - if err != nil { - return err - } + +/* +u := User{Id: "US123", Balance: 8} + b := new(bytes.Buffer) + json.NewEncoder(b).Encode(u) + res, _ := http.Post("https://httpbin.org/post", "application/json; charset=utf-8", b) + var body struct { + // httpbin.org sends back key/value pairs, no map[string][]string + Headers map[string]string `json:"headers"` + Origin string `json:"origin"` + } + json.NewDecoder(res.Body).Decode(&body) + fmt.Println(body) + */ // Return error code from deserialisation, result is put into // result via referenced object. - return json.Unmarshal(responseBytes, result) + // return json.Unmarshal(responseBytes, result) } -func executeGenericEs2plusCommand(jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) ([]byte, error) { +func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) (error) { // Build and execute an ES2+ protocol request by using the serialised JSON in the // byte array jsonStrB in a POST request. Set up the required @@ -317,7 +328,7 @@ func executeGenericEs2plusCommand(jsonStrB *bytes.Buffer, hostport string, es2pl url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) req, err := http.NewRequest("POST", url, jsonStrB) if err != nil { - return nil, err + return err } req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") @@ -331,20 +342,13 @@ func executeGenericEs2plusCommand(jsonStrB *bytes.Buffer, hostport string, es2pl // On http protocol failure return quickly if err != nil { - return nil, err + return err } // TODO Should check response headers here! // (in particular X-admin-protocol) and fail if not OK. - // Get payload bytes from response body and return them. - // Return an error if the bytes can't be retrieved. - response, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - responseBytes := []byte(response) - return responseBytes, nil + return json.NewDecoder(resp.Body).Decode(&result) } /// From e516149802db55a8ac70ece805a5ec60d11e6955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 09:00:26 +0100 Subject: [PATCH 196/309] Nuking dead code, increasing esthetic pleasure. --- .../sim-batch-management/es2plus/es2plus.go | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 12f1fd351..e5671367e 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -238,40 +238,6 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } -/* -func originalMarshalUnmarshalGenericEs2plusCommand( - client *Es2PlusClientState, - es2plusCommand string, - payload interface{}, result interface{}) error { - - // Serialize payload as json. - jsonStrB, err := json.Marshal(payload) - if err != nil { - return err - } - - if client.logPayload { - log.Print("Payload ->", string(jsonStrB)) - } - - // Get the result of the HTTP POST as a byte array - // that can be deserialized into json. Fail fast - // an error has been detected. - responseBytes, err := executeGenericEs2plusCommand( - jsonStrB, - client.hostport, - es2plusCommand, - client.httpClient, - client.logHeaders) - if err != nil { - return err - } - - // Return error code from deserialisation, result is put into - // result via referenced object. - return json.Unmarshal(responseBytes, result) -} -*/ func marshalUnmarshalGenericEs2plusCommand( client *Es2PlusClientState, @@ -300,24 +266,6 @@ func marshalUnmarshalGenericEs2plusCommand( es2plusCommand, client.httpClient, client.logHeaders) - - -/* -u := User{Id: "US123", Balance: 8} - b := new(bytes.Buffer) - json.NewEncoder(b).Encode(u) - res, _ := http.Post("https://httpbin.org/post", "application/json; charset=utf-8", b) - var body struct { - // httpbin.org sends back key/value pairs, no map[string][]string - Headers map[string]string `json:"headers"` - Origin string `json:"origin"` - } - json.NewDecoder(res.Body).Decode(&body) - fmt.Println(body) - */ - // Return error code from deserialisation, result is put into - // result via referenced object. - // return json.Unmarshal(responseBytes, result) } func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) (error) { From 2a0162ec532114303fdd095eb7d69e5779acba61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 09:54:08 +0100 Subject: [PATCH 197/309] whitespace --- sim-administration/sim-batch-management/es2plus/es2plus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index e5671367e..6af02f372 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -441,7 +441,6 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, e return nil, err } } - } result, err = client.GetStatus(iccid) return result, err From 1b4bc18b1f9c662203deaf5f9ba530d9b4459026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 21:37:30 +0100 Subject: [PATCH 198/309] Compressing if err ... --- .../sim-batch-management/es2plus/es2plus.go | 36 ++++++++----------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 6af02f372..09765320f 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -168,7 +168,7 @@ func Client(certFilePath string, keyFilePath string, hostport string, requesterI func newHttpClient(certFilePath string, keyFilePath string) *http.Client { cert, err := tls.LoadX509KeyPair( certFilePath, - keyFilePath) + keyFilePath); if err != nil { log.Fatalf("server: loadkeys: %s", err) } @@ -221,7 +221,7 @@ func formatRequest(r *http.Request) string { } func newUuid() (string, error) { - uuid, err := uuid.NewRandom() + uuid, err := uuid.NewRandom(); if err != nil { return "", err } @@ -274,7 +274,7 @@ func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, h // byte array jsonStrB in a POST request. Set up the required // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) - req, err := http.NewRequest("POST", url, jsonStrB) + req, err := http.NewRequest("POST", url, jsonStrB); if err != nil { return err } @@ -306,7 +306,7 @@ func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, h func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - header, err := newEs2plusHeader(client) + header, err := newEs2plusHeader(client); if err != nil { return nil, err } @@ -314,8 +314,7 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error Header: *header, IccidList: []ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { + if err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); err != nil { return nil, err } @@ -331,7 +330,7 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { result := new(ES2PlusRecoverProfileResponse) es2plusCommand := "recoverProfile" - header, err := newEs2plusHeader(client) + header, err := newEs2plusHeader(client); if err != nil { return nil, err } @@ -346,7 +345,7 @@ func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState strin func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { result := new(ES2PlusCancelOrderResponse) es2plusCommand := "cancelOrder" - header, err := newEs2plusHeader(client) + header, err := newEs2plusHeader(client); if err != nil { return nil, err } @@ -361,7 +360,7 @@ func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { result := new(ES2PlusDownloadOrderResponse) es2plusCommand := "downloadOrder" - header, err := newEs2plusHeader(client) + header, err := newEs2plusHeader(client); if err != nil { return nil, err } @@ -371,8 +370,8 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadO Eid: "", Profiletype: "", } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); + if err != nil { return nil, err } @@ -401,8 +400,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd ReleaseFlag: true, } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) - if err != nil { + if err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); err != nil { return nil, err } @@ -416,8 +414,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { - result, err := client.GetStatus(iccid) - + result, err := client.GetStatus(iccid); if err != nil { panic(err) } @@ -425,19 +422,16 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, e if result.ACToken == "" { if result.State == "AVAILABLE" { - _, err := client.DownloadOrder(iccid) - if err != nil { + if _, err := client.DownloadOrder(iccid); err != nil { return nil, err } - result, err = client.GetStatus(iccid) - if err != nil { + if result, err = client.GetStatus(iccid); err != nil { return nil, err } } if result.State == "ALLOCATED" { - _, err = client.ConfirmOrder(iccid) - if err != nil { + if _, err = client.ConfirmOrder(iccid) ; err != nil { return nil, err } } From 332d7ffe228389786714afba671c46aab4906d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 22:40:06 +0100 Subject: [PATCH 199/309] Prettier error handling --- .../sim-batch-management/es2plus/es2plus.go | 77 +++++++++---------- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 3 +- .../outfileparser/outfileparser.go | 24 +++--- 3 files changed, 49 insertions(+), 55 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 09765320f..2e180bde5 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -148,27 +148,27 @@ type ES2PlusConfirmOrderResponse struct { // type Es2PlusClientState struct { - httpClient *http.Client - hostport string - requesterId string - logPayload bool - logHeaders bool + httpClient *http.Client + hostport string + requesterId string + logPayload bool + logHeaders bool } func Client(certFilePath string, keyFilePath string, hostport string, requesterId string) *Es2PlusClientState { return &Es2PlusClientState{ - httpClient: newHttpClient(certFilePath, keyFilePath), - hostport: hostport, - requesterId: requesterId, - logPayload: false, - logHeaders: false, + httpClient: newHttpClient(certFilePath, keyFilePath), + hostport: hostport, + requesterId: requesterId, + logPayload: false, + logHeaders: false, } } func newHttpClient(certFilePath string, keyFilePath string) *http.Client { cert, err := tls.LoadX509KeyPair( certFilePath, - keyFilePath); + keyFilePath) if err != nil { log.Fatalf("server: loadkeys: %s", err) } @@ -221,7 +221,7 @@ func formatRequest(r *http.Request) string { } func newUuid() (string, error) { - uuid, err := uuid.NewRandom(); + uuid, err := uuid.NewRandom() if err != nil { return "", err } @@ -238,15 +238,14 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } - func marshalUnmarshalGenericEs2plusCommand( - client *Es2PlusClientState, - es2plusCommand string, - payload interface{}, result interface{}) error { + client *Es2PlusClientState, + es2plusCommand string, + payload interface{}, result interface{}) error { // Serialize payload as json. - jsonStrB := new(bytes.Buffer) - err := json.NewEncoder(jsonStrB).Encode(payload) + jsonStrB := new(bytes.Buffer) + err := json.NewEncoder(jsonStrB).Encode(payload) if err != nil { return err @@ -260,23 +259,23 @@ func marshalUnmarshalGenericEs2plusCommand( // that can be deserialized into json. Fail fast // an error has been detected. return executeGenericEs2plusCommand( - result, - jsonStrB, - client.hostport, - es2plusCommand, - client.httpClient, - client.logHeaders) + result, + jsonStrB, + client.hostport, + es2plusCommand, + client.httpClient, + client.logHeaders) } -func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) (error) { +func executeGenericEs2plusCommand(result interface{}, jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) error { // Build and execute an ES2+ protocol request by using the serialised JSON in the // byte array jsonStrB in a POST request. Set up the required // headers for ES2+ and content type. url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) - req, err := http.NewRequest("POST", url, jsonStrB); + req, err := http.NewRequest("POST", url, jsonStrB) if err != nil { - return err + return err } req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") @@ -286,12 +285,10 @@ func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, h } resp, err := httpClient.Do(req) - defer resp.Body.Close() - - // On http protocol failure return quickly if err != nil { return err } + defer resp.Body.Close() // TODO Should check response headers here! // (in particular X-admin-protocol) and fail if not OK. @@ -306,7 +303,7 @@ func executeGenericEs2plusCommand(result interface {}, jsonStrB *bytes.Buffer, h func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - header, err := newEs2plusHeader(client); + header, err := newEs2plusHeader(client) if err != nil { return nil, err } @@ -330,7 +327,7 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { result := new(ES2PlusRecoverProfileResponse) es2plusCommand := "recoverProfile" - header, err := newEs2plusHeader(client); + header, err := newEs2plusHeader(client) if err != nil { return nil, err } @@ -345,7 +342,7 @@ func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState strin func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { result := new(ES2PlusCancelOrderResponse) es2plusCommand := "cancelOrder" - header, err := newEs2plusHeader(client); + header, err := newEs2plusHeader(client) if err != nil { return nil, err } @@ -360,7 +357,7 @@ func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { result := new(ES2PlusDownloadOrderResponse) es2plusCommand := "downloadOrder" - header, err := newEs2plusHeader(client); + header, err := newEs2plusHeader(client) if err != nil { return nil, err } @@ -370,8 +367,8 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadO Eid: "", Profiletype: "", } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); - if err != nil { + err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + if err != nil { return nil, err } @@ -400,7 +397,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd ReleaseFlag: true, } - if err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); err != nil { + if err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); err != nil { return nil, err } @@ -414,7 +411,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { - result, err := client.GetStatus(iccid); + result, err := client.GetStatus(iccid) if err != nil { panic(err) } @@ -425,13 +422,13 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, e if _, err := client.DownloadOrder(iccid); err != nil { return nil, err } - if result, err = client.GetStatus(iccid); err != nil { + if result, err = client.GetStatus(iccid); err != nil { return nil, err } } if result.State == "ALLOCATED" { - if _, err = client.ConfirmOrder(iccid) ; err != nil { + if _, err = client.ConfirmOrder(iccid); err != nil { return nil, err } } diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go index d7fce64e8..ba3b25e8c 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -93,8 +93,7 @@ func CheckMSISDNSyntax(name string, potentialMSISDN string) { } func CheckURLSyntax(name string, theUrl string) { - _, err := url.ParseRequestURI(theUrl) - if err != nil { + if _, err := url.ParseRequestURI(theUrl); err != nil { log.Fatalf("Not a valid %s URL: '%s'.", name, theUrl) } } diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index e18a9b1b6..a6592b235 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -89,13 +89,12 @@ func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { // Implement a state machine that parses an output file. func ParseOutputFile(filename string) OutputFileRecord { - _, err := os.Stat(filename) - - if os.IsNotExist(err) { - log.Fatalf("Couldn't find file '%s'\n", filename) - } - if err != nil { - log.Fatalf("Couldn't stat file '%s'\n", filename) + if _, err := os.Stat(filename) ; err != nil { + if os.IsNotExist(err) { + log.Fatalf("Couldn't find file '%s'\n", filename) + } else { + log.Fatalf("Couldn't stat file '%s'\n", filename) + } } file, err := os.Open(filename) // For read access. @@ -153,8 +152,7 @@ func ParseOutputFile(filename string) OutputFileRecord { if (len(state.csvFieldMap) != 0) { log.Fatal("Parsing multiple 'var_out' lines can't be right") } - err := ParseVarOutLine(line, &(state.csvFieldMap)) - if err != nil { + if err := ParseVarOutLine(line, &(state.csvFieldMap)); err != nil { log.Fatalf("Couldn't parse output variable declaration '%s'\n", err) } continue @@ -293,17 +291,17 @@ func fileExists(filename string) bool { func WriteHssCsvFile(filename string, entries []model.SimEntry) error { if fileExists(filename) { - log.Fatal("Output file already exists. '", filename, "'.") + return fmt.Errorf("output file already exists. '%s'", filename) } f, err := os.Create(filename) if err != nil { - log.Fatal("Couldn't create hss csv file '", filename, "': ", err) + return fmt.Errorf("couldn't create hss csv file '%s', %v", filename, err) } _, err = f.WriteString("ICCID, IMSI, KI\n") if err != nil { - log.Fatal("Couldn't header to hss csv file '", filename, "': ", err) + return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) } max := 0 @@ -311,7 +309,7 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { s := fmt.Sprintf("%s, %s, %s\n", entry.IccidWithChecksum, entry.Imsi, entry.Ki) _, err = f.WriteString(s) if err != nil { - log.Fatal("Couldn't write to hss csv file '", filename, "': ", err) + return fmt.Errorf("couldn't write to hss csv file '%s', %v", filename, err) } max = i + 1 } From 19e483b9b8ee2f39db3854d5c56da62ed4e6e3cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 22:40:28 +0100 Subject: [PATCH 200/309] Whitespacre --- sim-administration/sim-batch-management/es2plus/es2plus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 2e180bde5..f94f9ff39 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -27,7 +27,6 @@ type Es2PlusClient interface { /// /// Generic headers for invocations and responses /// - type ES2PlusHeader struct { FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` FunctionCallIdentifier string `json:"functionCallIdentifier"` From 3efbcf5906c3ffe6aa4b2988dae8f85005373681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 22:41:27 +0100 Subject: [PATCH 201/309] Prettier error handling --- .../sim-batch-management/outfileparser/outfileparser.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index a6592b235..0056d10ff 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -299,16 +299,14 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { return fmt.Errorf("couldn't create hss csv file '%s', %v", filename, err) } - _, err = f.WriteString("ICCID, IMSI, KI\n") - if err != nil { + if _, err = f.WriteString("ICCID, IMSI, KI\n"); err != nil { return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) } max := 0 for i, entry := range entries { s := fmt.Sprintf("%s, %s, %s\n", entry.IccidWithChecksum, entry.Imsi, entry.Ki) - _, err = f.WriteString(s) - if err != nil { + if _, err = f.WriteString(s); err != nil { return fmt.Errorf("couldn't write to hss csv file '%s', %v", filename, err) } max = i + 1 From e50a051e5face4a09e97e1eeddf51c84ff81c5b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 22:44:03 +0100 Subject: [PATCH 202/309] Prettier error handling --- .../outfileparser/outfileparser_test.go | 4 +--- sim-administration/sim-batch-management/store/store.go | 8 +++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index cd0b5ccdf..d64bb9a0d 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -48,10 +48,8 @@ func TestParseOutputVariablesLine(t *testing.T) { varOutLine := "var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4" m := make(map[string]int) - err := ParseVarOutLine(varOutLine, &m) - if err != nil { + if err := ParseVarOutLine(varOutLine, &m); err != nil { t.Error("Couldn't parse var_out line:", err) - t.Fail() } assert.Equal(t, m["ICCID"], 0) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 1be8fc831..863bf2441 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -359,9 +359,8 @@ func (sdb SimBatchDB) DeclareBatch( }() // Persist the newly created batch, - err = sdb.CreateBatch(&batch) - if err != nil { - panic(err) + if err = sdb.CreateBatch(&batch) ; err != nil { + return nil, err } imsi, err := strconv.Atoi(batch.FirstImsi) @@ -395,8 +394,7 @@ func (sdb SimBatchDB) DeclareBatch( Ki: "", // Should be null } - err = sdb.CreateSimEntry(simEntry) - if err != nil { + if err = sdb.CreateSimEntry(simEntry); err != nil { panic(err) } From 896afe3ba4958014cbcc4ce1aeb886c6020ccf86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 22:45:26 +0100 Subject: [PATCH 203/309] Prettier error handling --- .../sim-batch-management/store/store_test.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index a7241ec9c..7ab7da304 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -35,8 +35,7 @@ func setup() { panic(fmt.Errorf("Couldn't open new in memory database '%s", err)) } - err = sdb.GenerateTables() - if err != nil { + if err = sdb.GenerateTables(); err != nil { panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) } @@ -62,8 +61,7 @@ func cleanTables() { func shutdown() { cleanTables() - err := sdb.DropTables() - if err != nil { + if err := sdb.DropTables(); err != nil { panic(fmt.Sprintf("Couldn't drop tables '%s'", err)) } sdb.Db.Close() @@ -71,8 +69,7 @@ func shutdown() { // ... just to know that everything is sane. func TestMemoryDbPing(t *testing.T) { - err = sdb.Db.Ping() - if err != nil { + if err = sdb.Db.Ping(); err != nil { t.Errorf("Could not ping in-memory database. '%s'", err) } } From b62167258ff1de53f6defaa9ecd58ae66ab16d7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 22:47:32 +0100 Subject: [PATCH 204/309] Prettier error handling --- sim-administration/sim-batch-management/sim-batch-mgt.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 0d60e55b7..46beb143b 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -167,8 +167,7 @@ func main() { outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" fmt.Println("outputFile = ", outputFile) - err := outfileparser.WriteHssCsvFile(outputFile, outRecord.Entries) - if err != nil { + if err := outfileparser.WriteHssCsvFile(outputFile, outRecord.Entries); err != nil { log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") } From d75b89882811fb4c90c2ad1470b2a564d9eef522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 13 Nov 2019 23:42:40 +0100 Subject: [PATCH 205/309] Prettier error handling --- .../sim-batch-management/store/store.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 863bf2441..05a2a19c8 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -275,11 +275,11 @@ func (sdb SimBatchDB) DeclareBatch( batchLength, err := strconv.Atoi(batchLengthString) if err != nil { - log.Fatalf("Not a valid batch Quantity string '%s'.\n", batchLengthString) + return nil, fmt.Errorf("not a valid batch Quantity string '%s'", batchLengthString) } if batchLength <= 0 { - log.Fatalf("OutputBatch Quantity must be positive, but was '%d'", batchLength) + return nil, fmt.Errorf("OutputBatch Quantity must be positive, but was '%d'", batchLength) } uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", @@ -322,7 +322,7 @@ func (sdb SimBatchDB) DeclareBatch( tail := flag.Args() if len(tail) != 0 { - log.Printf("Unknown parameters: %s", flag.Args()) + return nil, fmt.Errorf("Unknown parameters: %s", flag.Args()) } filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) @@ -365,7 +365,7 @@ func (sdb SimBatchDB) DeclareBatch( imsi, err := strconv.Atoi(batch.FirstImsi) if err != nil { - panic(err) + return nil, err } // Now create all the sim profiles @@ -375,7 +375,7 @@ func (sdb SimBatchDB) DeclareBatch( // XXX !!! TODO THis is wrong, but I'm doing it now, just to get started! var msisdn, err2 = strconv.Atoi(batch.FirstMsisdn) if err2 != nil { - panic(err) + return nil, err } for i := 0; i < batch.Quantity; i++ { @@ -395,7 +395,7 @@ func (sdb SimBatchDB) DeclareBatch( } if err = sdb.CreateSimEntry(simEntry); err != nil { - panic(err) + return nil, err } iccidWithoutLuhnChecksum += batch.IccidIncrement From 7263d7a7ca238e6292362c32fa58978c22a66d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 00:10:39 +0100 Subject: [PATCH 206/309] Prettier error handling, and only one exit point (if I haven't missed any) --- .../sim-batch-management/sim-batch-mgt.go | 121 ++++++++++-------- .../sim-batch-management/store/store.go | 2 +- 2 files changed, 69 insertions(+), 54 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 46beb143b..88168aa4c 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -148,10 +148,16 @@ var ( ) func main() { + if err := parseCommandLine() ; err != nil { + panic(err) + } +} + +func parseCommandLine() error { db, err := store.OpenFileSqliteDatabase("foobar.db") if err != nil { - panic(fmt.Sprintf("Couldn't open sqlite database. '%s'", err)) + return fmt.Errorf("couldn't open sqlite database. '%s'", err) } db.GenerateTables() @@ -165,10 +171,10 @@ func main() { outRecord := outfileparser.ParseOutputFile(inputFile) outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" - fmt.Println("outputFile = ", outputFile) + log.Println("outputFile = ", outputFile) if err := outfileparser.WriteHssCsvFile(outputFile, outRecord.Entries); err != nil { - log.Fatal("Couldn't close output file '", outputFilePrefix, "'. Error = '", err, "'") + return fmt.Errorf("couldn't close output file '%s', . Error = '%v'", outputFilePrefix, err) } case "declare-profile-vendor": @@ -178,7 +184,7 @@ func main() { allBatches, err := db.GetAllBatches() if err != nil { - panic(err) + return err } fmt.Println("Names of current batches: ") @@ -190,15 +196,15 @@ func main() { batch, err := db.GetBatchByName(*describeBatchBatch) if err != nil { - panic(err) + return err } if batch == nil { - fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) + return fmt.Errorf("no batch found with name '%s'", *describeBatchBatch) } else { bytes, err := json.MarshalIndent(batch, " ", " ") if err != nil { - fmt.Println("Can't serialize", batch) + return fmt.Errorf("can't serialize batch '%v'", batch) } fmt.Printf("%v\n", string(bytes)) @@ -207,13 +213,12 @@ func main() { case "generate-activation-code-updating-sql": batch, err := db.GetBatchByName(*generateActivationCodeSqlBatch) if err != nil { - msg := fmt.Sprintf("Couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) - panic(msg) + return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) } simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { - panic(err) + return err } for _, b := range simEntries { @@ -226,11 +231,11 @@ func main() { case "generate-batch-upload-script": batch, err := db.GetBatchByName(*generateUploadBatchBatch) if err != nil { - panic(err) + return err } if batch == nil { - fmt.Printf("No batch found with name '%s'\n", *describeBatchBatch) + return fmt.Errorf("no batch found with name '%s'", *describeBatchBatch) } else { var csvPayload = uploadtoprime.GenerateCsvPayload3(db, *batch) uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) @@ -243,7 +248,7 @@ func main() { } if batch == nil { - fmt.Printf("No batch found with name '%s'\n", *generateInputFileBatchname) + return fmt.Errorf("no batch found with name '%s'", *generateInputFileBatchname) } else { var result = GenerateInputFile(batch) fmt.Println(result) @@ -256,7 +261,7 @@ func main() { batch, err := db.GetBatchByName(batchName) if err != nil { - panic(err) + return err } csvFile, _ := os.Open(csvFilename) @@ -264,11 +269,11 @@ func main() { defer csvFile.Close() - headerLine, error := reader.Read() - if error == io.EOF { + headerLine, err := reader.Read() + if err == io.EOF { break - } else if error != nil { - log.Fatal(error) + } else if err != nil { + return err } var columnMap map[string]int @@ -279,15 +284,15 @@ func main() { } if _, hasIccid := columnMap["Iccid"]; !hasIccid { - panic("No ICCID column in CSV file") + return fmt.Errorf("no ICCID column in CSV file") } if _, hasMsisdn := columnMap["msisdn"]; !hasMsisdn { - panic("No MSISDN column in CSV file") + return fmt.Errorf("no MSISDN column in CSV file") } if _, hasImsi := columnMap["imsi"]; !hasImsi { - panic("No IMSI column in CSV file") + return fmt.Errorf("no IMSI column in CSV file") } type csvRecord struct { @@ -321,7 +326,7 @@ func main() { } if _, duplicateRecordExists := recordMap[record.iccid]; duplicateRecordExists { - panic(fmt.Sprintf("Duplicate ICCID record in map: %s", record.iccid)) + return fmt.Errorf("duplicate ICCID record in map: %s", record.iccid) } recordMap[record.iccid] = record @@ -344,12 +349,12 @@ func main() { if entry.Imsi != record.imsi { tx.Rollback() - panic(fmt.Sprintf("IMSI mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Imsi, record.iccid)) + return fmt.Errorf("IMSI mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Imsi, record.iccid) } if entry.Msisdn != "" && record.msisdn != "" && record.msisdn != entry.Msisdn { tx.Rollback() - panic(fmt.Sprintf("MSISDN mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Msisdn, record.msisdn)) + return fmt.Errorf("MSISDN mismatch for ICCID=%s. Batch has %s, csv file has %s", entry.Iccid, entry.Msisdn, record.msisdn) } if entry.Msisdn == "" && record.msisdn != "" { @@ -363,10 +368,10 @@ func main() { } tx.Commit() - fmt.Printf("Updated %d of a total of %d records in batch '%s'\n", noOfRecordsUpdated, len(simEntries), batchName) + log.Printf("Updated %d of a total of %d records in batch '%s'\n", noOfRecordsUpdated, len(simEntries), batchName) case "declare-batch": - fmt.Println("Declare batch") + log.Println("Declare batch") db.DeclareBatch( *dbName, *dbAddLuhn, @@ -398,34 +403,37 @@ func main() { result, err := client.GetStatus(iccid) if err != nil { - panic(err) + return err } - fmt.Printf("Iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) + log.Printf("Iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) case "recover-profile": - checkEs2TargetState(es2Target) + err := checkEs2TargetState(es2Target) + if err != nil { + return err + } result, err := client.RecoverProfile(iccid, *es2Target) if err != nil { - panic(err) + return err } - fmt.Println("result -> ", result) + log.Println("result -> ", result) case "download-order": result, err := client.DownloadOrder(iccid) if err != nil { - panic(err) + return err } - fmt.Println("result -> ", result) + log.Println("result -> ", result) case "confirm-order": result, err := client.ConfirmOrder(iccid) if err != nil { - panic(err) + return err } fmt.Println("result -> ", result) case "activate-Iccid": result, err := client.ActivateIccid(iccid) if err != nil { - panic(err) + return err } fmt.Printf("%s, %s\n", iccid, result.ACToken) @@ -464,11 +472,11 @@ func main() { // Read all the lines into the record map. for { - line, error := reader.Read() - if error == io.EOF { + line, err := reader.Read() + if err == io.EOF { break - } else if error != nil { - log.Fatal(error) + } else if err != nil { + return err } iccid := line[columnMap["Iccid"]] @@ -535,11 +543,11 @@ func main() { case "get-profile-activation-statuses-for-batch": batchName := iccid - fmt.Printf("Getting statuses for all profiles in batch named %s\n", batchName) + log.Printf("Getting statuses for all profiles in batch named %s\n", batchName) batch, err := db.GetBatchByName(batchName) if err != nil { - panic(fmt.Errorf("unknown batch '%s'", batchName)) + return fmt.Errorf("unknown batch '%s'", batchName) } entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) @@ -548,10 +556,10 @@ func main() { } if len(entries) != batch.Quantity { - panic(fmt.Sprintf("Batch quantity retrieved from database (%d) different from batch quantity (%d)\n", len(entries), batch.Quantity)) + return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) } - fmt.Printf("Found %d profiles\n", len(entries)) + log.Printf("Found %d profiles\n", len(entries)) // XXX Is this really necessary? I don't think so var mutex = &sync.Mutex{} @@ -585,7 +593,7 @@ func main() { } if result == nil { - panic(fmt.Sprintf("Couldn't find any status for Iccid='%s'\n", entry.Iccid)) + log.Printf("ERROR: Couldn't find any status for Iccid='%s'\n", entry.Iccid) } mutex.Lock() @@ -607,7 +615,7 @@ func main() { batch, err := db.GetBatchByName(batchName) if err != nil { - panic(fmt.Errorf("unknown batch '%s'", batchName)) + return fmt.Errorf("unknown batch '%s'", batchName) } entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) @@ -616,7 +624,7 @@ func main() { } if len(entries) != batch.Quantity { - panic(fmt.Sprintf("Batch quantity retrieved from database (%d) different from batch quantity (%d)\n", len(entries), batch.Quantity)) + return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) } // XXX Is this really necessary? I don't think so @@ -701,25 +709,32 @@ func main() { } case "cancel-profile": - checkEs2TargetState(es2Target) - _, err := client.CancelOrder(iccid, *es2Target) + err := checkEs2TargetState(es2Target) if err != nil { - panic(err) + return err + } + _, err = client.CancelOrder(iccid, *es2Target) + if err != nil { + return err } default: - panic(fmt.Sprintf("Unknown es2+ subcommand '%s', try --help", *es2cmd)) + return fmt.Errorf("unknown es2+ subcommand '%s', try --help", *es2cmd) } case "batch": fmt.Println("Doing the batch thing.") // storage.doTheBatchThing() default: - panic(fmt.Sprintf("Unknown command: '%s'\n", cmd)) + return fmt.Errorf("unknown command: '%s'", cmd) } + + return nil } -func checkEs2TargetState(target *string) { +func checkEs2TargetState(target *string) error { if *target != "AVAILABLE" { - panic("Target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") + return fmt.Errorf("target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") + } else { + return nil } } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 05a2a19c8..dc437c772 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -322,7 +322,7 @@ func (sdb SimBatchDB) DeclareBatch( tail := flag.Args() if len(tail) != 0 { - return nil, fmt.Errorf("Unknown parameters: %s", flag.Args()) + return nil, fmt.Errorf("unknown parameters: %s", flag.Args()) } filenameBase := fmt.Sprintf("%s%s%s", customer, orderDate, batchNo) From a55cd2c0dce9b218315fee148517dcf147ec78a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 00:17:28 +0100 Subject: [PATCH 207/309] Do we want a mutex here? --- sim-administration/sim-batch-management/store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index dc437c772..83feaffbc 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -104,7 +104,7 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { } func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { - // TODO: mutex + // TODO: mutex? res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", theBatch, From 0b6a95d8c93b03a11dc37721f40777691a461b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 00:35:46 +0100 Subject: [PATCH 208/309] Rename for readability, also hook two methods together that really is one method anyway --- .../sim-batch-management/es2plus/es2plus.go | 35 +++++-------------- .../sim-batch-management/store/store.go | 1 + 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index f94f9ff39..f9db24836 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -237,7 +237,7 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } -func marshalUnmarshalGenericEs2plusCommand( +func execute( client *Es2PlusClientState, es2plusCommand string, payload interface{}, result interface{}) error { @@ -254,24 +254,7 @@ func marshalUnmarshalGenericEs2plusCommand( log.Print("Payload ->", jsonStrB.String()) } - // Get the result of the HTTP POST as a byte array - // that can be deserialized into json. Fail fast - // an error has been detected. - return executeGenericEs2plusCommand( - result, - jsonStrB, - client.hostport, - es2plusCommand, - client.httpClient, - client.logHeaders) -} - -func executeGenericEs2plusCommand(result interface{}, jsonStrB *bytes.Buffer, hostport string, es2plusCommand string, httpClient *http.Client, logHeaders bool) error { - - // Build and execute an ES2+ protocol request by using the serialised JSON in the - // byte array jsonStrB in a POST request. Set up the required - // headers for ES2+ and content type. - url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", hostport, es2plusCommand) + url := fmt.Sprintf("https://%s/gsma/rsp2/es2plus/%s", client.hostport, es2plusCommand) req, err := http.NewRequest("POST", url, jsonStrB) if err != nil { return err @@ -279,11 +262,11 @@ func executeGenericEs2plusCommand(result interface{}, jsonStrB *bytes.Buffer, ho req.Header.Set("X-Admin-Protocol", "gsma/rsp/v2.0.0") req.Header.Set("Content-Type", "application/json") - if logHeaders { + if client.logHeaders { log.Printf("Request -> %s\n", formatRequest(req)) } - resp, err := httpClient.Do(req) + resp, err := client.httpClient.Do(req) if err != nil { return err } @@ -310,7 +293,7 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error Header: *header, IccidList: []ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - if err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); err != nil { + if err = execute(client, es2plusCommand, payload, result); err != nil { return nil, err } @@ -335,7 +318,7 @@ func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState strin Iccid: iccid, ProfileStatus: targetState, } - return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + return result, execute(client, es2plusCommand, payload, result) } func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { @@ -350,7 +333,7 @@ func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) Iccid: iccid, FinalProfileStatusIndicator: targetState, } - return result, marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + return result, execute(client, es2plusCommand, payload, result) } func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { @@ -366,7 +349,7 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadO Eid: "", Profiletype: "", } - err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result) + err = execute(client, es2plusCommand, payload, result) if err != nil { return nil, err } @@ -396,7 +379,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd ReleaseFlag: true, } - if err = marshalUnmarshalGenericEs2plusCommand(client, es2plusCommand, payload, result); err != nil { + if err = execute(client, es2plusCommand, payload, result); err != nil { return nil, err } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 83feaffbc..2d52c4ee9 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -23,6 +23,7 @@ type Store interface { GetBatchById(id int64) (*model.Batch, error) GetBatchByName(id string) (*model.Batch, error) + // TODO: Maybe make the argument list for this one a little shorter? DeclareBatch( db *SimBatchDB, firstIccid string, From 454ccfceb56b22181a088f34a6724df0a210befe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 00:37:50 +0100 Subject: [PATCH 209/309] Make the client execution a bit more idiomatic --- .../sim-batch-management/es2plus/es2plus.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index f9db24836..48f631a5e 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -237,8 +237,7 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } -func execute( - client *Es2PlusClientState, +func (client *Es2PlusClientState) execute( es2plusCommand string, payload interface{}, result interface{}) error { @@ -293,7 +292,7 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error Header: *header, IccidList: []ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, } - if err = execute(client, es2plusCommand, payload, result); err != nil { + if err = client.execute(es2plusCommand, payload, result); err != nil { return nil, err } @@ -318,7 +317,7 @@ func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState strin Iccid: iccid, ProfileStatus: targetState, } - return result, execute(client, es2plusCommand, payload, result) + return result, client.execute(es2plusCommand, payload, result) } func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { @@ -333,7 +332,7 @@ func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) Iccid: iccid, FinalProfileStatusIndicator: targetState, } - return result, execute(client, es2plusCommand, payload, result) + return result, client.execute(es2plusCommand, payload, result) } func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { @@ -349,7 +348,7 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadO Eid: "", Profiletype: "", } - err = execute(client, es2plusCommand, payload, result) + err = client.execute(es2plusCommand, payload, result) if err != nil { return nil, err } @@ -379,7 +378,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd ReleaseFlag: true, } - if err = execute(client, es2plusCommand, payload, result); err != nil { + if err = client.execute(es2plusCommand, payload, result); err != nil { return nil, err } From 8c415d4450f135cb4c2e97d961975266723f0df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 00:45:07 +0100 Subject: [PATCH 210/309] Sligthly prettier error testing --- sim-administration/sim-batch-management/es2plus/es2plus.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 48f631a5e..b846304ea 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -348,8 +348,7 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadO Eid: "", Profiletype: "", } - err = client.execute(es2plusCommand, payload, result) - if err != nil { + if err = client.execute(es2plusCommand, payload, result); err != nil { return nil, err } From 56d8ae97c38c6791157b0b8af0157c3a0dca32b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 00:50:36 +0100 Subject: [PATCH 211/309] Prettier error handling (only panicing when there is no other easier options) --- .../sim-batch-management/es2plus/es2plus.go | 2 +- .../sim-batch-management/sim-batch-mgt.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index b846304ea..8f441f4f3 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -393,7 +393,7 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, e result, err := client.GetStatus(iccid) if err != nil { - panic(err) + return nil, err } if result.ACToken == "" { diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 88168aa4c..2c08c9e9b 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -244,7 +244,7 @@ func parseCommandLine() error { case "generate-input-file": batch, err := db.GetBatchByName(*generateInputFileBatchname) if err != nil { - panic(err) + return err } if batch == nil { @@ -334,7 +334,7 @@ func parseCommandLine() error { simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { - panic(err) + return err } // Check for compatibility @@ -344,7 +344,7 @@ func parseCommandLine() error { record, iccidRecordIsPresent := recordMap[entry.Iccid] if !iccidRecordIsPresent { tx.Rollback() - panic(fmt.Sprintf("ICCID not in batch: %s", entry.Iccid)) + return fmt.Errorf("ICCID not in batch: %s", entry.Iccid) } if entry.Imsi != record.imsi { @@ -361,7 +361,7 @@ func parseCommandLine() error { err = db.UpdateSimEntryMsisdn(entry.Id, record.msisdn) if err != nil { tx.Rollback() - panic(err) + return err } noOfRecordsUpdated += 1 } @@ -460,7 +460,7 @@ func parseCommandLine() error { } if _, hasIccid := columnMap["iccid"]; !hasIccid { - panic("No ICCID column in CSV file") + return fmt.Errorf("no ICCID column in CSV file") } type csvRecord struct { @@ -486,7 +486,7 @@ func parseCommandLine() error { } if _, duplicateRecordExists := recordMap[record.Iccid]; duplicateRecordExists { - panic(fmt.Sprintf("Duplicate ICCID record in map: %s", record.Iccid)) + return fmt.Errorf("duplicate ICCID record in map: %s", record.Iccid) } recordMap[record.Iccid] = record @@ -552,7 +552,7 @@ func parseCommandLine() error { entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { - panic(err) + return err } if len(entries) != batch.Quantity { @@ -620,7 +620,7 @@ func parseCommandLine() error { entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { - panic(err) + return err } if len(entries) != batch.Quantity { From 28a8e780a82dbd88eac723bc7676f9544c9cafbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 01:00:42 +0100 Subject: [PATCH 212/309] Prettifying code --- .../sim-batch-management/sim-batch-mgt.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 2c08c9e9b..6b1118bcd 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -745,17 +745,17 @@ func checkEs2TargetState(target *string) error { func GenerateInputFile(batch *model.Batch) string { result := "*HEADER DESCRIPTION\n" + "***************************************\n" + - fmt.Sprintf("Customer : %s\n", (*batch).Customer) + - fmt.Sprintf("ProfileType : %s\n", (*batch).ProfileType) + - fmt.Sprintf("Order Date : %s\n", (*batch).OrderDate) + - fmt.Sprintf("Batch No : %s\n", (*batch).BatchNo) + - fmt.Sprintf("Quantity : %d\n", (*batch).Quantity) + + fmt.Sprintf("Customer : %s\n", batch.Customer) + + fmt.Sprintf("ProfileType : %s\n", batch.ProfileType) + + fmt.Sprintf("Order Date : %s\n", batch.OrderDate) + + fmt.Sprintf("Batch No : %s\n", batch.BatchNo) + + fmt.Sprintf("Quantity : %d\n", batch.Quantity) + "***************************************\n" + "*INPUT VARIABLES\n" + "***************************************\n" + "var_In:\n" + - fmt.Sprintf(" ICCID: %s\n", (*batch).FirstIccid) + - fmt.Sprintf("IMSI: %s\n", (*batch).FirstImsi) + + fmt.Sprintf(" ICCID: %s\n", batch.FirstIccid) + + fmt.Sprintf("IMSI: %s\n", batch.FirstImsi) + "***************************************\n" + "*OUTPUT VARIABLES\n" + "***************************************\n" + From d65866c25df7d8be617be679970da17a68afa18d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 09:55:04 +0100 Subject: [PATCH 213/309] Adding data model for profile vendors --- .../sim-batch-management/model/model.go | 13 +++++++++++-- .../sim-batch-management/sim-batch-mgt.go | 7 ++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 2d146a619..3fcef4c5d 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -1,8 +1,8 @@ package model -// TODO: This type SimEntry struct { struct isn't fully baked. - +// TODO: Delete all the ICCID entries that are not necessary, that would be at +// about three of them. type SimEntry struct { Id int64 `db:"id" json:"id"` BatchID int64 `db:"batchId" json:"batchId"` @@ -40,3 +40,12 @@ type Batch struct { ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` } + +type ProfileVendor { + Id int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` + Es2plusCert string `db:"es2PlusCert" json:"es2plusCert"` + Es2plusKey string `db:"es2PlusKey" json:"es2PlusKey"` + Es2plusHost string `db:"es2PlusHost" json:"es2plusHost"` + Es2plusPort string `db:"es2PlusPort" json:"es2plusPort"` +} \ No newline at end of file diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 6b1118bcd..6d728b9be 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -148,7 +148,7 @@ var ( ) func main() { - if err := parseCommandLine() ; err != nil { + if err := parseCommandLine(); err != nil { panic(err) } } @@ -213,7 +213,7 @@ func parseCommandLine() error { case "generate-activation-code-updating-sql": batch, err := db.GetBatchByName(*generateActivationCodeSqlBatch) if err != nil { - return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) + return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) } simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) @@ -456,7 +456,7 @@ func parseCommandLine() error { columnMap = make(map[string]int) for index, fieldname := range headerLine { - columnMap[strings.ToLower(fieldname)] = index + columnMap[strings.TrimSpace(strings.ToLower(fieldname))] = index } if _, hasIccid := columnMap["iccid"]; !hasIccid { @@ -480,6 +480,7 @@ func parseCommandLine() error { } iccid := line[columnMap["Iccid"]] + iccid = strings.TrimSpace(iccid) record := csvRecord{ Iccid: iccid, From de3916db6bab2bf388ed931380e32003212d206d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 10:37:38 +0100 Subject: [PATCH 214/309] Writing an untested CRUD (sans UD) for profile vendors --- .../sim-batch-management/model/model.go | 10 +- .../sim-batch-management/store/store.go | 100 +++++++++++++++--- 2 files changed, 88 insertions(+), 22 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 3fcef4c5d..04751fe1c 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -41,11 +41,11 @@ type Batch struct { FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` } -type ProfileVendor { +type ProfileVendor struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` - Es2plusCert string `db:"es2PlusCert" json:"es2plusCert"` - Es2plusKey string `db:"es2PlusKey" json:"es2PlusKey"` - Es2plusHost string `db:"es2PlusHost" json:"es2plusHost"` - Es2plusPort string `db:"es2PlusPort" json:"es2plusPort"` + Es2plusCert string `db:"es2PlusCertPath" json:"es2plusCertPath"` + Es2plusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` + Es2plusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` + Es2plusPort int `db:"es2PlusPort" json:"es2plusPort"` } \ No newline at end of file diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 2d52c4ee9..7ce98fb22 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -18,14 +18,15 @@ type Store interface { GenerateTables() error DropTables() error - Create(theBatch *model.Batch) error + CreateBath(theBatch *model.Batch) error GetAllBatches(id string) ([]model.Batch, error) GetBatchById(id int64) (*model.Batch, error) GetBatchByName(id string) (*model.Batch, error) - // TODO: Maybe make the argument list for this one a little shorter? + // TODO: Maybe make the argument list for this one a little shorter, or + // perhaps this should be taken out of the store interface altogether + // (probably the best)? DeclareBatch( - db *SimBatchDB, firstIccid string, lastIccid string, firstIMSI string, @@ -45,6 +46,10 @@ type Store interface { UpdateActivationCode(simId int64, activationCode string) error GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) + CreateProfileVendor(*model.ProfileVendor) error + GetProfileVendorById(id int64) (*model.ProfileVendor, error) + GetProfileVendorByName(name string) (*model.ProfileVendor, error) + Begin() } @@ -131,7 +136,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { } func (sdb *SimBatchDB) GenerateTables() error { - foo := `CREATE TABLE IF NOT EXISTS BATCH ( + sql := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, filenameBase VARCHAR, @@ -147,12 +152,12 @@ func (sdb *SimBatchDB) GenerateTables() error { imsiIncrement INTEGER, iccidIncrement INTEGER, url VARCHAR)` - _, err := sdb.Db.Exec(foo) + _, err := sdb.Db.Exec(sql) if err != nil { return err } - foo = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( + sql = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( id INTEGER PRIMARY KEY AUTOINCREMENT, batchId INTEGER NOT NULL, activationCode VARCHAR NOT NULL, @@ -163,22 +168,84 @@ func (sdb *SimBatchDB) GenerateTables() error { iccid VARCHAR NOT NULL, ki VARCHAR NOT NULL, msisdn VARCHAR NOT NULL)` - _, err = sdb.Db.Exec(foo) + _, err = sdb.Db.Exec(sql) + if err != nil { + return err + } + + sql = `CREATE TABLE IF NOT EXISTS PROFILE_VENDOR ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL UNIQUE, + es2PlusCertPath VARCHAR + es2PlusKeyPath VARCHAR + es2PlusHostPath VARCHAR + es2PlusPort VARCHAR + )` + _, err = sdb.Db.Exec(sql) + + return err } + + +func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { + // TODO: This insert string can be made through reflection, and at some point should be. + res, err := sdb.Db.NamedExec(`INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort) + VALUES (:name, :es2PlusKeyPath, :es2PlusKeyPath, :es2PlusHostPath, :es2PlusPort)`, + theEntry) + if err != nil { + return err + } + + id, err := res.LastInsertId() + if err != nil { + return fmt.Errorf("getting last inserted id failed '%s'", err) + } + theEntry.Id = id + return nil +} + +func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, error) { + result:= []model.ProfileVendor{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", id); err != nil { + return nil, err + } + + if len(result) == 0 { + return nil, nil + } else { + return &result[0], nil + } +} + +func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { + result:= []model.ProfileVendor{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where name = ?", name); err != nil { + return nil, err + } + + if len(result) == 0 { + return nil, nil + } else { + return &result[0], nil + } +} + + + func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", - (*theEntry).BatchID, // XXX Fix this! - (*theEntry).ActivationCode, - (*theEntry).RawIccid, - (*theEntry).IccidWithChecksum, - (*theEntry).IccidWithoutChecksum, - (*theEntry).Iccid, - (*theEntry).Imsi, - (*theEntry).Msisdn, - (*theEntry).Ki, + theEntry.BatchID, + theEntry.ActivationCode, + theEntry.RawIccid, + theEntry.IccidWithChecksum, + theEntry.IccidWithoutChecksum, + theEntry.Iccid, + theEntry.Imsi, + theEntry.Msisdn, + theEntry.Ki, ) id, err := res.LastInsertId() @@ -206,7 +273,6 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { "msisdn": msisdn, }) return err - } func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { From cbdc91814c4ced0b5f823d4500a1750abb41c7f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 10:38:24 +0100 Subject: [PATCH 215/309] whitespace --- .../sim-batch-management/store/store.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 7ce98fb22..8bbf19de5 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -128,7 +128,6 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { } theBatch.BatchId = id - _, err = sdb.Db.NamedExec("UPDATE BATCH SET firstIccid = :firstIccid, firstImsi = :firstImsi, firstMsisdn = :firstMsisdn, msisdnIncrement = :msisdnIncrement, iccidIncrement = :iccidIncrement, imsiIncrement = :imsiIncrement, url=:url WHERE id = :id", theBatch) @@ -183,12 +182,9 @@ func (sdb *SimBatchDB) GenerateTables() error { )` _, err = sdb.Db.Exec(sql) - return err } - - func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { // TODO: This insert string can be made through reflection, and at some point should be. res, err := sdb.Db.NamedExec(`INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort) @@ -207,8 +203,8 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { } func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, error) { - result:= []model.ProfileVendor{} - if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", id); err != nil { + result := []model.ProfileVendor{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", id); err != nil { return nil, err } @@ -220,8 +216,8 @@ func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, erro } func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { - result:= []model.ProfileVendor{} - if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where name = ?", name); err != nil { + result := []model.ProfileVendor{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where name = ?", name); err != nil { return nil, err } @@ -232,8 +228,6 @@ func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, } } - - func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", @@ -426,7 +420,7 @@ func (sdb SimBatchDB) DeclareBatch( }() // Persist the newly created batch, - if err = sdb.CreateBatch(&batch) ; err != nil { + if err = sdb.CreateBatch(&batch); err != nil { return nil, err } @@ -461,7 +455,7 @@ func (sdb SimBatchDB) DeclareBatch( Ki: "", // Should be null } - if err = sdb.CreateSimEntry(simEntry); err != nil { + if err = sdb.CreateSimEntry(simEntry); err != nil { return nil, err } From 6107ddbb48fece8933a4917f5a5faa488d910d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 13:33:00 +0100 Subject: [PATCH 216/309] Adding model for profile vendors --- sim-administration/sim-batch-management/model/model.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 04751fe1c..b86a23c4d 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -44,8 +44,8 @@ type Batch struct { type ProfileVendor struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` - Es2plusCert string `db:"es2PlusCertPath" json:"es2plusCertPath"` - Es2plusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` - Es2plusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` - Es2plusPort int `db:"es2PlusPort" json:"es2plusPort"` -} \ No newline at end of file + Es2PlusCert string `db:"es2PlusCertPath" json:"es2plusCertPath"` + Es2PlusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` + Es2PlusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` + Es2PlusPort int `db:"es2PlusPort" json:"es2plusPort"` +} From 3535ba68f2b511bb672fdc03f46d0c79f3a71878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 13:34:17 +0100 Subject: [PATCH 217/309] Adding code to store profile vendors --- .../sim-batch-management/store/store.go | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 8bbf19de5..cba60df7b 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -14,6 +14,12 @@ import ( "strconv" ) + + +type SimBatchDB struct { + Db *sqlx.DB +} + type Store interface { GenerateTables() error DropTables() error @@ -61,9 +67,7 @@ func (sdb *SimBatchDB) Begin() *sql.Tx { return tx } -type SimBatchDB struct { - Db *sqlx.DB -} + func NewInMemoryDatabase() (*SimBatchDB, error) { db, err := sqlx.Connect("sqlite3", ":memory:") @@ -175,9 +179,9 @@ func (sdb *SimBatchDB) GenerateTables() error { sql = `CREATE TABLE IF NOT EXISTS PROFILE_VENDOR ( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR NOT NULL UNIQUE, - es2PlusCertPath VARCHAR - es2PlusKeyPath VARCHAR - es2PlusHostPath VARCHAR + es2PlusCertPath VARCHAR, + es2PlusKeyPath VARCHAR, + es2PlusHostPath VARCHAR, es2PlusPort VARCHAR )` _, err = sdb.Db.Exec(sql) @@ -204,7 +208,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, error) { result := []model.ProfileVendor{} - if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", id); err != nil { + if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where id = ?", id); err != nil { return nil, err } @@ -217,7 +221,7 @@ func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, erro func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { result := []model.ProfileVendor{} - if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where name = ?", name); err != nil { + if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where name = ?", name); err != nil { return nil, err } From d653bec0cb792361ef177fdf12abcd645b7f8f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 13:48:20 +0100 Subject: [PATCH 218/309] Passing tests --- go.mod | 1 + go.sum | 107 ++++++++++++++++++ .../sim-batch-management/store/store.go | 5 +- .../sim-batch-management/store/store_test.go | 69 ++++++++--- 4 files changed, 164 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 418248118..63c304d32 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ostelco/ostelco-core go 1.13 require ( + cloud.google.com/go/logging v1.0.0 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect github.com/go-ozzo/ozzo-dbx v1.0.15 diff --git a/go.sum b/go.sum index cf5fb4a8b..2e23d27f4 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,53 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0 h1:banaiRPAM8kUVYneOSkhgcDsLzEvL25FinuiSZaH/2w= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= +cloud.google.com/go v0.48.0 h1:6ZHYIRlohUdU4LrLHbTsReY1eYy/MoZW1FsEyBuMXsk= +cloud.google.com/go/logging v1.0.0 h1:kaunpnoEh9L4hu6JUsBa8Y20LBfKnCuDhKUgdZp7oK8= +cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ozzo/ozzo-dbx v1.0.15 h1:cahBHmsueUkRDhSM71f6TLfd1LtYN4XDce7iCblSYKA= github.com/go-ozzo/ozzo-dbx v1.0.15/go.mod h1:48NXRgCxSU5JUSW+EA5lnokrL7ql1j6Qh2RsWeh6Fvs= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -31,17 +63,88 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac h1:MQEvx39qSf8vyrx3XRaOe+j1UDIzKwkYOVObRgGPVqI= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0 h1:9sdfJOzWlkqPltHAuzT2Cp+yrBeY1KRVYgms8soxMwM= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610 h1:Ygq9/SRJX9+dU0WCIICM8RkWvDw03lvB77hrhJnpxfU= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -50,5 +153,9 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index cba60df7b..40c0b6774 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -191,8 +191,9 @@ func (sdb *SimBatchDB) GenerateTables() error { func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { // TODO: This insert string can be made through reflection, and at some point should be. - res, err := sdb.Db.NamedExec(`INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort) - VALUES (:name, :es2PlusKeyPath, :es2PlusKeyPath, :es2PlusHostPath, :es2PlusPort)`, + res, err := sdb.Db.NamedExec(` + INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort) + VALUES (:name, :es2PlusCertPath, :es2PlusKeyPath, :es2PlusHostPath, :es2PlusPort)`, theEntry) if err != nil { return err diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 7ab7da304..a2b7c9335 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -10,8 +10,9 @@ import ( "testing" ) + var sdb *SimBatchDB -var err error +var sdbSetupError error func TestMain(m *testing.M) { setup() @@ -29,14 +30,18 @@ func setup() { // delete file, ignore any errors os.Remove(filename) - sdb, err = OpenFileSqliteDatabase(filename) - // sdb, err = NewInMemoryDatabase() - if err != nil { - panic(fmt.Errorf("Couldn't open new in memory database '%s", err)) + sdb, sdbSetupError = OpenFileSqliteDatabase(filename) + + if sdbSetupError != nil { + panic(fmt.Errorf("Couldn't open new in memory database '%s", sdbSetupError)) } - if err = sdb.GenerateTables(); err != nil { - panic(fmt.Sprintf("Couldn't generate tables '%s'", err)) + if sdb == nil { + panic("Returned null database object") + } + + if sdbSetupError = sdb.GenerateTables(); sdbSetupError != nil { + panic(fmt.Sprintf("Couldn't generate tables '%s'", sdbSetupError)) } cleanTables() @@ -44,19 +49,19 @@ func setup() { func cleanTables() { - // return - // The tables don't seem to be guaranteed to be empty - simProfileDeletionResult, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") + fmt.Println("1 --> ", *sdb) + _, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") + fmt.Println("1.1 -->") if err != nil { panic(fmt.Sprintf("Couldn't delete SIM_PROFILE '%s'", err)) } - batchDeleteResult, err := sdb.Db.Exec("DELETE FROM BATCH") + + fmt.Println("2 -->") + _, err = sdb.Db.Exec("DELETE FROM BATCH") if err != nil { panic(fmt.Sprintf("Couldn't delete BATCH '%s'", err)) } - - simProfileDeletionResult.RowsAffected() - batchDeleteResult.RowsAffected() + fmt.Println("3 -->") } func shutdown() { @@ -69,7 +74,7 @@ func shutdown() { // ... just to know that everything is sane. func TestMemoryDbPing(t *testing.T) { - if err = sdb.Db.Ping(); err != nil { + if err := sdb.Db.Ping(); err != nil { t.Errorf("Could not ping in-memory database. '%s'", err) } } @@ -187,6 +192,39 @@ func TestDeclareBatch(t *testing.T) { } +func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { + cleanTables() + v := &model.ProfileVendor{ + Name: "MyName", + Es2PlusCert: "cert", + Es2PlusKey: "key", + Es2PlusHost: "host", + Es2PlusPort: 4711, + } + + if err := sdb.CreateProfileVendor(v); err != nil { + t.Fatal(err) + } + + nameRetrievedVendor,err := sdb.GetProfileVendorByName("MyName") + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(nameRetrievedVendor, v) { + t.Fatalf("name retrieved and stored profile vendor entries are different, %v v.s. %v", nameRetrievedVendor, v) + } + + idRetrievedVendor,err := sdb.GetProfileVendorById(v.Id) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(idRetrievedVendor, v) { + t.Fatalf("name retrieved and stored profile vendor entries are different, %v v.s. %v", idRetrievedVendor, v) + } +} + func TestDeclareAndRetrieveSimEntries(t *testing.T) { cleanTables() theBatch := declareTestBatch(t) @@ -204,7 +242,6 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { ActivationCode: "8", } - sdb.CreateSimEntry(&entry) assert.Assert(t, entry.Id != 0) From a3bbf76ebaa9bdc6bd0268040e875a9139536537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 13:48:47 +0100 Subject: [PATCH 219/309] Passing tests --- sim-administration/sim-batch-management/store/store_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index a2b7c9335..e21934b7c 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -48,20 +48,15 @@ func setup() { } func cleanTables() { - - fmt.Println("1 --> ", *sdb) _, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") - fmt.Println("1.1 -->") if err != nil { panic(fmt.Sprintf("Couldn't delete SIM_PROFILE '%s'", err)) } - fmt.Println("2 -->") _, err = sdb.Db.Exec("DELETE FROM BATCH") if err != nil { panic(fmt.Sprintf("Couldn't delete BATCH '%s'", err)) } - fmt.Println("3 -->") } func shutdown() { From fe801367d14e945827bae587f8888c45d3f06be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 14:21:49 +0100 Subject: [PATCH 220/309] Crating new profile vendors with pointers to credentials --- .../sim-batch-management/sim-batch-mgt.go | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 6d728b9be..1f80ab753 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -28,14 +28,14 @@ var ( // Declare a profile-vendor with an SM-DP+ that can be referred to from // batches. Referential integrity required, so it won't be possible to // declare bathes with non-existing profile vendors. - /** TODO + dpv = kingpin.Command("declare-profile-vendor", "Declare a profile vendor with an SM-DP+ we can talk to") dpvName = dpv.Flag("name", "Name of profile-vendor").Required().String() dpvCertFilePath = dpv.Flag("cert", "Certificate pem file.").Required().String() dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() - dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().String() - */ + dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() + // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files // TODO: Declare hss-es, that can be refered to in profiles. @@ -164,6 +164,46 @@ func parseCommandLine() error { cmd := kingpin.Parse() switch cmd { + + case "declare-profile-vendor": + + vendor, err := db.GetProfileVendorByName(*dpvName) + if err != nil { + return err + } + + if vendor != nil { + return fmt.Errorf("already declared profile vendor '%s'", *dpvName) + } + + if _, err := os.Stat(*dpvCertFilePath); os.IsNotExist(err) { + return fmt.Errorf("can't find certificate file '%s'", *dpvCertFilePath) + } + + if _, err := os.Stat(*dpvKeyFilePath); os.IsNotExist(err) { + return fmt.Errorf("can't find key file '%s'", *dpvKeyFilePath) + } + + if *dpvPort <= 0 { + return fmt.Errorf("port must be positive was '%d'", *dpvPort) + } + + if 65534 < *dpvPort { + return fmt.Errorf("port must be smaller than or equal to 65535, was '%d'", *dpvPort) + } + + v := &model.ProfileVendor{ + Name: *dpvName, + Es2PlusCert: *dpvCertFilePath, + Es2PlusKey: *dpvKeyFilePath, + Es2PlusHost: *dpvHost, + Es2PlusPort: *dpvPort, + } + + if err := db.CreateProfileVendor(v); err != nil { + return err + } + case "sim-profile-upload": inputFile := *spUploadInputFile @@ -177,9 +217,6 @@ func parseCommandLine() error { return fmt.Errorf("couldn't close output file '%s', . Error = '%v'", outputFilePrefix, err) } - case "declare-profile-vendor": - fmt.Println("Declaration of profile-vendors not yet implemented.") - case "list-batches": allBatches, err := db.GetAllBatches() From a1cc20011d3ab4867315e91e71810b893db2738d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 14:25:31 +0100 Subject: [PATCH 221/309] Use absolute filepaths --- .../sim-batch-management/sim-batch-mgt.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 1f80ab753..c2f6dc1da 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -16,6 +16,7 @@ import ( "io" "log" "os" + "path/filepath" "strings" "sync" ) @@ -192,10 +193,20 @@ func parseCommandLine() error { return fmt.Errorf("port must be smaller than or equal to 65535, was '%d'", *dpvPort) } + // Modify the paths to absolute paths. + + absDpvCertFilePath, err := filepath.Abs(*dpvCertFilePath) + if err != nil { + return err + } + absDpvKeyFilePath, err := filepath.Abs(*dpvKeyFilePath) + if err != nil { + return err + } v := &model.ProfileVendor{ Name: *dpvName, - Es2PlusCert: *dpvCertFilePath, - Es2PlusKey: *dpvKeyFilePath, + Es2PlusCert: absDpvCertFilePath, + Es2PlusKey: absDpvKeyFilePath , Es2PlusHost: *dpvHost, Es2PlusPort: *dpvPort, } From f629e668b8ea84261a1bc6219cd9af5d51757230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 14:38:39 +0100 Subject: [PATCH 222/309] Adding requester id --- sim-administration/sim-batch-management/model/model.go | 1 + sim-administration/sim-batch-management/sim-batch-mgt.go | 5 +++++ sim-administration/sim-batch-management/store/store.go | 7 ++++--- .../sim-batch-management/store/store_test.go | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index b86a23c4d..7f36f7531 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -48,4 +48,5 @@ type ProfileVendor struct { Es2PlusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` Es2PlusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` Es2PlusPort int `db:"es2PlusPort" json:"es2plusPort"` + Es2PlusRequesterId string `db:"es2PlusRequesterId" json:"es2PlusRequesterId"` } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index c2f6dc1da..ba54bcebb 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -36,6 +36,8 @@ var ( dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() + dpvRequesterId = es2.Flag("requester-id", "ES2+ requester ID.").Required().String() + // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files @@ -209,8 +211,11 @@ func parseCommandLine() error { Es2PlusKey: absDpvKeyFilePath , Es2PlusHost: *dpvHost, Es2PlusPort: *dpvPort, + Es2PlusRequesterId: *dpvRequesterId, } + fmt.Printf("Profilevendor = %v", v) + os.Exit(1) if err := db.CreateProfileVendor(v); err != nil { return err } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 40c0b6774..4b5dced6b 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -182,7 +182,8 @@ func (sdb *SimBatchDB) GenerateTables() error { es2PlusCertPath VARCHAR, es2PlusKeyPath VARCHAR, es2PlusHostPath VARCHAR, - es2PlusPort VARCHAR + es2PlusPort VARCHAR, + es2PlusRequesterId VARCHAR )` _, err = sdb.Db.Exec(sql) @@ -192,8 +193,8 @@ func (sdb *SimBatchDB) GenerateTables() error { func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { // TODO: This insert string can be made through reflection, and at some point should be. res, err := sdb.Db.NamedExec(` - INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort) - VALUES (:name, :es2PlusCertPath, :es2PlusKeyPath, :es2PlusHostPath, :es2PlusPort)`, + INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort, es2PlusRequesterId) + VALUES (:name, :es2PlusCertPath, :es2PlusKeyPath, :es2PlusHostPath, :es2PlusPort, :es2PlusRequesterId)`, theEntry) if err != nil { return err diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index e21934b7c..b66613d95 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -195,6 +195,7 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { Es2PlusKey: "key", Es2PlusHost: "host", Es2PlusPort: 4711, + Es2PlusRequesterId: "1.2.3", } if err := sdb.CreateProfileVendor(v); err != nil { From 4299e162ad6e0c2e7c5938d241e1b6f9792028f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 14:45:37 +0100 Subject: [PATCH 223/309] Fix bug in parsing, increase niceness overall --- sim-administration/sim-batch-management/sim-batch-mgt.go | 8 ++++---- sim-administration/sim-batch-management/store/store.go | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index ba54bcebb..8c5f6e3f2 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -36,7 +36,7 @@ var ( dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() - dpvRequesterId = es2.Flag("requester-id", "ES2+ requester ID.").Required().String() + dpvRequesterId = dpv.Flag("requester-id", "ES2+ requester ID.").Required().String() // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... @@ -213,13 +213,13 @@ func parseCommandLine() error { Es2PlusPort: *dpvPort, Es2PlusRequesterId: *dpvRequesterId, } - - fmt.Printf("Profilevendor = %v", v) - os.Exit(1) + if err := db.CreateProfileVendor(v); err != nil { return err } + fmt.Println("Declared a new vendor named ", *dpvName) + case "sim-profile-upload": inputFile := *spUploadInputFile diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 4b5dced6b..0667c3a20 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -183,8 +183,7 @@ func (sdb *SimBatchDB) GenerateTables() error { es2PlusKeyPath VARCHAR, es2PlusHostPath VARCHAR, es2PlusPort VARCHAR, - es2PlusRequesterId VARCHAR - )` + es2PlusRequesterId VARCHAR)` _, err = sdb.Db.Exec(sql) return err From 48294fec31dc8c9be5edc3c2d4ae4c638689538f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 14:57:13 +0100 Subject: [PATCH 224/309] yeeha, now we can do es2 commands for specific hss vendors using stored parameters --- .../sim-batch-management/sim-batch-mgt.go | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 8c5f6e3f2..63daea98f 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -53,10 +53,11 @@ var ( "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-Iccid, get-profile-activation-statuses-for-batch, get-profile-activation-statuses-for-iccids-in-file").Required().String() es2iccid = es2.Arg("Iccid", "Iccid of profile to manipulate").String() es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() - es2CertFilePath = es2.Flag("cert", "Certificate pem file.").Required().String() - es2KeyFilePath = es2.Flag("key", "Certificate key file.").Required().String() - es2Hostport = es2.Flag("hostport", "host:port of ES2+ endpoint.").Required().String() - es2RequesterId = es2.Flag("requesterid", "ES2+ requester ID.").Required().String() + es2CertFilePath = es2.Flag("cert", "Certificate pem file.").String() + es2KeyFilePath = es2.Flag("key", "Certificate key file.").String() + es2Hostport = es2.Flag("hostport", "host:port of ES2+ endpoint.").String() + es2RequesterId = es2.Flag("requesterid", "ES2+ requester ID.").String() + es2ProfileVendor = es2.Flag("profile-vendor", "Name of profile-vendor").Required().String() // // Convert an output (.out) file from an sim profile producer into an input file @@ -213,7 +214,7 @@ func parseCommandLine() error { Es2PlusPort: *dpvPort, Es2PlusRequesterId: *dpvRequesterId, } - + if err := db.CreateProfileVendor(v); err != nil { return err } @@ -447,8 +448,18 @@ func parseCommandLine() error { case "es2": - // TODO: Vet all the parameters, they can very easily be bogus. - client := es2plus.Client(*es2CertFilePath, *es2KeyFilePath, *es2Hostport, *es2RequesterId) + vendor, err := db.GetProfileVendorByName(*es2ProfileVendor) + if err != nil { + return err + } + if vendor == nil { + return fmt.Errorf("unknown profile vendor '%s'", *es2ProfileVendor) + } + + + hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) + client := es2plus.Client(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) + iccid := *es2iccid switch *es2cmd { From b2df47c2f59884cd2b1be1227ce6c7bfcd94501c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 15:11:59 +0100 Subject: [PATCH 225/309] First of the new config aware batch commands online --- .../sim-batch-management/sim-batch-mgt.go | 226 ++++++++++++------ 1 file changed, 158 insertions(+), 68 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 63daea98f..0f2a3d284 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -59,6 +59,9 @@ var ( es2RequesterId = es2.Flag("requesterid", "ES2+ requester ID.").String() es2ProfileVendor = es2.Flag("profile-vendor", "Name of profile-vendor").Required().String() + + getProfActActStatusesForBatch = kingpin.Command("get-profile-activation-statuses-for-batch", "Get current activation statuses from SM-DP+ for named batch.") + getProfActActStatusesForBatchBatch =getProfActActStatusesForBatch.Arg("batcgh", "The batch to get activation statuses for.").Required().String() // // Convert an output (.out) file from an sim profile producer into an input file // for Prime. @@ -221,6 +224,92 @@ func parseCommandLine() error { fmt.Println("Declared a new vendor named ", *dpvName) + case "get-profile-activation-statuses-for-batch": + batchName := *getProfActActStatusesForBatchBatch + + log.Printf("Getting statuses for all profiles in batch named %s\n", batchName) + + batch, err := db.GetBatchByName(batchName) + if err != nil { + return fmt.Errorf("unknown batch '%s'", batchName) + } + + // This is a monkeypatch + profileVendorName := "Idemia" + // TODO Should have been profileVendorName := batdh.ProfileVendor + + vendor, err := db.GetProfileVendorByName(profileVendorName) + if err != nil { + return err + } + if vendor == nil { + return fmt.Errorf("unknown profile vendor '%s'", profileVendorName) + } + + + hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) + client := es2plus.Client(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) + + + + entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + return err + } + + if len(entries) != batch.Quantity { + return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) + } + + log.Printf("Found %d profiles\n", len(entries)) + + // XXX Is this really necessary? I don't think so + var mutex = &sync.Mutex{} + + var waitgroup sync.WaitGroup + + // Limit concurrency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. + + concurrency := 160 + sem := make(chan bool, concurrency) + for _, entry := range entries { + + // + // Only apply activation if not already noted in the + // database. + // + + sem <- true + + waitgroup.Add(1) + go func(entry model.SimEntry) { + + defer func() { <-sem }() + + result, err := client.GetStatus(entry.Iccid) + if err != nil { + panic(err) + } + + if result == nil { + log.Printf("ERROR: Couldn't find any status for Iccid='%s'\n", entry.Iccid) + } + + mutex.Lock() + fmt.Printf("%s, %s\n", entry.Iccid, result.State) + mutex.Unlock() + waitgroup.Done() + }(entry) + } + + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } + case "sim-profile-upload": inputFile := *spUploadInputFile @@ -463,6 +552,75 @@ func parseCommandLine() error { iccid := *es2iccid switch *es2cmd { + + case "get-profile-activation-statuses-for-batch": + batchName := iccid + + log.Printf("Getting statuses for all profiles in batch named %s\n", batchName) + + batch, err := db.GetBatchByName(batchName) + if err != nil { + return fmt.Errorf("unknown batch '%s'", batchName) + } + + entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + return err + } + + if len(entries) != batch.Quantity { + return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) + } + + log.Printf("Found %d profiles\n", len(entries)) + + // XXX Is this really necessary? I don't think so + var mutex = &sync.Mutex{} + + var waitgroup sync.WaitGroup + + // Limit concurrency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. + + concurrency := 160 + sem := make(chan bool, concurrency) + for _, entry := range entries { + + // + // Only apply activation if not already noted in the + // database. + // + + sem <- true + + waitgroup.Add(1) + go func(entry model.SimEntry) { + + defer func() { <-sem }() + + result, err := client.GetStatus(entry.Iccid) + if err != nil { + panic(err) + } + + if result == nil { + log.Printf("ERROR: Couldn't find any status for Iccid='%s'\n", entry.Iccid) + } + + mutex.Lock() + fmt.Printf("%s, %s\n", entry.Iccid, result.State) + mutex.Unlock() + waitgroup.Done() + }(entry) + } + + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } + case "get-status": result, err := client.GetStatus(iccid) @@ -605,74 +763,6 @@ func parseCommandLine() error { sem <- true } - case "get-profile-activation-statuses-for-batch": - batchName := iccid - - log.Printf("Getting statuses for all profiles in batch named %s\n", batchName) - - batch, err := db.GetBatchByName(batchName) - if err != nil { - return fmt.Errorf("unknown batch '%s'", batchName) - } - - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) - if err != nil { - return err - } - - if len(entries) != batch.Quantity { - return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) - } - - log.Printf("Found %d profiles\n", len(entries)) - - // XXX Is this really necessary? I don't think so - var mutex = &sync.Mutex{} - - var waitgroup sync.WaitGroup - - // Limit concurrency of the for-loop below - // to 160 goroutines. The reason is that if we get too - // many we run out of file descriptors, and we don't seem to - // get much speedup after hundred or so. - - concurrency := 160 - sem := make(chan bool, concurrency) - for _, entry := range entries { - - // - // Only apply activation if not already noted in the - // database. - // - - sem <- true - - waitgroup.Add(1) - go func(entry model.SimEntry) { - - defer func() { <-sem }() - - result, err := client.GetStatus(entry.Iccid) - if err != nil { - panic(err) - } - - if result == nil { - log.Printf("ERROR: Couldn't find any status for Iccid='%s'\n", entry.Iccid) - } - - mutex.Lock() - fmt.Printf("%s, %s\n", entry.Iccid, result.State) - mutex.Unlock() - waitgroup.Done() - }(entry) - } - - waitgroup.Wait() - for i := 0; i < cap(sem); i++ { - sem <- true - } - case "set-batch-activation-codes": batchName := iccid From 264e07b857e4dd483dff8017d54e1c8b56cba139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 15:14:12 +0100 Subject: [PATCH 226/309] Remove redundant code --- .../sim-batch-management/sim-batch-mgt.go | 74 +------------------ 1 file changed, 1 insertion(+), 73 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 0f2a3d284..3ff888ef4 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -50,13 +50,9 @@ var ( es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", - "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-Iccid, get-profile-activation-statuses-for-batch, get-profile-activation-statuses-for-iccids-in-file").Required().String() + "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-Iccid, get-profile-activation-statuses-for-iccids-in-file").Required().String() es2iccid = es2.Arg("Iccid", "Iccid of profile to manipulate").String() es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() - es2CertFilePath = es2.Flag("cert", "Certificate pem file.").String() - es2KeyFilePath = es2.Flag("key", "Certificate key file.").String() - es2Hostport = es2.Flag("hostport", "host:port of ES2+ endpoint.").String() - es2RequesterId = es2.Flag("requesterid", "ES2+ requester ID.").String() es2ProfileVendor = es2.Flag("profile-vendor", "Name of profile-vendor").Required().String() @@ -553,74 +549,6 @@ func parseCommandLine() error { switch *es2cmd { - case "get-profile-activation-statuses-for-batch": - batchName := iccid - - log.Printf("Getting statuses for all profiles in batch named %s\n", batchName) - - batch, err := db.GetBatchByName(batchName) - if err != nil { - return fmt.Errorf("unknown batch '%s'", batchName) - } - - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) - if err != nil { - return err - } - - if len(entries) != batch.Quantity { - return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) - } - - log.Printf("Found %d profiles\n", len(entries)) - - // XXX Is this really necessary? I don't think so - var mutex = &sync.Mutex{} - - var waitgroup sync.WaitGroup - - // Limit concurrency of the for-loop below - // to 160 goroutines. The reason is that if we get too - // many we run out of file descriptors, and we don't seem to - // get much speedup after hundred or so. - - concurrency := 160 - sem := make(chan bool, concurrency) - for _, entry := range entries { - - // - // Only apply activation if not already noted in the - // database. - // - - sem <- true - - waitgroup.Add(1) - go func(entry model.SimEntry) { - - defer func() { <-sem }() - - result, err := client.GetStatus(entry.Iccid) - if err != nil { - panic(err) - } - - if result == nil { - log.Printf("ERROR: Couldn't find any status for Iccid='%s'\n", entry.Iccid) - } - - mutex.Lock() - fmt.Printf("%s, %s\n", entry.Iccid, result.State) - mutex.Unlock() - waitgroup.Done() - }(entry) - } - - waitgroup.Wait() - for i := 0; i < cap(sem); i++ { - sem <- true - } - case "get-status": result, err := client.GetStatus(iccid) From d25f6cc91896ebdc0a448337fcb5aa701d607408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 16:38:49 +0100 Subject: [PATCH 227/309] Pass tests --- .../sim-batch-management/model/model.go | 1 + .../sim-batch-management/store/store.go | 20 +++++++- .../sim-batch-management/store/store_test.go | 51 ++++++++++++++----- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 7f36f7531..e4d5a34e0 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -39,6 +39,7 @@ type Batch struct { IccidIncrement int `db:"iccidIncrement" json:"msisdnIncrement"` ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` + ProfileVendor string `db:"profileVendor" json:"profileVendor"` } type ProfileVendor struct { diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 0667c3a20..427abbb37 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -116,7 +116,7 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { // TODO: mutex? - res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity)", + res, err := sdb.Db.NamedExec("INSERT INTO BATCH (name, filenameBase, orderDate, customer, profileType, batchNo, quantity, profileVendor) values (:name, :filenameBase, :orderDate, :customer, :profileType, :batchNo, :quantity, :profileVendor)", theBatch, ) @@ -142,6 +142,7 @@ func (sdb *SimBatchDB) GenerateTables() error { sql := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, + profileVendor VARCHAR NOT NULL, filenameBase VARCHAR, customer VARCHAR, profileType VARCHAR, @@ -191,6 +192,12 @@ func (sdb *SimBatchDB) GenerateTables() error { func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { // TODO: This insert string can be made through reflection, and at some point should be. + + vendor, _ := sdb.GetProfileVendorByName(theEntry.Name) + if vendor != nil { + return fmt.Errorf("Duplicate profile vendor named %s, %v\n", theEntry.Name, vendor) + } + res, err := sdb.Db.NamedExec(` INSERT INTO PROFILE_VENDOR (name, es2PlusCertPath, es2PlusKeyPath, es2PlusHostPath, es2PlusPort, es2PlusRequesterId) VALUES (:name, :es2PlusCertPath, :es2PlusKeyPath, :es2PlusHostPath, :es2PlusPort, :es2PlusRequesterId)`, @@ -313,10 +320,18 @@ func (sdb SimBatchDB) DeclareBatch( batchLengthString string, hssVendor string, uploadHostname string, - uploadPortnumber string, + uploadPortnumber string, profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { + vendor, err := sdb.GetProfileVendorByName(profileVendor) + if err != nil { + return nil, err + } + if vendor == nil { + return nil, fmt.Errorf("unknown profile vendor: '%s'", profileVendor) + } + // TODO: // 1. Check all the arguments (methods already written). // 2. Check that the name isn't already registred. @@ -408,6 +423,7 @@ func (sdb SimBatchDB) DeclareBatch( ImsiIncrement: loltelutils.Sign(imsiLen), FirstMsisdn: firstMsisdn, MsisdnIncrement: msisdnIncrement, + ProfileVendor: profileVendor, } tx := sdb.Begin() diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index b66613d95..8003562f2 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -10,9 +10,10 @@ import ( "testing" ) - -var sdb *SimBatchDB -var sdbSetupError error +var ( + sdb *SimBatchDB + sdbSetupError error +) func TestMain(m *testing.M) { setup() @@ -30,7 +31,8 @@ func setup() { // delete file, ignore any errors os.Remove(filename) - sdb, sdbSetupError = OpenFileSqliteDatabase(filename) + sdb, sdbSetupError = + OpenFileSqliteDatabase(filename) if sdbSetupError != nil { panic(fmt.Errorf("Couldn't open new in memory database '%s", sdbSetupError)) @@ -48,6 +50,7 @@ func setup() { } func cleanTables() { + fmt.Println("Cleaning tables ...") _, err := sdb.Db.Exec("DELETE FROM SIM_PROFILE") if err != nil { panic(fmt.Sprintf("Couldn't delete SIM_PROFILE '%s'", err)) @@ -57,6 +60,18 @@ func cleanTables() { if err != nil { panic(fmt.Sprintf("Couldn't delete BATCH '%s'", err)) } + + _, err = sdb.Db.Exec("DELETE FROM PROFILE_VENDOR") + if err != nil { + panic(fmt.Sprintf("Couldn't delete PROFILE_VENDOR '%s'", err)) + } + fmt.Println(" Cleaned tables ...") + + vendor, _ := sdb.GetProfileVendorByName("Durian") + if vendor != nil { + fmt.Println(" Unclean Durian detected") + } + } func shutdown() { @@ -85,6 +100,7 @@ func injectTestBatch() *model.Batch { Url: "http://vg.no", FirstIccid: "1234567890123456789", FirstImsi: "123456789012345", + ProfileVendor: "Durian", MsisdnIncrement: -1, } @@ -104,7 +120,7 @@ func TestGetBatchById(t *testing.T) { cleanTables() batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") - if batch.BatchId != -1 { + if batch.BatchId != -1 { // TODO: REWRITE TO USE NULL TESTS INSTEAD!! t.Errorf("Duplicate detected, error in test setup") } @@ -161,16 +177,17 @@ func declareTestBatch(t *testing.T) *model.Batch { "LOL", // hssVendor string, "localhost", // uploadHostname string, "8088", // uploadPortnumber string, - "snuff", // profileVendor string, + "Durian", // profileVendor string, "ACTIVE") // initialHlrActivationStatusOfProfiles string if err != nil { - t.Fatal(err) + panic(err) } return theBatch } func TestDeclareBatch(t *testing.T) { + injectTestprofileVendor(t) theBatch := declareTestBatch(t) retrievedValue, _ := sdb.GetBatchById(theBatch.BatchId) if !reflect.DeepEqual(*retrievedValue, *theBatch) { @@ -187,10 +204,10 @@ func TestDeclareBatch(t *testing.T) { } -func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { - cleanTables() + +func injectTestprofileVendor( t *testing.T) *model.ProfileVendor { v := &model.ProfileVendor{ - Name: "MyName", + Name: "Durian", Es2PlusCert: "cert", Es2PlusKey: "key", Es2PlusHost: "host", @@ -199,10 +216,19 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { } if err := sdb.CreateProfileVendor(v); err != nil { - t.Fatal(err) + panic(err) + // t.Fatal(err) } + return v +} + +func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { + cleanTables() + + v := injectTestprofileVendor(t) + - nameRetrievedVendor,err := sdb.GetProfileVendorByName("MyName") + nameRetrievedVendor,err := sdb.GetProfileVendorByName(v.Name) if err != nil { t.Fatal(err) } @@ -223,6 +249,7 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { func TestDeclareAndRetrieveSimEntries(t *testing.T) { cleanTables() + injectTestprofileVendor(t) theBatch := declareTestBatch(t) batchId := theBatch.BatchId From 13d8ba91c50d4088cf98aad5f3a96a91a002d2f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 16:54:06 +0100 Subject: [PATCH 228/309] Prettifying codetop --- .../sim-batch-management/store/store.go | 47 +++++++++++++------ .../sim-batch-management/store/store_test.go | 4 +- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 427abbb37..95d9eb605 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -14,8 +14,6 @@ import ( "strconv" ) - - type SimBatchDB struct { Db *sqlx.DB } @@ -67,8 +65,6 @@ func (sdb *SimBatchDB) Begin() *sql.Tx { return tx } - - func NewInMemoryDatabase() (*SimBatchDB, error) { db, err := sqlx.Connect("sqlite3", ":memory:") if err != nil { @@ -103,14 +99,21 @@ func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { } func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { - var result model.Batch - return &result, sdb.Db.Get(&result, "select * from BATCH where id = ?", id) + result := []model.Batch{} + if err := sdb.Db.Get(&result, "select * from BATCH where where id = ?", id); err != nil { + return nil, err + } else { + return &(result[0]), nil + } } func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { - var result model.Batch - result.BatchId = -1 - return &result, sdb.Db.Get(&result, "select * from BATCH where name = ?", name) + result := []model.Batch{} + if err := sdb.Db.Get(&result, "select * from BATCH where name = ?", name); err != nil { + return nil, err + } else { + return &(result[0]), nil + } } func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { @@ -195,7 +198,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { vendor, _ := sdb.GetProfileVendorByName(theEntry.Name) if vendor != nil { - return fmt.Errorf("Duplicate profile vendor named %s, %v\n", theEntry.Name, vendor) + return fmt.Errorf("duplicate profile vendor named %s, %v", theEntry.Name, vendor) } res, err := sdb.Db.NamedExec(` @@ -263,13 +266,29 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { } func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { - var result model.SimEntry - return &result, sdb.Db.Get(&result, "select * from SIM_PROFILE where id = ?", simId) + result := []model.SimEntry{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", simId); err != nil { + return nil, err + } + + if len(result) == 0 { + return nil, nil + } else { + return &result[0], nil + } } func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) { result := []model.SimEntry{} - return result, sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId) + if err := sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId) ; err != nil { + return nil, err + } + + if len(result) == 0 { + return nil, nil + } else { + return result, nil + } } func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { @@ -320,7 +339,7 @@ func (sdb SimBatchDB) DeclareBatch( batchLengthString string, hssVendor string, uploadHostname string, - uploadPortnumber string, + uploadPortnumber string, profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 8003562f2..2730cc023 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -105,7 +105,7 @@ func injectTestBatch() *model.Batch { } batch, _ := sdb.GetBatchByName(theBatch.Name) - if batch.BatchId != -1 { + if batch != nil { panic(fmt.Errorf("Duplicate batch detected '%s'", theBatch.Name)) } @@ -120,7 +120,7 @@ func TestGetBatchById(t *testing.T) { cleanTables() batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") - if batch.BatchId != -1 { // TODO: REWRITE TO USE NULL TESTS INSTEAD!! + if batch != nil { // TODO: REWRITE TO USE NULL TESTS INSTEAD!! t.Errorf("Duplicate detected, error in test setup") } From 270384f550e3e0d795cbf29ce49c337ce31651e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 19:45:53 +0100 Subject: [PATCH 229/309] Passing tests now --- .../sim-batch-management/store/store.go | 12 +++++++++-- .../sim-batch-management/store/store_test.go | 20 +++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 95d9eb605..58e6c435a 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -100,18 +100,26 @@ func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { result := []model.Batch{} - if err := sdb.Db.Get(&result, "select * from BATCH where where id = ?", id); err != nil { + if err := sdb.Db.Select(&result, "SELECT * FROM BATCH WHERE id = ?", id); err != nil { return nil, err + } else if len(result) == 0 { + fmt.Println("returning null") + return nil, nil } else { + fmt.Println("returning batch: ", result[0]) return &(result[0]), nil } } func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { result := []model.Batch{} - if err := sdb.Db.Get(&result, "select * from BATCH where name = ?", name); err != nil { + if err := sdb.Db.Select(&result, "select * from BATCH where name = ?", name); err != nil { return nil, err + } else if len(result) == 0 { + // fmt.Println("GetBatchByName: returning null while searching for ", name) + return nil, nil } else { + fmt.Println("returning batch: ", result[0]) return &(result[0]), nil } } diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 2730cc023..0217221ef 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -118,16 +118,22 @@ func injectTestBatch() *model.Batch { func TestGetBatchById(t *testing.T) { + fmt.Print("TestGetBatchById starts") cleanTables() batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") - if batch != nil { // TODO: REWRITE TO USE NULL TESTS INSTEAD!! + if batch != nil { t.Errorf("Duplicate detected, error in test setup") } + // Inject a sample batch + injectTestprofileVendor(t) theBatch := injectTestBatch() - firstInputBatch, _ := sdb.GetBatchById(theBatch.BatchId) - if !reflect.DeepEqual(*firstInputBatch, *theBatch) { + batchById, _ := sdb.GetBatchById(theBatch.BatchId) + if !reflect.DeepEqual(batchById, theBatch) { + + t.Logf("theBatch = %v\n", theBatch) + t.Logf("batchById = %v\n", batchById) t.Errorf("getBatchById failed") } } @@ -189,8 +195,14 @@ func declareTestBatch(t *testing.T) *model.Batch { func TestDeclareBatch(t *testing.T) { injectTestprofileVendor(t) theBatch := declareTestBatch(t) + + fmt.Println("TestDeclareBatch: The batch id =", theBatch.BatchId) + retrievedValue, _ := sdb.GetBatchById(theBatch.BatchId) - if !reflect.DeepEqual(*retrievedValue, *theBatch) { + if retrievedValue == nil { + t.Fatalf("Null retrievedValue") + } + if !reflect.DeepEqual(retrievedValue, theBatch) { t.Fatal("getBatchById failed, stored batch not equal to retrieved batch") } From 0dc1e37a6ca08eb66f048f6f203afdd902ea667c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 19:46:51 +0100 Subject: [PATCH 230/309] Passing tests now --- sim-administration/sim-batch-management/store/store.go | 1 - sim-administration/sim-batch-management/store/store_test.go | 3 --- 2 files changed, 4 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 58e6c435a..b4dcb00d3 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -116,7 +116,6 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { if err := sdb.Db.Select(&result, "select * from BATCH where name = ?", name); err != nil { return nil, err } else if len(result) == 0 { - // fmt.Println("GetBatchByName: returning null while searching for ", name) return nil, nil } else { fmt.Println("returning batch: ", result[0]) diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 0217221ef..10cb311fe 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -195,8 +195,6 @@ func declareTestBatch(t *testing.T) *model.Batch { func TestDeclareBatch(t *testing.T) { injectTestprofileVendor(t) theBatch := declareTestBatch(t) - - fmt.Println("TestDeclareBatch: The batch id =", theBatch.BatchId) retrievedValue, _ := sdb.GetBatchById(theBatch.BatchId) if retrievedValue == nil { @@ -213,7 +211,6 @@ func TestDeclareBatch(t *testing.T) { assert.Equal(t, 1, len(retrievedEntries)) // TODO: Add check for content of retrieved entity - } From ce6e3a2bef6a7ed9c34b69da959a0af8ff6c4044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 20:13:29 +0100 Subject: [PATCH 231/309] Move generation of client into a functeion --- .../sim-batch-management/es2plus/es2plus.go | 2 +- .../sim-batch-management/sim-batch-mgt.go | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 8f441f4f3..d89c387bc 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -154,7 +154,7 @@ type Es2PlusClientState struct { logHeaders bool } -func Client(certFilePath string, keyFilePath string, hostport string, requesterId string) *Es2PlusClientState { +func NewClient(certFilePath string, keyFilePath string, hostport string, requesterId string) *Es2PlusClientState { return &Es2PlusClientState{ httpClient: newHttpClient(certFilePath, keyFilePath), hostport: hostport, diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 3ff888ef4..faa47af78 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -234,19 +234,11 @@ func parseCommandLine() error { profileVendorName := "Idemia" // TODO Should have been profileVendorName := batdh.ProfileVendor - vendor, err := db.GetProfileVendorByName(profileVendorName) + + client, err := ClientForVendor(db, profileVendorName) if err != nil { return err } - if vendor == nil { - return fmt.Errorf("unknown profile vendor '%s'", profileVendorName) - } - - - hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) - client := es2plus.Client(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) - - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { @@ -537,18 +529,18 @@ func parseCommandLine() error { if err != nil { return err } + if vendor == nil { return fmt.Errorf("unknown profile vendor '%s'", *es2ProfileVendor) } hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) - client := es2plus.Client(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) + client := es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) iccid := *es2iccid switch *es2cmd { - case "get-status": result, err := client.GetStatus(iccid) @@ -845,3 +837,17 @@ func GenerateInputFile(batch *model.Batch) string { "var_Out: ICCID/IMSI/KI\n" return result } + +func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Es2PlusClient, error) { + vendor, err := db.GetProfileVendorByName(vendorName) + if err != nil { + return nil, err + } + + if vendor == nil { + return nil, fmt.Errorf("unknown profile vendor '%s'", vendorName) + } + + hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) + return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId), nil +} \ No newline at end of file From d400c99208547b5c24b1dcf323d72445b413a8e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 20:24:00 +0100 Subject: [PATCH 232/309] Remove monkeypatch --- sim-administration/sim-batch-management/model/model.go | 2 +- sim-administration/sim-batch-management/sim-batch-mgt.go | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index e4d5a34e0..e87490978 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -39,7 +39,7 @@ type Batch struct { IccidIncrement int `db:"iccidIncrement" json:"msisdnIncrement"` ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` - ProfileVendor string `db:"profileVendor" json:"profileVendor"` + ProfileVendor string `db:"profileVendor" json:"profileVendor"` } type ProfileVendor struct { diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index faa47af78..c248c4021 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -229,13 +229,8 @@ func parseCommandLine() error { if err != nil { return fmt.Errorf("unknown batch '%s'", batchName) } - - // This is a monkeypatch - profileVendorName := "Idemia" - // TODO Should have been profileVendorName := batdh.ProfileVendor - - - client, err := ClientForVendor(db, profileVendorName) + + client, err := ClientForVendor(db, batch.ProfileVendor) if err != nil { return err } From 454f4b253d48af551f45825c89edbc17ccaab740 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 20:57:35 +0100 Subject: [PATCH 233/309] Add a couple of more commands to the list of top-leve commands with simplified syntax --- .../sim-batch-management/sim-batch-mgt.go | 66 ++++++++++++------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index c248c4021..4f67bd225 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -38,6 +38,14 @@ var ( dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() dpvRequesterId = dpv.Flag("requester-id", "ES2+ requester ID.").Required().String() + getStatus = kingpin.Command("get-status", "Get status for an iccid") + getStatusProfileVendor = getStatus.Flag("profile-vendor", "Name of profile vendor").Required().String() + getStatusProfileIccid = getStatus.Arg("iccid", "Iccid to get status for").Required().String() + + recoverProfile = kingpin.Command("recover-profile", "Change state of profile.") + recoverProfileVendor = recoverProfile.Flag("profile-vendor", "Name of profile vendor").Required().String() + recoverProfileIccid = recoverProfile.Flag("iccid", "Iccid to recover profile for").Required().String() + recoverProfileTarget = recoverProfile.Flag("target-state", "Desired target state").Required().String() // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files @@ -57,7 +65,7 @@ var ( getProfActActStatusesForBatch = kingpin.Command("get-profile-activation-statuses-for-batch", "Get current activation statuses from SM-DP+ for named batch.") - getProfActActStatusesForBatchBatch =getProfActActStatusesForBatch.Arg("batcgh", "The batch to get activation statuses for.").Required().String() + getProfActActStatusesForBatchBatch = getProfActActStatusesForBatch.Arg("batcgh", "The batch to get activation statuses for.").Required().String() // // Convert an output (.out) file from an sim profile producer into an input file // for Prime. @@ -229,7 +237,7 @@ func parseCommandLine() error { if err != nil { return fmt.Errorf("unknown batch '%s'", batchName) } - + client, err := ClientForVendor(db, batch.ProfileVendor) if err != nil { return err @@ -518,6 +526,35 @@ func parseCommandLine() error { *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) + + case "get-status": + client, err := ClientForVendor(db, *getStatusProfileVendor) + if err != nil { + return err + } + + result, err := client.GetStatus(*getStatusProfileIccid) + if err != nil { + return err + } + log.Printf("Iccid='%s', state='%s', acToken='%s'\n", *getStatusProfileIccid, (*result).State, (*result).ACToken) + + case "recover-profile": + client, err := ClientForVendor(db, *recoverProfileVendor) + if err != nil { + return err + } + + err = checkEs2TargetState(*recoverProfileTarget) + if err != nil { + return err + } + result, err := client.RecoverProfile(*recoverProfileIccid, *recoverProfileTarget) + if err != nil { + return err + } + log.Println("result -> ", result) + case "es2": vendor, err := db.GetProfileVendorByName(*es2ProfileVendor) @@ -529,31 +566,12 @@ func parseCommandLine() error { return fmt.Errorf("unknown profile vendor '%s'", *es2ProfileVendor) } - hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) client := es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) iccid := *es2iccid switch *es2cmd { - case "get-status": - - result, err := client.GetStatus(iccid) - if err != nil { - return err - } - - log.Printf("Iccid='%s', state='%s', acToken='%s'\n", iccid, (*result).State, (*result).ACToken) - case "recover-profile": - err := checkEs2TargetState(es2Target) - if err != nil { - return err - } - result, err := client.RecoverProfile(iccid, *es2Target) - if err != nil { - return err - } - log.Println("result -> ", result) case "download-order": result, err := client.DownloadOrder(iccid) if err != nil { @@ -779,7 +797,7 @@ func parseCommandLine() error { } case "cancel-profile": - err := checkEs2TargetState(es2Target) + err := checkEs2TargetState(*es2Target) if err != nil { return err } @@ -800,8 +818,8 @@ func parseCommandLine() error { return nil } -func checkEs2TargetState(target *string) error { - if *target != "AVAILABLE" { +func checkEs2TargetState(target string) error { + if target != "AVAILABLE" { return fmt.Errorf("target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") } else { return nil From 5a4b03f870c8045489cfe51db2650b86733b1954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 21:20:58 +0100 Subject: [PATCH 234/309] More commands on the top level --- .../sim-batch-management/sim-batch-mgt.go | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 4f67bd225..759d655ae 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -47,6 +47,20 @@ var ( recoverProfileIccid = recoverProfile.Flag("iccid", "Iccid to recover profile for").Required().String() recoverProfileTarget = recoverProfile.Flag("target-state", "Desired target state").Required().String() + downloadOrder = kingpin.Command("download-order", "Execute es2p download-order.") + downloadOrderVendor = downloadOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() + downloadOrderIccid = downloadOrder.Flag("iccid", "Iccid to recover profile for").Required().String() + + confirmOrder = kingpin.Command("confirm-order", "Execute es2p confirm-order.") + confirmOrderVendor = confirmOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() + confirmOrderIccid = confirmOrder.Flag("iccid", "Iccid to confirm profile for").Required().String() + + activateIccid = kingpin.Command("activate-iccid", "Execute es2p confirm-order.") + activateIccidVendor = activateIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() + activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() + + + // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files // TODO: Declare hss-es, that can be refered to in profiles. @@ -302,7 +316,6 @@ func parseCommandLine() error { } case "sim-profile-upload": - inputFile := *spUploadInputFile outputFilePrefix := *spUploadOutputFilePrefix @@ -315,7 +328,6 @@ func parseCommandLine() error { } case "list-batches": - allBatches, err := db.GetAllBatches() if err != nil { return err @@ -555,6 +567,42 @@ func parseCommandLine() error { } log.Println("result -> ", result) + case "download-order": + client, err := ClientForVendor(db, *downloadOrderVendor) + if err != nil { + return err + } + result, err := client.DownloadOrder(*downloadOrderIccid) + if err != nil { + return err + } + log.Println("result -> ", result) + + case "confirm-order": + client, err := ClientForVendor(db, *confirmOrderVendor) + if err != nil { + return err + } + result, err := client.ConfirmOrder(*confirmOrderIccid) + if err != nil { + return err + } + fmt.Println("result -> ", result) + + case "activate-iccid": + client, err := ClientForVendor(db, *activateIccidVendor) + if err != nil { + return err + } + + result, err := client.ActivateIccid(*activateIccidIccid) + + if err != nil { + return err + } + fmt.Printf("%s, %s\n", *activateIccidIccid, result.ACToken) + + case "es2": vendor, err := db.GetProfileVendorByName(*es2ProfileVendor) @@ -572,25 +620,8 @@ func parseCommandLine() error { iccid := *es2iccid switch *es2cmd { - case "download-order": - result, err := client.DownloadOrder(iccid) - if err != nil { - return err - } - log.Println("result -> ", result) - case "confirm-order": - result, err := client.ConfirmOrder(iccid) - if err != nil { - return err - } - fmt.Println("result -> ", result) - case "activate-Iccid": - result, err := client.ActivateIccid(iccid) - if err != nil { - return err - } - fmt.Printf("%s, %s\n", iccid, result.ACToken) + case "get-profile-activation-statuses-for-iccids-in-file": csvFilename := iccid From b4e59f10a122311438cd92207e489929aeac33ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 21:24:49 +0100 Subject: [PATCH 235/309] Fix bug, command parser was done wrong --- .../sim-batch-management/sim-batch-mgt.go | 49 +++++++------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 759d655ae..182ca618e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -42,24 +42,22 @@ var ( getStatusProfileVendor = getStatus.Flag("profile-vendor", "Name of profile vendor").Required().String() getStatusProfileIccid = getStatus.Arg("iccid", "Iccid to get status for").Required().String() - recoverProfile = kingpin.Command("recover-profile", "Change state of profile.") + recoverProfile = kingpin.Command("recover-profile", "Change state of profile.") recoverProfileVendor = recoverProfile.Flag("profile-vendor", "Name of profile vendor").Required().String() - recoverProfileIccid = recoverProfile.Flag("iccid", "Iccid to recover profile for").Required().String() + recoverProfileIccid = recoverProfile.Flag("iccid", "Iccid to recover profile for").Required().String() recoverProfileTarget = recoverProfile.Flag("target-state", "Desired target state").Required().String() - downloadOrder = kingpin.Command("download-order", "Execute es2p download-order.") + downloadOrder = kingpin.Command("download-order", "Execute es2p download-order.") downloadOrderVendor = downloadOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() - downloadOrderIccid = downloadOrder.Flag("iccid", "Iccid to recover profile for").Required().String() + downloadOrderIccid = downloadOrder.Flag("iccid", "Iccid to recover profile for").Required().String() - confirmOrder = kingpin.Command("confirm-order", "Execute es2p confirm-order.") + confirmOrder = kingpin.Command("confirm-order", "Execute es2p confirm-order.") confirmOrderVendor = confirmOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() - confirmOrderIccid = confirmOrder.Flag("iccid", "Iccid to confirm profile for").Required().String() + confirmOrderIccid = confirmOrder.Flag("iccid", "Iccid to confirm profile for").Required().String() - activateIccid = kingpin.Command("activate-iccid", "Execute es2p confirm-order.") + activateIccid = kingpin.Command("activate-iccid", "Execute es2p confirm-order.") activateIccidVendor = activateIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() - activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() - - + activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files @@ -73,12 +71,11 @@ var ( es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") es2cmd = es2.Arg("cmd", "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-Iccid, get-profile-activation-statuses-for-iccids-in-file").Required().String() - es2iccid = es2.Arg("Iccid", "Iccid of profile to manipulate").String() - es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() + es2iccid = es2.Arg("Iccid", "Iccid of profile to manipulate").String() + es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() es2ProfileVendor = es2.Flag("profile-vendor", "Name of profile-vendor").Required().String() - - getProfActActStatusesForBatch = kingpin.Command("get-profile-activation-statuses-for-batch", "Get current activation statuses from SM-DP+ for named batch.") + getProfActActStatusesForBatch = kingpin.Command("get-profile-activation-statuses-for-batch", "Get current activation statuses from SM-DP+ for named batch.") getProfActActStatusesForBatchBatch = getProfActActStatusesForBatch.Arg("batcgh", "The batch to get activation statuses for.").Required().String() // // Convert an output (.out) file from an sim profile producer into an input file @@ -223,16 +220,16 @@ func parseCommandLine() error { if err != nil { return err } - absDpvKeyFilePath, err := filepath.Abs(*dpvKeyFilePath) + absDpvKeyFilePath, err := filepath.Abs(*dpvKeyFilePath) if err != nil { return err } v := &model.ProfileVendor{ - Name: *dpvName, - Es2PlusCert: absDpvCertFilePath, - Es2PlusKey: absDpvKeyFilePath , - Es2PlusHost: *dpvHost, - Es2PlusPort: *dpvPort, + Name: *dpvName, + Es2PlusCert: absDpvCertFilePath, + Es2PlusKey: absDpvKeyFilePath, + Es2PlusHost: *dpvHost, + Es2PlusPort: *dpvPort, Es2PlusRequesterId: *dpvRequesterId, } @@ -252,7 +249,7 @@ func parseCommandLine() error { return fmt.Errorf("unknown batch '%s'", batchName) } - client, err := ClientForVendor(db, batch.ProfileVendor) + client, err := ClientForVendor(db, batch.ProfileVendor) if err != nil { return err } @@ -538,7 +535,6 @@ func parseCommandLine() error { *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) - case "get-status": client, err := ClientForVendor(db, *getStatusProfileVendor) if err != nil { @@ -602,7 +598,6 @@ func parseCommandLine() error { } fmt.Printf("%s, %s\n", *activateIccidIccid, result.ACToken) - case "es2": vendor, err := db.GetProfileVendorByName(*es2ProfileVendor) @@ -620,9 +615,6 @@ func parseCommandLine() error { iccid := *es2iccid switch *es2cmd { - - - case "get-profile-activation-statuses-for-iccids-in-file": csvFilename := iccid @@ -839,9 +831,6 @@ func parseCommandLine() error { default: return fmt.Errorf("unknown es2+ subcommand '%s', try --help", *es2cmd) } - case "batch": - fmt.Println("Doing the batch thing.") - // storage.doTheBatchThing() default: return fmt.Errorf("unknown command: '%s'", cmd) } @@ -894,4 +883,4 @@ func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Es2PlusCl hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId), nil -} \ No newline at end of file +} From 837bcd315804c4018d7e9c08d01110e076a04b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 21:48:45 +0100 Subject: [PATCH 236/309] Intermediate checkin, a little more cleanup necessary --- .../sim-batch-management/sim-batch-mgt.go | 360 +++++++++--------- 1 file changed, 188 insertions(+), 172 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 182ca618e..c0d0d049e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -59,6 +59,23 @@ var ( activateIccidVendor = activateIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() + activateIccidFile = kingpin.Command("activate-iccids-from-file", "Execute es2p confirm-order.") + activateIccidFileVendor = activateIccidFile.Flag("profile-vendor", "Name of profile vendor").Required().String() + activateIccidFileFile = activateIccidFile.Flag("iccid-file", "Iccid to confirm profile for").Required().String() + + setBatchActivationCodes = kingpin.Command("set-batch-activation-codes", "Execute es2p confirm-order.") + setBatchActivationCodesVendor = setBatchActivationCodes.Flag("profile-vendor", "Name of profile vendor").Required().String() + setBatchActivationCodesFile = setBatchActivationCodes.Flag("iccid-file", "Iccid to confirm profile for").Required().String() + + bulkActivateIccids = kingpin.Command("bulk-activate-iccids", "Execute es2p confirm-order.") + bulkActivateIccidsVendor = bulkActivateIccids.Flag("profile-vendor", "Name of profile vendor").Required().String() + bulkActivateIccidsIccids = bulkActivateIccids.Flag("iccid-file", "Iccid to confirm profile for").Required().String() + + cancelIccid = kingpin.Command("cancel-iccid", "Execute es2p confirm-order.") + cancelIccidIccid = cancelIccid.Flag("iccid", "The iccid to cancel").Required().String() + cancelIccidVendor = cancelIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() + cancelIccidTarget = cancelIccid.Flag("target", "Tarfget t set profile to").Required().String() + // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files // TODO: Declare hss-es, that can be refered to in profiles. @@ -598,239 +615,238 @@ func parseCommandLine() error { } fmt.Printf("%s, %s\n", *activateIccidIccid, result.ACToken) - case "es2": - - vendor, err := db.GetProfileVendorByName(*es2ProfileVendor) + case "cancel-iccid": + client, err := ClientForVendor(db, *activateIccidVendor) + if err != nil { + return err + } + err = checkEs2TargetState(*cancelIccidTarget) + if err != nil { + return err + } + _, err = client.CancelOrder(*cancelIccidIccid, *cancelIccidTarget) if err != nil { return err } - if vendor == nil { - return fmt.Errorf("unknown profile vendor '%s'", *es2ProfileVendor) + case "activate-iccids-from-file": + client, err := ClientForVendor(db, *activateIccidFileVendor) + if err != nil { + return err } - hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) - client := es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId) + csvFilename := *activateIccidFileFile - iccid := *es2iccid - switch *es2cmd { + csvFile, _ := os.Open(csvFilename) + reader := csv.NewReader(bufio.NewReader(csvFile)) - case "get-profile-activation-statuses-for-iccids-in-file": - csvFilename := iccid + defer csvFile.Close() - csvFile, _ := os.Open(csvFilename) - reader := csv.NewReader(bufio.NewReader(csvFile)) + headerLine, error := reader.Read() + if error == io.EOF { + break + } else if error != nil { + log.Fatal(error) + } - defer csvFile.Close() + var columnMap map[string]int + columnMap = make(map[string]int) - headerLine, error := reader.Read() - if error == io.EOF { - break - } else if error != nil { - log.Fatal(error) - } + for index, fieldname := range headerLine { + columnMap[strings.TrimSpace(strings.ToLower(fieldname))] = index + } - var columnMap map[string]int - columnMap = make(map[string]int) + if _, hasIccid := columnMap["iccid"]; !hasIccid { + return fmt.Errorf("no ICCID column in CSV file") + } - for index, fieldname := range headerLine { - columnMap[strings.TrimSpace(strings.ToLower(fieldname))] = index - } + type csvRecord struct { + Iccid string + } - if _, hasIccid := columnMap["iccid"]; !hasIccid { - return fmt.Errorf("no ICCID column in CSV file") - } + var recordMap map[string]csvRecord + recordMap = make(map[string]csvRecord) - type csvRecord struct { - Iccid string + // Read all the lines into the record map. + for { + line, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return err } - var recordMap map[string]csvRecord - recordMap = make(map[string]csvRecord) - - // Read all the lines into the record map. - for { - line, err := reader.Read() - if err == io.EOF { - break - } else if err != nil { - return err - } - - iccid := line[columnMap["Iccid"]] - iccid = strings.TrimSpace(iccid) - - record := csvRecord{ - Iccid: iccid, - } - - if _, duplicateRecordExists := recordMap[record.Iccid]; duplicateRecordExists { - return fmt.Errorf("duplicate ICCID record in map: %s", record.Iccid) - } + iccid := line[columnMap["Iccid"]] + iccid = strings.TrimSpace(iccid) - recordMap[record.Iccid] = record + record := csvRecord{ + Iccid: iccid, } - // XXX Is this really necessary? I don't think so - var mutex = &sync.Mutex{} - - var waitgroup sync.WaitGroup - - // Limit concurrency of the for-loop below - // to 160 goroutines. The reason is that if we get too - // many we run out of file descriptors, and we don't seem to - // get much speedup after hundred or so. + if _, duplicateRecordExists := recordMap[record.Iccid]; duplicateRecordExists { + return fmt.Errorf("duplicate ICCID record in map: %s", record.Iccid) + } - concurrency := 160 - sem := make(chan bool, concurrency) - fmt.Printf("%s, %s\n", "ICCID", "STATE") - for _, entry := range recordMap { + recordMap[record.Iccid] = record + } - // - // Only apply activation if not already noted in the - // database. - // + // XXX Is this really necessary? I don't think so + var mutex = &sync.Mutex{} - sem <- true + var waitgroup sync.WaitGroup - waitgroup.Add(1) - go func(entry csvRecord) { + // Limit concurrency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. - defer func() { <-sem }() + concurrency := 160 + sem := make(chan bool, concurrency) + fmt.Printf("%s, %s\n", "ICCID", "STATE") + for _, entry := range recordMap { - result, err := client.GetStatus(entry.Iccid) - if err != nil { - panic(err) - } + // + // Only apply activation if not already noted in the + // database. + // - if result == nil { - panic(fmt.Sprintf("Couldn't find any status for Iccid='%s'\n", entry.Iccid)) - } + sem <- true - mutex.Lock() - fmt.Printf("%s, %s\n", entry.Iccid, result.State) - mutex.Unlock() - waitgroup.Done() - }(entry) - } + waitgroup.Add(1) + go func(entry csvRecord) { - waitgroup.Wait() - for i := 0; i < cap(sem); i++ { - sem <- true - } + defer func() { <-sem }() - case "set-batch-activation-codes": - batchName := iccid + result, err := client.GetStatus(entry.Iccid) + if err != nil { + panic(err) + } - fmt.Printf("Getting batch named %s\n", batchName) + if result == nil { + panic(fmt.Sprintf("Couldn't find any status for Iccid='%s'\n", entry.Iccid)) + } - batch, err := db.GetBatchByName(batchName) - if err != nil { - return fmt.Errorf("unknown batch '%s'", batchName) - } + mutex.Lock() + fmt.Printf("%s, %s\n", entry.Iccid, result.State) + mutex.Unlock() + waitgroup.Done() + }(entry) + } - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) - if err != nil { - return err - } + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } - if len(entries) != batch.Quantity { - return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) - } + case "set-batch-activation-codes": + client, err := ClientForVendor(db, *setBatchActivationCodesVendor) + if err != nil { + return err + } - // XXX Is this really necessary? I don't think so - var mutex = &sync.Mutex{} + batchName := *setBatchActivationCodesVendor - var waitgroup sync.WaitGroup + fmt.Printf("Getting batch named %s\n", batchName) - // Limit concurrency of the for-loop below - // to 160 goroutines. The reason is that if we get too - // many we run out of file descriptors, and we don't seem to - // get much speedup after hundred or so. + batch, err := db.GetBatchByName(batchName) + if err != nil { + return fmt.Errorf("unknown batch '%s'", batchName) + } - concurrency := 160 - sem := make(chan bool, concurrency) - tx := db.Begin() - for _, entry := range entries { + entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + if err != nil { + return err + } - // - // Only apply activation if not already noted in the - // database. + if len(entries) != batch.Quantity { + return fmt.Errorf("batch quantity retrieved from database (%d) different from batch quantity (%d)", len(entries), batch.Quantity) + } - if entry.ActivationCode == "" { + // XXX Is this really necessary? I don't think so + var mutex = &sync.Mutex{} - sem <- true + var waitgroup sync.WaitGroup - waitgroup.Add(1) - go func(entry model.SimEntry) { + // Limit concurrency of the for-loop below + // to 160 goroutines. The reason is that if we get too + // many we run out of file descriptors, and we don't seem to + // get much speedup after hundred or so. - defer func() { <-sem }() + concurrency := 160 + sem := make(chan bool, concurrency) + tx := db.Begin() + for _, entry := range entries { - result, err := client.ActivateIccid(entry.Iccid) - if err != nil { - panic(err) - } + // + // Only apply activation if not already noted in the + // database. - mutex.Lock() - fmt.Printf("%s, %s\n", entry.Iccid, result.ACToken) - db.UpdateActivationCode(entry.Id, result.ACToken) - mutex.Unlock() - waitgroup.Done() - }(entry) - } - } + if entry.ActivationCode == "" { - waitgroup.Wait() - for i := 0; i < cap(sem); i++ { sem <- true - } - tx.Commit() - - case "bulk-activate-iccids": - - file, err := os.Open(iccid) - if err != nil { - log.Fatal(err) - } - defer file.Close() - scanner := bufio.NewScanner(file) - var mutex = &sync.Mutex{} - var waitgroup sync.WaitGroup - for scanner.Scan() { - iccid := scanner.Text() waitgroup.Add(1) - go func(i string) { + go func(entry model.SimEntry) { - result, err := client.ActivateIccid(i) + defer func() { <-sem }() + + result, err := client.ActivateIccid(entry.Iccid) if err != nil { panic(err) } + mutex.Lock() - fmt.Printf("%s, %s\n", i, result.ACToken) + fmt.Printf("%s, %s\n", entry.Iccid, result.ACToken) + db.UpdateActivationCode(entry.Id, result.ACToken) mutex.Unlock() waitgroup.Done() - }(iccid) + }(entry) } + } - waitgroup.Wait() + waitgroup.Wait() + for i := 0; i < cap(sem); i++ { + sem <- true + } + tx.Commit() - if err := scanner.Err(); err != nil { - log.Fatal(err) - } + case "bulk-activate-iccids": + client, err := ClientForVendor(db, *bulkActivateIccidsVendor) + if err != nil { + return err + } - case "cancel-profile": - err := checkEs2TargetState(*es2Target) - if err != nil { - return err - } - _, err = client.CancelOrder(iccid, *es2Target) - if err != nil { - return err - } - default: - return fmt.Errorf("unknown es2+ subcommand '%s', try --help", *es2cmd) + file, err := os.Open(*bulkActivateIccidsIccids) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + var mutex = &sync.Mutex{} + var waitgroup sync.WaitGroup + for scanner.Scan() { + iccid := scanner.Text() + waitgroup.Add(1) + go func(i string) { + + result, err := client.ActivateIccid(i) + if err != nil { + panic(err) + } + mutex.Lock() + fmt.Printf("%s, %s\n", i, result.ACToken) + mutex.Unlock() + waitgroup.Done() + }(iccid) } + + waitgroup.Wait() + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + default: return fmt.Errorf("unknown command: '%s'", cmd) } From 7cd43e5ffea70b3bebdd61703aa9925260c163ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 14 Nov 2019 23:13:56 +0100 Subject: [PATCH 237/309] Refactor client generation for vendors and batches --- .../sim-batch-management/sim-batch-mgt.go | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index c0d0d049e..8d92a3017 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -64,8 +64,7 @@ var ( activateIccidFileFile = activateIccidFile.Flag("iccid-file", "Iccid to confirm profile for").Required().String() setBatchActivationCodes = kingpin.Command("set-batch-activation-codes", "Execute es2p confirm-order.") - setBatchActivationCodesVendor = setBatchActivationCodes.Flag("profile-vendor", "Name of profile vendor").Required().String() - setBatchActivationCodesFile = setBatchActivationCodes.Flag("iccid-file", "Iccid to confirm profile for").Required().String() + setBatchActivationCodesBatch = setBatchActivationCodes.Arg("batch", "Batch to get activation codes for").Required().String() bulkActivateIccids = kingpin.Command("bulk-activate-iccids", "Execute es2p confirm-order.") bulkActivateIccidsVendor = bulkActivateIccids.Flag("profile-vendor", "Name of profile vendor").Required().String() @@ -85,12 +84,6 @@ var ( // profile/vendor/hss/prime combos are constrained. It should be possible // to specify prod/dev primes. - es2 = kingpin.Command("es2", "Do things with the ES2+ protocol") - es2cmd = es2.Arg("cmd", - "The ES2+ subcommand, one of get-status, recover-profile, download-order, confirm-order, cancel-profile, bulk-activate-iccids, activate-Iccid, get-profile-activation-statuses-for-iccids-in-file").Required().String() - es2iccid = es2.Arg("Iccid", "Iccid of profile to manipulate").String() - es2Target = es2.Arg("target-state", "Target state of recover-profile or cancel-profile command").Default("AVAILABLE").String() - es2ProfileVendor = es2.Flag("profile-vendor", "Name of profile-vendor").Required().String() getProfActActStatusesForBatch = kingpin.Command("get-profile-activation-statuses-for-batch", "Get current activation statuses from SM-DP+ for named batch.") getProfActActStatusesForBatchBatch = getProfActActStatusesForBatch.Arg("batcgh", "The batch to get activation statuses for.").Required().String() @@ -616,7 +609,7 @@ func parseCommandLine() error { fmt.Printf("%s, %s\n", *activateIccidIccid, result.ACToken) case "cancel-iccid": - client, err := ClientForVendor(db, *activateIccidVendor) + client, err := ClientForVendor(db, *cancelIccidVendor) if err != nil { return err } @@ -739,19 +732,13 @@ func parseCommandLine() error { } case "set-batch-activation-codes": - client, err := ClientForVendor(db, *setBatchActivationCodesVendor) + + batchName := *setBatchActivationCodesBatch + client, batch, err := ClientForBatch(db, batchName) if err != nil { return err } - batchName := *setBatchActivationCodesVendor - - fmt.Printf("Getting batch named %s\n", batchName) - - batch, err := db.GetBatchByName(batchName) - if err != nil { - return fmt.Errorf("unknown batch '%s'", batchName) - } entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { @@ -900,3 +887,24 @@ func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Es2PlusCl hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId), nil } + + +func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Es2PlusClient, *model.Batch, error) { + + batch, err := db.GetBatchByName(batchName) + if err != nil { + return nil, nil, err + } + if batch == nil { + return nil, nil, fmt.Errorf("unknown batch '%s'", batchName) + } + + client, err := ClientForVendor(db, batch.ProfileVendor) + if err != nil { + return nil, nil, err + } + + return client, batch, nil +} + + From e55d2aef422a151985c35869d5e8cb78ebec44d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 15 Nov 2019 13:39:50 +0100 Subject: [PATCH 238/309] Moving towards something that can be used by someone else than me, need a little documentation and we're set, maybe. --- .../sim-batch-management/TODO.md | 16 +- .../sim-batch-management/build-all.sh | 8 + .../sim-batch-management/sim-batch-mgt.go | 240 ++++++++---------- .../sim-batch-management/store/store.go | 19 +- 4 files changed, 147 insertions(+), 136 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index 29f3f1a4d..e36fc5fc2 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,8 +1,9 @@ TODO == -1. Make the build-all script run without errors (including linter errors) -1. Clean up the code a lot -1. Take pending code review comments into account. + +1. Use a different database than hardcoded "foobar.db" :-) Read it from an + environment variable or something at least a bit portable. +1. Write up a nice markdown documentation describing common usecases. 1. Create a very clean PR for future code review. 1. Compress the utility scripts into very close to nothing, by adding their functionality to the .go code. 1. Add crypto resources so that the program can talk to external parties. @@ -18,4 +19,13 @@ TODO directly from the script later. In either case it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. + + // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... + // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files + // TODO: Declare hss-es, that can be refered to in profiles. + // TODO: Declare legal hss/dpv combinations, batches must use legal combos. + // TODO: Declare contact methods for primes. It might be a good idea to + // impose referential integrity constraint on this too, so that + // profile/vendor/hss/prime combos are constrained. It should be possible + // to specify prod/dev primes. diff --git a/sim-administration/sim-batch-management/build-all.sh b/sim-administration/sim-batch-management/build-all.sh index 8860fc0b9..51e8638b3 100755 --- a/sim-administration/sim-batch-management/build-all.sh +++ b/sim-administration/sim-batch-management/build-all.sh @@ -28,3 +28,11 @@ fi ~/go/bin/staticcheck ./... +mv sim-batch-management sbm + +# If sourcing this script, then the line below +# will modify command line compesion in bash + +if [[ $_ != $0 ]] ; then + eval "$(./sbm --completion-script-bash)" +fi diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 8d92a3017..d43f02203 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -26,160 +26,137 @@ var ( // TODO: Global flags can be added to Kingpin, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() + + /// + /// Converting an output file to an upload script + /// for HSS configs. Not something you would want to use in + /// a production-like setting, but nice to have in a pinch when + /// dealing with a test system. + /// + + spUpload = kingpin.Command("hss-convert-out-file-to-upload-script", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() + spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", + "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() + + + + /// + /// Profile-vendor - centric commands + /// // Declare a profile-vendor with an SM-DP+ that can be referred to from // batches. Referential integrity required, so it won't be possible to // declare bathes with non-existing profile vendors. - dpv = kingpin.Command("declare-profile-vendor", "Declare a profile vendor with an SM-DP+ we can talk to") + dpv = kingpin.Command("profile-vendor-declare", "Declare a profile vendor with an SM-DP+ we can talk to") dpvName = dpv.Flag("name", "Name of profile-vendor").Required().String() dpvCertFilePath = dpv.Flag("cert", "Certificate pem file.").Required().String() dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() dpvRequesterId = dpv.Flag("requester-id", "ES2+ requester ID.").Required().String() + + /// + /// ICCID - centric commands + /// - getStatus = kingpin.Command("get-status", "Get status for an iccid") + getStatus = kingpin.Command("iccid-get-status", "Get status for an iccid") getStatusProfileVendor = getStatus.Flag("profile-vendor", "Name of profile vendor").Required().String() getStatusProfileIccid = getStatus.Arg("iccid", "Iccid to get status for").Required().String() - recoverProfile = kingpin.Command("recover-profile", "Change state of profile.") - recoverProfileVendor = recoverProfile.Flag("profile-vendor", "Name of profile vendor").Required().String() - recoverProfileIccid = recoverProfile.Flag("iccid", "Iccid to recover profile for").Required().String() - recoverProfileTarget = recoverProfile.Flag("target-state", "Desired target state").Required().String() - - downloadOrder = kingpin.Command("download-order", "Execute es2p download-order.") + downloadOrder = kingpin.Command("iccid-download-order", "Execute es2p download-order.") downloadOrderVendor = downloadOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() downloadOrderIccid = downloadOrder.Flag("iccid", "Iccid to recover profile for").Required().String() - confirmOrder = kingpin.Command("confirm-order", "Execute es2p confirm-order.") - confirmOrderVendor = confirmOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() - confirmOrderIccid = confirmOrder.Flag("iccid", "Iccid to confirm profile for").Required().String() + recoverProfile = kingpin.Command("iccid-recover-profile", "Change state of profile.") + recoverProfileVendor = recoverProfile.Flag("profile-vendor", "Name of profile vendor").Required().String() + recoverProfileIccid = recoverProfile.Flag("iccid", "Iccid to recover profile for").Required().String() + recoverProfileTarget = recoverProfile.Flag("target-state", "Desired target state").Required().String() - activateIccid = kingpin.Command("activate-iccid", "Execute es2p confirm-order.") - activateIccidVendor = activateIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() - activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() + cancelIccid = kingpin.Command("iccid-cancel", "Execute es2p iccid-confirm-order.") + cancelIccidIccid = cancelIccid.Flag("iccid", "The iccid to cancel").Required().String() + cancelIccidVendor = cancelIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() + cancelIccidTarget = cancelIccid.Flag("target", "Tarfget t set profile to").Required().String() - activateIccidFile = kingpin.Command("activate-iccids-from-file", "Execute es2p confirm-order.") + activateIccidFile = kingpin.Command("iccids-activate-from-file", "Execute es2p iccid-confirm-order.") activateIccidFileVendor = activateIccidFile.Flag("profile-vendor", "Name of profile vendor").Required().String() activateIccidFileFile = activateIccidFile.Flag("iccid-file", "Iccid to confirm profile for").Required().String() - setBatchActivationCodes = kingpin.Command("set-batch-activation-codes", "Execute es2p confirm-order.") - setBatchActivationCodesBatch = setBatchActivationCodes.Arg("batch", "Batch to get activation codes for").Required().String() - - bulkActivateIccids = kingpin.Command("bulk-activate-iccids", "Execute es2p confirm-order.") + bulkActivateIccids = kingpin.Command("iccids-bulk-activate", "Execute es2p iccid-confirm-order.") bulkActivateIccidsVendor = bulkActivateIccids.Flag("profile-vendor", "Name of profile vendor").Required().String() bulkActivateIccidsIccids = bulkActivateIccids.Flag("iccid-file", "Iccid to confirm profile for").Required().String() - cancelIccid = kingpin.Command("cancel-iccid", "Execute es2p confirm-order.") - cancelIccidIccid = cancelIccid.Flag("iccid", "The iccid to cancel").Required().String() - cancelIccidVendor = cancelIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() - cancelIccidTarget = cancelIccid.Flag("target", "Tarfget t set profile to").Required().String() + activateIccid = kingpin.Command("iccid-activate", "Execute es2p iccid-confirm-order.") + activateIccidVendor = activateIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() + activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() - // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... - // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files - // TODO: Declare hss-es, that can be refered to in profiles. - // TODO: Declare legal hss/dpv combinations, batches must use legal combos. - // TODO: Declare contact methods for primes. It might be a good idea to - // impose referential integrity constraint on this too, so that - // profile/vendor/hss/prime combos are constrained. It should be possible - // to specify prod/dev primes. - - - getProfActActStatusesForBatch = kingpin.Command("get-profile-activation-statuses-for-batch", "Get current activation statuses from SM-DP+ for named batch.") - getProfActActStatusesForBatchBatch = getProfActActStatusesForBatch.Arg("batcgh", "The batch to get activation statuses for.").Required().String() - // - // Convert an output (.out) file from an sim profile producer into an input file - // for Prime. - // - - /** - * OLD COMMENTS: Not yet reworked into doc for this script, but mostly accurate - * nonetheless. - * - * This program is intended to be used from the command line, and will convert an - * output file from a sim card vendor into an input file for a HSS. The assumptions - * necessary for this to work are: - * - * * The SIM card vendor produces output files similar to the example .out file - * found in the same source directory as this program - * - * * The HSS accepts input as a CSV file, with header line 'ICCID, IMSI, KI' and subsequent - * lines containing ICCID/IMSI/Ki fields, all separated by commas. - * - * Needless to say, the outmost care should be taken when handling Ki values and - * this program must, as a matter of course, be considered a security risk, as - * must all software that touch SIM values. - * - * With that caveat in place, the usage of this program typically looks like - * this: - * - * ./outfile_to_hss_input_converter.go \ - * -input-file sample_out_file_for_testing.out - * -output-file-prefix ./hss-input-for- - * - * (followed by cryptographically strong erasure of the .out file, - * encapsulation of the .csv file in strong cryptography etc., none - * of which are handled by this script). - */ - - spUpload = kingpin.Command("sim-profile-upload", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() - spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", - "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() - // TODO: Check if this can be used for the key files. - // postImage = post.Flag("image", "image to post").ExistingFile() + confirmOrder = kingpin.Command("iccid-iccid-confirm-order", "Execute es2p iccid-iccid-confirm-order.") + confirmOrderVendor = confirmOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() + confirmOrderIccid = confirmOrder.Flag("iccid", "Iccid to confirm profile for").Required().String() + + + /// + /// Batch - centric commands + /// + + setBatchActivationCodes = kingpin.Command("batch-activate-all-profiles", "Execute activation of all profiles in batch, get activation codes from SM-DP+ and put these codes into the local database.") + setBatchActivationCodesBatch = setBatchActivationCodes.Arg("batch-name", "Batch to get activation codes for").Required().String() - // TODO: listBatches = kingpin.Command("list-batches", "List all known batches.") + getProfActActStatusesForBatch = kingpin.Command("batch-get-activation-statuses", "Get current activation statuses from SM-DP+ for named batch.") + getProfActActStatusesForBatchBatch = getProfActActStatusesForBatch.Arg("batch-name", "The batch to get activation statuses for.").Required().String() - describeBatch = kingpin.Command("describe-batch", "Describe a batch with a particular name.") - describeBatchBatch = describeBatch.Arg("batch", "The batch to describe").String() + describeBatch = kingpin.Command("batch-describe", "Describe a batch with a particular name.") + describeBatchBatch = describeBatch.Arg("batch-name", "The batch to describe").String() - generateInputFile = kingpin.Command("generate-input-file", "Generate input file for a named batch using stored parameters") - generateInputFileBatchname = generateInputFile.Arg("batchname", "The batch to generate the input file for.").String() + generateInputFile = kingpin.Command("batch-generate-input-file", "Generate input file for a named batch using stored parameters") + generateInputFileBatchname = generateInputFile.Arg("batch-name", "The batch to generate the input file for.").String() - addMsisdnFromFile = kingpin.Command("add-msisdn-from-file", "Add MSISDN from CSV file containing at least ICCID/MSISDN, but also possibly IMSI.") - addMsisdnFromFileBatch = addMsisdnFromFile.Flag("batch", "The batch to augment").Required().String() + addMsisdnFromFile = kingpin.Command("batch-add-msisdn-from-file", "Add MSISDN from CSV file containing at least ICCID/MSISDN, but also possibly IMSI.") + addMsisdnFromFileBatch = addMsisdnFromFile.Flag("batch-name", "The batch to augment").Required().String() addMsisdnFromFileCsvfile = addMsisdnFromFile.Flag("csv-file", "The CSV file to read from").Required().ExistingFile() addMsisdnFromFileAddLuhn = addMsisdnFromFile.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() - generateUploadBatch = kingpin.Command("generate-batch-upload-script", "Generate a batch upload script") - generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to generate upload script from").String() - - generateActivationCodeSql = kingpin.Command("generate-activation-code-updating-sql", "Generate SQL code to update access codes") - generateActivationCodeSqlBatch = generateActivationCodeSql.Arg("batch", "The batch to generate sql coce for").String() - - db = kingpin.Command("declare-batch", "Declare a batch to be persisted, and used by other commands") - dbName = db.Flag("name", "Unique name of this batch").Required().String() - dbAddLuhn = db.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() - dbCustomer = db.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() - dbBatchNo = db.Flag("batch-no", "Unique number of this batch (with respect to the profile vendor)").Required().String() - dbOrderDate = db.Flag("order-date", "Order date in format ddmmyyyy").Required().String() - dbFirstIccid = db.Flag("first-rawIccid", - "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - dbLastIccid = db.Flag("last-rawIccid", - "An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present").Required().String() - dbFirstIMSI = db.Flag("first-imsi", "First IMSI in batch").Required().String() - dbLastIMSI = db.Flag("last-imsi", "Last IMSI in batch").Required().String() - dbFirstMsisdn = db.Flag("first-msisdn", "First MSISDN in batch").Required().String() - dbLastMsisdn = db.Flag("last-msisdn", "Last MSISDN in batch").Required().String() - dbProfileType = db.Flag("profile-type", "SIM profile type").Required().String() - dbBatchLengthString = db.Flag( + generateUploadBatch = kingpin.Command("batch-generate-upload-script", "Generate a batch upload script") + generateUploadBatchBatch = generateUploadBatch.Arg("batch-name", "The batch to generate upload script from").String() + + generateActivationCodeSql = kingpin.Command("batch-generate-activation-code-updating-sql", "Generate SQL code to update access codes") + generateActivationCodeSqlBatch = generateActivationCodeSql.Arg("batch-name", "The batch to generate sql coce for").String() + + bd = kingpin.Command("batch-declare", "Declare a batch to be persisted, and used by other commands") + dbName = bd.Flag("name", "Unique name of this batch").Required().String() + dbAddLuhn = bd.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() + dbCustomer = bd.Flag("customer", "Name of the customer of this batch (with respect to the sim profile vendor)").Required().String() + dbBatchNo = bd.Flag("batch-no", "Unique number of this batch (with respect to the profile vendor)").Required().String() + dbOrderDate = bd.Flag("order-date", "Order date in format ddmmyyyy").Required().String() + dbFirstIccid = bd.Flag("first-rawIccid", + "An 18 or 19 digit long string. The 19-th digit being a luhn Checksum digit, if present").Required().String() + dbLastIccid = bd.Flag("last-rawIccid", + "An 18 or 19 digit long string. The 19-th digit being a luhn Checksum digit, if present").Required().String() + dbFirstIMSI = bd.Flag("first-imsi", "First IMSI in batch").Required().String() + dbLastIMSI = bd.Flag("last-imsi", "Last IMSI in batch").Required().String() + dbFirstMsisdn = bd.Flag("first-msisdn", "First MSISDN in batch").Required().String() + dbLastMsisdn = bd.Flag("last-msisdn", "Last MSISDN in batch").Required().String() + dbProfileType = bd.Flag("profile-type", "SIM profile type").Required().String() + dbBatchLengthString = bd.Flag( "batch-quantity", "Number of sim cards in batch").Required().String() - dbHssVendor = db.Flag("hss-vendor", "The HSS vendor").Default("M1").String() - dbUploadHostname = db.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() - dbUploadPortnumber = db.Flag("upload-portnumber", "port to upload to").Default("8080").String() + dbHssVendor = bd.Flag("hss-vendor", "The HSS vendor").Default("M1").String() + dbUploadHostname = bd.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() + dbUploadPortnumber = bd.Flag("upload-portnumber", "port to upload to").Default("8080").String() - dbProfileVendor = db.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() + dbProfileVendor = bd.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() - dbInitialHlrActivationStatusOfProfiles = db.Flag( + dbInitialHlrActivationStatusOfProfiles = bd.Flag( "initial-hlr-activation-status-of-profiles", "Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED.").Default("ACTIVATED").String() ) func main() { + + kingpin.Command("batches-list", "List all known batches.") + if err := parseCommandLine(); err != nil { panic(err) } @@ -188,6 +165,7 @@ func main() { func parseCommandLine() error { db, err := store.OpenFileSqliteDatabase("foobar.db") + if err != nil { return fmt.Errorf("couldn't open sqlite database. '%s'", err) } @@ -197,7 +175,7 @@ func parseCommandLine() error { cmd := kingpin.Parse() switch cmd { - case "declare-profile-vendor": + case "profile-vendor-declare": vendor, err := db.GetProfileVendorByName(*dpvName) if err != nil { @@ -249,7 +227,7 @@ func parseCommandLine() error { fmt.Println("Declared a new vendor named ", *dpvName) - case "get-profile-activation-statuses-for-batch": + case "batch-get-activation-statuses": batchName := *getProfActActStatusesForBatchBatch log.Printf("Getting statuses for all profiles in batch named %s\n", batchName) @@ -322,7 +300,7 @@ func parseCommandLine() error { sem <- true } - case "sim-profile-upload": + case "hss-convert-out-file-to-upload-script": inputFile := *spUploadInputFile outputFilePrefix := *spUploadOutputFilePrefix @@ -334,7 +312,7 @@ func parseCommandLine() error { return fmt.Errorf("couldn't close output file '%s', . Error = '%v'", outputFilePrefix, err) } - case "list-batches": + case "batches-list": allBatches, err := db.GetAllBatches() if err != nil { return err @@ -345,7 +323,7 @@ func parseCommandLine() error { fmt.Printf(" %s\n", batch.Name) } - case "describe-batch": + case "batch-describe": batch, err := db.GetBatchByName(*describeBatchBatch) if err != nil { @@ -363,7 +341,7 @@ func parseCommandLine() error { fmt.Printf("%v\n", string(bytes)) } - case "generate-activation-code-updating-sql": + case "batch-generate-activation-code-updating-sql": batch, err := db.GetBatchByName(*generateActivationCodeSqlBatch) if err != nil { return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) @@ -381,7 +359,7 @@ func parseCommandLine() error { b.Iccid) } - case "generate-batch-upload-script": + case "batch-generate-upload-script": batch, err := db.GetBatchByName(*generateUploadBatchBatch) if err != nil { return err @@ -394,7 +372,7 @@ func parseCommandLine() error { uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) } - case "generate-input-file": + case "batch-generate-input-file": batch, err := db.GetBatchByName(*generateInputFileBatchname) if err != nil { return err @@ -407,7 +385,7 @@ func parseCommandLine() error { fmt.Println(result) } - case "add-msisdn-from-file": + case "batch-add-msisdn-from-file": batchName := *addMsisdnFromFileBatch csvFilename := *addMsisdnFromFileCsvfile addLuhns := *addMsisdnFromFileAddLuhn @@ -523,7 +501,7 @@ func parseCommandLine() error { log.Printf("Updated %d of a total of %d records in batch '%s'\n", noOfRecordsUpdated, len(simEntries), batchName) - case "declare-batch": + case "batch-declare": log.Println("Declare batch") db.DeclareBatch( *dbName, @@ -545,7 +523,7 @@ func parseCommandLine() error { *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) - case "get-status": + case "iccid-get-status": client, err := ClientForVendor(db, *getStatusProfileVendor) if err != nil { return err @@ -557,7 +535,7 @@ func parseCommandLine() error { } log.Printf("Iccid='%s', state='%s', acToken='%s'\n", *getStatusProfileIccid, (*result).State, (*result).ACToken) - case "recover-profile": + case "iccid-recover-profile": client, err := ClientForVendor(db, *recoverProfileVendor) if err != nil { return err @@ -573,7 +551,7 @@ func parseCommandLine() error { } log.Println("result -> ", result) - case "download-order": + case "iccid-download-order": client, err := ClientForVendor(db, *downloadOrderVendor) if err != nil { return err @@ -584,7 +562,7 @@ func parseCommandLine() error { } log.Println("result -> ", result) - case "confirm-order": + case "iccid-confirm-order": client, err := ClientForVendor(db, *confirmOrderVendor) if err != nil { return err @@ -595,7 +573,7 @@ func parseCommandLine() error { } fmt.Println("result -> ", result) - case "activate-iccid": + case "iccid-activate": client, err := ClientForVendor(db, *activateIccidVendor) if err != nil { return err @@ -608,7 +586,7 @@ func parseCommandLine() error { } fmt.Printf("%s, %s\n", *activateIccidIccid, result.ACToken) - case "cancel-iccid": + case "iccid-cancel": client, err := ClientForVendor(db, *cancelIccidVendor) if err != nil { return err @@ -622,7 +600,7 @@ func parseCommandLine() error { return err } - case "activate-iccids-from-file": + case "iccids-activate-from-file": client, err := ClientForVendor(db, *activateIccidFileVendor) if err != nil { return err @@ -731,15 +709,13 @@ func parseCommandLine() error { sem <- true } - case "set-batch-activation-codes": + case "batch-activate-all-profiles": - batchName := *setBatchActivationCodesBatch - client, batch, err := ClientForBatch(db, batchName) + client, batch, err := ClientForBatch(db, *setBatchActivationCodesBatch) if err != nil { return err } - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) if err != nil { return err @@ -797,7 +773,7 @@ func parseCommandLine() error { } tx.Commit() - case "bulk-activate-iccids": + case "iccids-bulk-activate": client, err := ClientForVendor(db, *bulkActivateIccidsVendor) if err != nil { return err diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index b4dcb00d3..f09aacb5d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -12,6 +12,7 @@ import ( "log" "os" "strconv" + "strings" ) type SimBatchDB struct { @@ -80,12 +81,28 @@ func NewInMemoryDatabase() (*SimBatchDB, error) { } func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (*SimBatchDB, error) { - variableValue := os.Getenv(variablename) + variableValue := strings.TrimSpace(os.Getenv(variablename)) + if variableValue == "" { + return nil, fmt.Errorf("Environment variable '%s' is empty, is should contain the path to a sqlite database used by this program.", variablename) + } db, err := OpenFileSqliteDatabase(variableValue) return db, err } func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { + + if _, err := os.Stat(path); err == nil { + log.Printf("Using database file at '%s'", path) + + } else if os.IsNotExist(err) { + log.Printf("No databasefile found at '%s', will create one", path) + // path/to/whatever does *not* exist + + } else { + // Schrodinger: file may or may not exist. See err for details. + // Therefore, do *NOT* use !os.IsNotExist(err) to test for file existence + } + db, err := sqlx.Open("sqlite3", path) if err != nil { return nil, err From c127e64c8dd147201c2cfec8b1aba656a5ac220a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 15 Nov 2019 13:57:36 +0100 Subject: [PATCH 239/309] A little more user friendly --- .../sim-batch-management/TODO.md | 21 +++++++------------ .../sim-batch-management/build-all.sh | 5 +++-- .../sim-batch-management/sim-batch-mgt.go | 2 +- .../sim-batch-management/store/store.go | 15 +++++-------- 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md index e36fc5fc2..7baf58781 100644 --- a/sim-administration/sim-batch-management/TODO.md +++ b/sim-administration/sim-batch-management/TODO.md @@ -1,17 +1,15 @@ TODO == -1. Use a different database than hardcoded "foobar.db" :-) Read it from an - environment variable or something at least a bit portable. -1. Write up a nice markdown documentation describing common usecases. + 1. Create a very clean PR for future code review. -1. Compress the utility scripts into very close to nothing, by adding their functionality to the .go code. +1. Write up a nice markdown documentation describing common usecases. 1. Add crypto resources so that the program can talk to external parties. +1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) 1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external components accessed from gocode. 1. Figure out how to handle workflows. Be explicit! -1. Handle both parameterized lists of MSISDNs and list-based input. 1. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determined) for msisdn lists. @@ -19,13 +17,8 @@ TODO directly from the script later. In either case it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. - - // TODO: Some command to list all profile-vendors, hsses, etc. , e.g. lspv, lshss, ... - // TODO: Add sftp coordinates to be used when fetching/uploding input/utput-files - // TODO: Declare hss-es, that can be refered to in profiles. - // TODO: Declare legal hss/dpv combinations, batches must use legal combos. - // TODO: Declare contact methods for primes. It might be a good idea to - // impose referential integrity constraint on this too, so that - // profile/vendor/hss/prime combos are constrained. It should be possible - // to specify prod/dev primes. +1. Declare legal hss/dpv combinations, batches must use legal combos. +1. Declare prime instances (should make sense to have both prod and dev defined + with different constraints on them). + diff --git a/sim-administration/sim-batch-management/build-all.sh b/sim-administration/sim-batch-management/build-all.sh index 51e8638b3..de2d3f979 100755 --- a/sim-administration/sim-batch-management/build-all.sh +++ b/sim-administration/sim-batch-management/build-all.sh @@ -33,6 +33,7 @@ mv sim-batch-management sbm # If sourcing this script, then the line below # will modify command line compesion in bash -if [[ $_ != $0 ]] ; then - eval "$(./sbm --completion-script-bash)" +if [[ $_ != $0 ]] ; then + rm -f /tmp/tmp.db + eval "$(SIM_BATCH_DATABASE=/tmp/tmp.db ./sbm --completion-script-bash)" fi diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index d43f02203..e6f9f449b 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -164,7 +164,7 @@ func main() { func parseCommandLine() error { - db, err := store.OpenFileSqliteDatabase("foobar.db") + db, err := store.OpenFileSqliteDatabaseFromPathInEnvironmentVariable("SIM_BATCH_DATABASE") if err != nil { return fmt.Errorf("couldn't open sqlite database. '%s'", err) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index f09aacb5d..cc174efa3 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -83,25 +83,21 @@ func NewInMemoryDatabase() (*SimBatchDB, error) { func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (*SimBatchDB, error) { variableValue := strings.TrimSpace(os.Getenv(variablename)) if variableValue == "" { - return nil, fmt.Errorf("Environment variable '%s' is empty, is should contain the path to a sqlite database used by this program.", variablename) + return nil, fmt.Errorf("environment variable '%s' is empty, is should contain the path to a sqlite database used by this program. The file will be created if it's not there, but the path can't be empty", variablename) } db, err := OpenFileSqliteDatabase(variableValue) return db, err } func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { - + + /* TODO: Introduce 'debug' flag, and let that flag light up this code. if _, err := os.Stat(path); err == nil { log.Printf("Using database file at '%s'", path) - - } else if os.IsNotExist(err) { + } else { log.Printf("No databasefile found at '%s', will create one", path) - // path/to/whatever does *not* exist - - } else { - // Schrodinger: file may or may not exist. See err for details. - // Therefore, do *NOT* use !os.IsNotExist(err) to test for file existence } + */ db, err := sqlx.Open("sqlite3", path) if err != nil { @@ -123,7 +119,6 @@ func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { fmt.Println("returning null") return nil, nil } else { - fmt.Println("returning batch: ", result[0]) return &(result[0]), nil } } From 484b306c5f21b5e81b1462f60436b87a7cc72f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 15 Nov 2019 14:29:12 +0100 Subject: [PATCH 240/309] Updating readme to be less misleading --- .../README-upload-sim-batch.md | 92 ------------------- .../sim-batch-management/README.md | 91 ++++++++++++++++-- .../sim-batch-management/TODO.md | 24 ----- .../sim-batch-management/store/store.go | 8 +- 4 files changed, 87 insertions(+), 128 deletions(-) delete mode 100644 sim-administration/sim-batch-management/TODO.md diff --git a/sim-administration/sim-batch-management/README-upload-sim-batch.md b/sim-administration/sim-batch-management/README-upload-sim-batch.md index 26c815894..e69de29bb 100644 --- a/sim-administration/sim-batch-management/README-upload-sim-batch.md +++ b/sim-administration/sim-batch-management/README-upload-sim-batch.md @@ -1,92 +0,0 @@ -# How to upload batch information to prime using the - -## Introduction -Prime has REST endpoint for uploading sim batches. This is an -interface with little error checking (beyond the bare miniumums) -and a low abstraction layer: It requires a CSV file of ICCID/IMSI/MSISDN/PROFILE tuples. - -This is convenient as a starting point, but in practice it has turned -out to be a little too simple, hence the script upload-sim-batch.go. - -This script takes assumes that there is already a way to talk HTTP -(no encryption) to the upload profiles. The default assumption is that -a tunnel has been set up from localhost:8080 to somewhere more -appropriate, but these coordinaters can be tweaked using command line -parameters. - -The basic REST interface assumes an incoming .csv file, but that is -bulky, and the information content is low. In practice we will -more often than not know the ranges of IMSI, ICCID and MSISDN numbers -involved, and obviously also the profile type names. The script can -take these values as parameters and generate a valid CSV file, and -upload it automatically. - -The parameters are checked for consistency, so that if there are -more MSISDNs than ICCIDs in the ranges given as parameters, for instance, -then the script will terminate with an error message. - -(these are reasonable things to check btw, errors have been made -that justifies adding these checks). - -##Prerequisites - -* Go has to be installed on the system being run. -* Prime needs to be accessible via ssh tunnel or otherwise from the host - where the script is being run. - - -##A typical invocation looks like this: - - -(The parameters below have correct lengths, but are otherwise bogus, -and will cause error messages.) - -``` - ./upload-sim-batch.go \ - -first-iccid 1234567678901234567689 \ - -last-iccid 1234567678901234567689 \ - -first-imsi 12345676789012345 \ - -last-imsi 12345676789012345 \ - -first-msisdn 12345676789012345 \ - -last-msisdn 12345676789012345 \ - -profile-type gargle-blaster-zot \ - -profile-vendor idemalto \ - -upload-hostname localhost \ - -upload-portnumber 8080 -``` - -##The full set of command line options - -``` - - -batch-length integer - The number of profiles in the batch. Must match with iccid, msisdn and imsi ranges (if present). - -first-iccid string - An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present (default "not a valid iccid") - -first-imsi string - First IMSI in batch (default "Not a valid IMSI") - -first-msisdn string - First MSISDN in batch (default "Not a valid MSISDN") - -hss-vendor string - The HSS vendor (default "M1") - -initial-hlr-activation-status-of-profiles string - Initial hss activation state. Legal values are ACTIVATED and NOT_ACTIVATED. (default "ACTIVATED") - -last-iccid string - An 18 or 19 digit long string. The 19-th digit being a luhn luhnChecksum digit, if present (default "not a valid iccid") - -last-imsi string - Last IMSI in batch (default "Not a valid IMSI") - -last-msisdn string - Last MSISDN in batch (default "Not a valid MSISDN") - -profile-type string - SIM profile type (default "Not a valid sim profile type") - -profile-vendor string - Vendor of SIM profiles (default "Idemia") - -upload-hostname string - host to upload batch to (default "localhost") - -upload-portnumber string - port to upload to (default "8080") - -``` - - - diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index 6ab2a6422..d7b9e2d0a 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -22,14 +22,91 @@ For both of these programmes, see the source code, in particular the comments near the top of the files for instructions on how to use them. -# TODO -* Make a build command that runs tests and reports test coverage (etc), - make it part of the "build-all.go" script. -* Write some code that allows sim-batches to be kept in database. - - Pick up the location of the database and also other - secrets via an environment variable. - - Figure out if we can make multiple command-line programs that have equal value +To build everythning +== +### Prerequisites +* Go has to be installed on the system being run. +* Prime needs to be accessible via ssh tunnel or otherwise from the host + where the script is being run. +### Building + + ./build-all.sh + +... will compile and test the program and leave an executable called +"sbm" in the current directory + + . ./build-all.sh + +... will compile and test the program, then if you're running bash +extend your shell with command line extensions for the sbm program. + + +Some common usecases +== + + +### How to upload batch information to prime + +### Introduction + +Prime has REST endpoint for uploading sim batches. This is an +interface with little error checking (beyond the bare miniumums) +and a low abstraction layer: It requires a CSV file of ICCID/IMSI/MSISDN/PROFILE tuples. + +This is convenient as a starting point, but in practice it has turned +out to be a little too simple, hence the script upload-sim-batch.go. + +This script takes assumes that there is already a way to talk HTTP +(no encryption) to the upload profiles. The default assumption is that +a tunnel has been set up from localhost:8080 to somewhere more +appropriate, but these coordinaters can be tweaked using command line +parameters. + +The basic REST interface assumes an incoming .csv file, but that is +bulky, and the information content is low. In practice we will +more often than not know the ranges of IMSI, ICCID and MSISDN numbers +involved, and obviously also the profile type names. The script can +take these values as parameters and generate a valid CSV file, and +upload it automatically. + +The parameters are checked for consistency, so that if there are +more MSISDNs than ICCIDs in the ranges given as parameters, for instance, +then the script will terminate with an error message. + +(these are reasonable things to check btw, errors have been made +that justifies adding these checks). + + + +##A typical invocation looks like this: + + TBD + + + +TODO +== + + +1. Create a very clean PR for future code review. +1. Write up a nice markdown documentation describing common usecases. +1. Add crypto resources so that the program can talk to external parties. +1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) +1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that + batches can be properly constrained, defaults set the right way and external + components accessed from gocode. +1. Figure out how to handle workflows. Be explicit! +1. The interfaces to external parties will be + - input/output files for profile generation. + - some kind of file (not yet determined) for msisdn lists. + - HTTP upload commands, either indirectly via curl (as now), or + directly from the script later. In either case + it will be assumed that tunnels are set up out of band, and + tunnel setup is not part of this program. +1. Declare legal hss/dpv combinations, batches must use legal combos. +1. Declare prime instances (should make sense to have both prod and dev defined + with different constraints on them). diff --git a/sim-administration/sim-batch-management/TODO.md b/sim-administration/sim-batch-management/TODO.md deleted file mode 100644 index 7baf58781..000000000 --- a/sim-administration/sim-batch-management/TODO.md +++ /dev/null @@ -1,24 +0,0 @@ -TODO -== - - -1. Create a very clean PR for future code review. -1. Write up a nice markdown documentation describing common usecases. -1. Add crypto resources so that the program can talk to external parties. -1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) -1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that - batches can be properly constrained, defaults set the right way and external - components accessed from gocode. -1. Figure out how to handle workflows. Be explicit! -1. The interfaces to external parties will be - - input/output files for profile generation. - - some kind of file (not yet determined) for msisdn lists. - - HTTP upload commands, either indirectly via curl (as now), or - directly from the script later. In either case - it will be assumed that tunnels are set up out of band, and - tunnel setup is not part of this program. -1. Declare legal hss/dpv combinations, batches must use legal combos. -1. Declare prime instances (should make sense to have both prod and dev defined - with different constraints on them). - - diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index cc174efa3..27ffec551 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -142,15 +142,13 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { theBatch, ) - if err != nil { - // XXX Should be error logging - return fmt.Errorf("failed to insert new batch '%s'", err) + if err != nil {git sta + return log.Printf("failed to insert new batch '%s'", err) } id, err := res.LastInsertId() if err != nil { - // XXX Should be error logging - return fmt.Errorf("getting last inserted id failed '%s'", err) + return log.Printf("getting last inserted id failed '%s'", err) } theBatch.BatchId = id From 904d12edd6f2b3564bdc5da5f6913dd7cf8dc8a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Fri, 15 Nov 2019 14:36:13 +0100 Subject: [PATCH 241/309] Fix minor issues --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 +- sim-administration/sim-batch-management/store/store.go | 6 +++--- .../sim-batch-management/uploadtoprime/uploadtoprime.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index e6f9f449b..7d001026e 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -368,7 +368,7 @@ func parseCommandLine() error { if batch == nil { return fmt.Errorf("no batch found with name '%s'", *describeBatchBatch) } else { - var csvPayload = uploadtoprime.GenerateCsvPayload3(db, *batch) + var csvPayload = uploadtoprime.GenerateCsvPayload(db, *batch) uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 27ffec551..d8fbededd 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -142,13 +142,13 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { theBatch, ) - if err != nil {git sta - return log.Printf("failed to insert new batch '%s'", err) + if err != nil { + return fmt.Errorf("failed to insert new batch '%s'", err) } id, err := res.LastInsertId() if err != nil { - return log.Printf("getting last inserted id failed '%s'", err) + return fmt.Errorf("getting last inserted id failed '%s'", err) } theBatch.BatchId = id diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index 6c34ef0bb..05cf9a3e6 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -15,7 +15,7 @@ func GeneratePostingCurlscript(url string, payload string) { fmt.Print("EOF\n") } -func GenerateCsvPayload3(db *store.SimBatchDB, batch model.Batch) string { +func GenerateCsvPayload(db *store.SimBatchDB, batch model.Batch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") From d337327d49f9964122c5467657d8056fc9674318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 11:58:59 +0100 Subject: [PATCH 242/309] fix typo --- .../sim-batch-management/sim-batch-mgt.go | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 7d001026e..22e91a725 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -26,7 +26,6 @@ var ( // TODO: Global flags can be added to Kingpin, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - /// /// Converting an output file to an upload script /// for HSS configs. Not something you would want to use in @@ -38,8 +37,6 @@ var ( spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() - - /// /// Profile-vendor - centric commands @@ -55,7 +52,7 @@ var ( dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() dpvRequesterId = dpv.Flag("requester-id", "ES2+ requester ID.").Required().String() - + /// /// ICCID - centric commands /// @@ -90,18 +87,16 @@ var ( activateIccidVendor = activateIccid.Flag("profile-vendor", "Name of profile vendor").Required().String() activateIccidIccid = activateIccid.Flag("iccid", "Iccid to confirm profile for").Required().String() - - confirmOrder = kingpin.Command("iccid-iccid-confirm-order", "Execute es2p iccid-iccid-confirm-order.") + confirmOrder = kingpin.Command("iccid-confirm-order", "Execute es2p iccid-confirm-order.") confirmOrderVendor = confirmOrder.Flag("profile-vendor", "Name of profile vendor").Required().String() confirmOrderIccid = confirmOrder.Flag("iccid", "Iccid to confirm profile for").Required().String() - /// /// Batch - centric commands /// - setBatchActivationCodes = kingpin.Command("batch-activate-all-profiles", "Execute activation of all profiles in batch, get activation codes from SM-DP+ and put these codes into the local database.") - setBatchActivationCodesBatch = setBatchActivationCodes.Arg("batch-name", "Batch to get activation codes for").Required().String() + setBatchActivationCodes = kingpin.Command("batch-activate-all-profiles", "Execute activation of all profiles in batch, get activation codes from SM-DP+ and put these codes into the local database.") + setBatchActivationCodesBatch = setBatchActivationCodes.Arg("batch-name", "Batch to get activation codes for").Required().String() getProfActActStatusesForBatch = kingpin.Command("batch-get-activation-statuses", "Get current activation statuses from SM-DP+ for named batch.") getProfActActStatusesForBatchBatch = getProfActActStatusesForBatch.Arg("batch-name", "The batch to get activation statuses for.").Required().String() @@ -864,8 +859,7 @@ func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Es2PlusCl return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId), nil } - -func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Es2PlusClient, *model.Batch, error) { +func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Es2PlusClient, *model.Batch, error) { batch, err := db.GetBatchByName(batchName) if err != nil { @@ -880,7 +874,5 @@ func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Es2PlusClie return nil, nil, err } - return client, batch, nil + return client, batch, nil } - - From ae66451e8ead12f64a426144a087e5f5d6df317d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 15:22:53 +0100 Subject: [PATCH 243/309] Fix bugxs, be able to declare a profile, and fail loudly when then profile vendor isn't declared --- .../sim-batch-management/sim-batch-mgt.go | 15 +++++++++++---- .../sim-batch-management/store/store.go | 2 ++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 22e91a725..06738ce6c 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -33,7 +33,7 @@ var ( /// dealing with a test system. /// - spUpload = kingpin.Command("hss-convert-out-file-to-upload-script", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() @@ -140,7 +140,6 @@ var ( dbHssVendor = bd.Flag("hss-vendor", "The HSS vendor").Default("M1").String() dbUploadHostname = bd.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() dbUploadPortnumber = bd.Flag("upload-portnumber", "port to upload to").Default("8080").String() - dbProfileVendor = bd.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() dbInitialHlrActivationStatusOfProfiles = bd.Flag( @@ -295,7 +294,7 @@ func parseCommandLine() error { sem <- true } - case "hss-convert-out-file-to-upload-script": + case "batch-read-out-file": inputFile := *spUploadInputFile outputFilePrefix := *spUploadOutputFilePrefix @@ -498,7 +497,7 @@ func parseCommandLine() error { case "batch-declare": log.Println("Declare batch") - db.DeclareBatch( + batch, err := db.DeclareBatch( *dbName, *dbAddLuhn, *dbCustomer, @@ -518,6 +517,14 @@ func parseCommandLine() error { *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) + + if err != nil { + return err + } + log.Printf("Declared batch '%s'", batch.Name) + return nil + + case "iccid-get-status": client, err := ClientForVendor(db, *getStatusProfileVendor) if err != nil { diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index d8fbededd..8e11b40e8 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -360,6 +360,8 @@ func (sdb SimBatchDB) DeclareBatch( profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { + log.Println("Declaring batch ...") + vendor, err := sdb.GetProfileVendorByName(profileVendor) if err != nil { return nil, err From 15b438ba6dec91752b63b94d2fd7ddc9f47fb954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 16:25:54 +0100 Subject: [PATCH 244/309] Pass all tests, again --- .../sim-batch-management/sim-batch-mgt.go | 55 ++++++++++++++----- .../sim-batch-management/store/store.go | 26 +++++++++ 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 06738ce6c..df6878b00 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -26,17 +26,7 @@ var ( // TODO: Global flags can be added to Kingpin, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - /// - /// Converting an output file to an upload script - /// for HSS configs. Not something you would want to use in - /// a production-like setting, but nice to have in a pinch when - /// dealing with a test system. - /// - spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() - spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", - "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() /// /// Profile-vendor - centric commands @@ -115,6 +105,16 @@ var ( generateUploadBatch = kingpin.Command("batch-generate-upload-script", "Generate a batch upload script") generateUploadBatchBatch = generateUploadBatch.Arg("batch-name", "The batch to generate upload script from").String() + + spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spBatchName = spUpload.Flag("batch-name", "The batch to augment").Required().String() + spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() + +// TODO: Delete this asap! +// spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", +// "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() + + generateActivationCodeSql = kingpin.Command("batch-generate-activation-code-updating-sql", "Generate SQL code to update access codes") generateActivationCodeSqlBatch = generateActivationCodeSql.Arg("batch-name", "The batch to generate sql coce for").String() @@ -295,16 +295,45 @@ func parseCommandLine() error { } case "batch-read-out-file": - inputFile := *spUploadInputFile - outputFilePrefix := *spUploadOutputFilePrefix - outRecord := outfileparser.ParseOutputFile(inputFile) + batch, err := db.GetBatchByName(*spBatchName) + + if err != nil { + return err + } + + if batch == nil { + return fmt.Errorf("no batch found with name '%s'", *spBatchName) + } + + + // TODO handle error values + outRecord := outfileparser.ParseOutputFile(*spUploadInputFile) + + // TODO: Handle inconsistencies in the outRecord to ensure that + // we're not reading an incongruent output file. + + + // TODO: Do all of this in a transaction! + for _, e := range outRecord.Entries { + simProfile, err := db.GetSimProfileByIccid(e.Iccid) + if err != nil {return err} + if simProfile == nil { return fmt.Errorf("couldn't find profile enty for ICCID=%s", e.Iccid)} + if simProfile.Imsi != e.Imsi{ + return fmt.Errorf("profile enty for ICCID=%s has IMSI (%s), but we expected (%s)",e.Iccid, e.Imsi, simProfile.Imsi) + } + db.UpdateSimEntryKi(simProfile.Id, e.Ki) + } + + /** TODO: Put this into another command outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" log.Println("outputFile = ", outputFile) if err := outfileparser.WriteHssCsvFile(outputFile, outRecord.Entries); err != nil { return fmt.Errorf("couldn't close output file '%s', . Error = '%v'", outputFilePrefix, err) } + */ + case "batches-list": allBatches, err := db.GetAllBatches() diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 8e11b40e8..15e7e7d2d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -49,7 +49,9 @@ type Store interface { CreateSimEntry(simEntry *model.SimEntry) error UpdateSimEntryMsisdn(simId int64, msisdn string) UpdateActivationCode(simId int64, activationCode string) error + UpdateSimEntryKi(simId int64, ki string) error GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) + GetSimProfileByIccid(msisdn string) (*model.SimEntry, error) CreateProfileVendor(*model.ProfileVendor) error GetProfileVendorById(id int64) (*model.ProfileVendor, error) @@ -247,6 +249,8 @@ func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, erro } } + + func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { result := []model.ProfileVendor{} if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where name = ?", name); err != nil { @@ -307,6 +311,18 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, return result, nil } } +func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error) { + result := []model.SimEntry{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILES where icccid = ?", iccid); err != nil { + return nil, err + } + + if len(result) == 0 { + return nil, nil + } else { + return &result[0], nil + } +} func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", @@ -317,6 +333,16 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { return err } +func (sdb SimBatchDB) UpdateSimEntryKi(simId int64, ki string) error { + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET ki=:ki WHERE id = :simId", + map[string]interface{}{ + "simId": simId, + "ki": ki, + }) + return err +} + + func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE id = :simId", map[string]interface{}{ From 6dfa1e1f52e32a8db261c4b32dc34745282af7ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 16:40:55 +0100 Subject: [PATCH 245/309] Cheat, read via imsi instead of iccid# --- .../sim-batch-management/sim-batch-mgt.go | 10 ++++++---- .../sim-batch-management/store/store.go | 20 ++++++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index df6878b00..6e1be3c5b 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -107,8 +107,8 @@ var ( spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spBatchName = spUpload.Flag("batch-name", "The batch to augment").Required().String() - spUploadInputFile = spUpload.Flag("input-file", "path to .out file used as input file").Required().String() + spBatchName = spUpload.Arg("batch-name", "The batch to augment").Required().String() + spUploadInputFile = spUpload.Arg("input-file", "path to .out file used as input file").Required().String() // TODO: Delete this asap! // spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", @@ -316,9 +316,11 @@ func parseCommandLine() error { // TODO: Do all of this in a transaction! for _, e := range outRecord.Entries { - simProfile, err := db.GetSimProfileByIccid(e.Iccid) + log.Printf("Processing entry %v\n", e) + // simProfile, err := db.GetSimProfileByIccid(e.Iccid) + simProfile, err := db.GetSimProfileByImsi(e.Imsi) if err != nil {return err} - if simProfile == nil { return fmt.Errorf("couldn't find profile enty for ICCID=%s", e.Iccid)} + if simProfile == nil { return fmt.Errorf("couldn't find profile enty for IMSI=%s", e.Imsi)} if simProfile.Imsi != e.Imsi{ return fmt.Errorf("profile enty for ICCID=%s has IMSI (%s), but we expected (%s)",e.Iccid, e.Imsi, simProfile.Imsi) } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 15e7e7d2d..1063d0087 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -311,9 +311,13 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, return result, nil } } + +// TODO: Add unit test for this method. + + func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error) { result := []model.SimEntry{} - if err := sdb.Db.Select(&result, "select * from SIM_PROFILES where icccid = ?", iccid); err != nil { + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where iccid = ?", iccid); err != nil { return nil, err } @@ -324,6 +328,20 @@ func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error } } +func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) { + result := []model.SimEntry{} + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where imsi = ?", imsi); err != nil { + return nil, err + } + + if len(result) == 0 { + return nil, nil + } else { + return &result[0], nil + } +} + + func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", map[string]interface{}{ From 66478a75f4c4fdebf39c480c798feb9841b6bb41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 17:28:28 +0100 Subject: [PATCH 246/309] Adding unit test for setting of ki value --- .../sim-batch-management/sim-batch-mgt.go | 2 + .../sim-batch-management/store/store_test.go | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 6e1be3c5b..dc5f8a5df 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -317,6 +317,8 @@ func parseCommandLine() error { // TODO: Do all of this in a transaction! for _, e := range outRecord.Entries { log.Printf("Processing entry %v\n", e) + // TODO: The ICCIDs may be paddec with F values, and I don't want to + // deal with that now, so I'm // simProfile, err := db.GetSimProfileByIccid(e.Iccid) simProfile, err := db.GetSimProfileByImsi(e.Imsi) if err != nil {return err} diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 10cb311fe..8bf1e601f 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -286,3 +286,41 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { t.Fatal("Retrieved and stored sim entry are different") } } + +func TestSimBatchDB_UpdateSimEntryKi(t *testing.T) { + cleanTables() + injectTestprofileVendor(t) + theBatch := declareTestBatch(t) + batchId := theBatch.BatchId + + entry := model.SimEntry{ + BatchID: batchId, + RawIccid: "1", + IccidWithChecksum: "2", + IccidWithoutChecksum: "3", + Iccid: "4", + Imsi: "5", + Msisdn: "6", + Ki: "7", + ActivationCode: "8", + } + + sdb.CreateSimEntry(&entry) + assert.Assert(t, entry.Id != 0) + + newKi := "12" + err := sdb.UpdateSimEntryKi(entry.Id, newKi) + + if err != nil { + t.Fatal(err) + } + + retrivedEntry, err := sdb.GetSimEntryById(entry.Id) + if err != nil { + t.Fatal(err) + } + + if retrivedEntry.Ki != "12" { + t.Fatalf("Retrieved (%s) and stored (%s) ki values are different", retrivedEntry.Ki, newKi) + } +} From 06ff16749aff8209bfd9e69e45e4b6898064173b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 17:56:22 +0100 Subject: [PATCH 247/309] Mostly building --- .../outfileparser/outfileparser.go | 11 ++++++- .../sim-batch-management/sim-batch-mgt.go | 32 +++++++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 0056d10ff..005d741e7 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" + "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/store" + "log" "os" "regexp" @@ -286,9 +288,12 @@ func fileExists(filename string) bool { return !info.IsDir() } + + + // TODO: Move this into some other package. "hssoutput" or something. // TODO: Consider rewriting using https://golang.org/pkg/encoding/csv/ -func WriteHssCsvFile(filename string, entries []model.SimEntry) error { +func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) error { if fileExists(filename) { return fmt.Errorf("output file already exists. '%s'", filename) @@ -303,6 +308,10 @@ func WriteHssCsvFile(filename string, entries []model.SimEntry) error { return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) } + + entries, err := sdb.GetAllSimEntriesForBatch(batch.BatchId) + + max := 0 for i, entry := range entries { s := fmt.Sprintf("%s, %s, %s\n", entry.IccidWithChecksum, entry.Imsi, entry.Ki) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index dc5f8a5df..a792b05c2 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -102,14 +102,21 @@ var ( addMsisdnFromFileCsvfile = addMsisdnFromFile.Flag("csv-file", "The CSV file to read from").Required().ExistingFile() addMsisdnFromFileAddLuhn = addMsisdnFromFile.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() - generateUploadBatch = kingpin.Command("batch-generate-upload-script", "Generate a batch upload script") - generateUploadBatchBatch = generateUploadBatch.Arg("batch-name", "The batch to generate upload script from").String() + + bwBatch = kingpin.Command("batch-write-hss", "Generate a batch upload script") + bwBatchName = bwBatch.Arg("batch-name", "The batch to generate upload script from").String() + bwOutputDirName = bwBatch.Arg("output-dir-name", "The directory in which to place the output file.").String() spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") spBatchName = spUpload.Arg("batch-name", "The batch to augment").Required().String() spUploadInputFile = spUpload.Arg("input-file", "path to .out file used as input file").Required().String() + + generateUploadBatch = kingpin.Command("batch-generate-upload-script", "Write a file that can be used by an HSS to insert profiles.") + generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to output from").Required().String() + + // TODO: Delete this asap! // spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", // "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() @@ -329,14 +336,25 @@ func parseCommandLine() error { db.UpdateSimEntryKi(simProfile.Id, e.Ki) } - /** TODO: Put this into another command - outputFile := outputFilePrefix + outRecord.OutputFileName + ".csv" + + case "batch-write-hss": + + batch, err := db.GetBatchByName(*bwBatchName) + + if err != nil { + return err + } + + if batch == nil { + return fmt.Errorf("no batch found with name '%s'", *bwBatchName) + } + + outputFile := fmt.Sprintf("%s%s.csv", *bwOutputDirName, batch.Name) log.Println("outputFile = ", outputFile) - if err := outfileparser.WriteHssCsvFile(outputFile, outRecord.Entries); err != nil { - return fmt.Errorf("couldn't close output file '%s', . Error = '%v'", outputFilePrefix, err) + if err := outfileparser.WriteHssCsvFile(outputFile, db, batch); err != nil { + return fmt.Errorf("couldn't write hss output to file '%s', . Error = '%v'", outputFile, err) } - */ case "batches-list": From d6db51c79553b34afeb92311111cdd013b16fc76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 18:30:03 +0100 Subject: [PATCH 248/309] Remove now less than useful trace output --- sim-administration/sim-batch-management/store/store.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 1063d0087..db0c55910 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -132,7 +132,6 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { } else if len(result) == 0 { return nil, nil } else { - fmt.Println("returning batch: ", result[0]) return &(result[0]), nil } } From b3808fc2e23eb92fd32fac375013616f9a20e374 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Mon, 18 Nov 2019 18:30:46 +0100 Subject: [PATCH 249/309] Remove no longer helpful log, fix generation of filename so it is now generating more correct filenames --- sim-administration/sim-batch-management/sim-batch-mgt.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index a792b05c2..df05595ea 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -323,7 +323,6 @@ func parseCommandLine() error { // TODO: Do all of this in a transaction! for _, e := range outRecord.Entries { - log.Printf("Processing entry %v\n", e) // TODO: The ICCIDs may be paddec with F values, and I don't want to // deal with that now, so I'm // simProfile, err := db.GetSimProfileByIccid(e.Iccid) @@ -349,7 +348,7 @@ func parseCommandLine() error { return fmt.Errorf("no batch found with name '%s'", *bwBatchName) } - outputFile := fmt.Sprintf("%s%s.csv", *bwOutputDirName, batch.Name) + outputFile := fmt.Sprintf("%s/%s.csv", *bwOutputDirName, batch.Name) log.Println("outputFile = ", outputFile) if err := outfileparser.WriteHssCsvFile(outputFile, db, batch); err != nil { From 10a557dee593ffeb7bb47da41e0b2f6de580759e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 08:44:33 +0100 Subject: [PATCH 250/309] Fix typo --- sim-administration/sim-batch-management/store/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index db0c55910..1e0e48993 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -23,7 +23,7 @@ type Store interface { GenerateTables() error DropTables() error - CreateBath(theBatch *model.Batch) error + CreateBatch(theBatch *model.Batch) error GetAllBatches(id string) ([]model.Batch, error) GetBatchById(id int64) (*model.Batch, error) GetBatchByName(id string) (*model.Batch, error) From 3e81a03427db97398cfac4d97a4b2306ee4c1850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 08:50:52 +0100 Subject: [PATCH 251/309] Add a test for an error situation, remove DeclareBatch from Store interface --- .../outfileparser/outfileparser.go | 5 +++-- .../sim-batch-management/store/store.go | 18 ------------------ 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 005d741e7..ee9ef34d7 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -308,9 +308,10 @@ func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) } - entries, err := sdb.GetAllSimEntriesForBatch(batch.BatchId) - + if err != nil { + return err + } max := 0 for i, entry := range entries { diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 1e0e48993..87cae4ded 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -28,24 +28,6 @@ type Store interface { GetBatchById(id int64) (*model.Batch, error) GetBatchByName(id string) (*model.Batch, error) - // TODO: Maybe make the argument list for this one a little shorter, or - // perhaps this should be taken out of the store interface altogether - // (probably the best)? - DeclareBatch( - firstIccid string, - lastIccid string, - firstIMSI string, - lastIMSI string, - firstMsisdn string, - lastMsisdn string, - profileType string, - batchLengthString string, - hssVendor string, - uploadHostname string, - uploadPortnumber string, - profileVendor string, - initialHlrActivationStatusOfProfiles string) (*model.Batch, error) - CreateSimEntry(simEntry *model.SimEntry) error UpdateSimEntryMsisdn(simId int64, msisdn string) UpdateActivationCode(simId int64, activationCode string) error From f8b0b951ff358e94524833eed4811f80dc5d8fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 08:51:36 +0100 Subject: [PATCH 252/309] Remove misleading comment --- .../outfileparser/outfileparser.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index ee9ef34d7..0bb21c237 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -70,7 +70,7 @@ type ParserState struct { csvFieldMap map[string]int } -func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { +func ParseVarOutLine(varOutLine string, result *map[string]int) error { varOutSplit := strings.Split(varOutLine, ":") if len(varOutSplit) != 2 { @@ -88,10 +88,11 @@ func ParseVarOutLine(varOutLine string, result *map[string]int) (error) { } return nil } + // Implement a state machine that parses an output file. func ParseOutputFile(filename string) OutputFileRecord { - if _, err := os.Stat(filename) ; err != nil { + if _, err := os.Stat(filename); err != nil { if os.IsNotExist(err) { log.Fatalf("Couldn't find file '%s'\n", filename) } else { @@ -150,8 +151,8 @@ func ParseOutputFile(filename string) OutputFileRecord { line = strings.TrimSpace(line) lowercaseLine := strings.ToLower(line) - if (strings.HasPrefix(lowercaseLine, "var_out:")) { - if (len(state.csvFieldMap) != 0) { + if strings.HasPrefix(lowercaseLine, "var_out:") { + if len(state.csvFieldMap) != 0 { log.Fatal("Parsing multiple 'var_out' lines can't be right") } if err := ParseVarOutLine(line, &(state.csvFieldMap)); err != nil { @@ -160,7 +161,7 @@ func ParseOutputFile(filename string) OutputFileRecord { continue } - if (len(state.csvFieldMap) == 0) { + if len(state.csvFieldMap) == 0 { fmt.Println("Line = ", line) log.Fatal("Cannot parse CSV part of input file without having first parsed a CSV header.") } @@ -288,11 +289,7 @@ func fileExists(filename string) bool { return !info.IsDir() } - - - // TODO: Move this into some other package. "hssoutput" or something. -// TODO: Consider rewriting using https://golang.org/pkg/encoding/csv/ func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) error { if fileExists(filename) { @@ -304,7 +301,7 @@ func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) return fmt.Errorf("couldn't create hss csv file '%s', %v", filename, err) } - if _, err = f.WriteString("ICCID, IMSI, KI\n"); err != nil { + if _, err = f.WriteString("ICCID, IMSI, KI\n"); err != nil { return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) } From bda9fcb3693e2eb8550779fa1a2f6999f109e831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 08:55:59 +0100 Subject: [PATCH 253/309] Clean up oufileparser. Reduce public interface, remove unused method etc. --- .../outfileparser/outfileparser.go | 48 ++++++------------- .../outfileparser/outfileparser_test.go | 4 +- 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 0bb21c237..c9aebf86b 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -2,7 +2,6 @@ package outfileparser import ( "bufio" - "flag" "fmt" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" @@ -32,27 +31,8 @@ type OutputFileRecord struct { OutputFileName string } -/// -/// Functions -/// -func ParseOutputToHssConverterCommandLine() (string, string) { - inputFile := flag.String("input-file", - "not a valid filename", - "path to .out file used as input file") - - outputFile := flag.String("output-file-prefix", - "not a valid filename", - "prefix to path to .csv file used as input file, filename will be autogenerated") - - // - // Parse input according to spec above - // - flag.Parse() - return *inputFile, *outputFile -} - -func ParseLineIntoKeyValueMap(line string, theMap map[string]string) { +func parseLineIntoKeyValueMap(line string, theMap map[string]string) { var splitString = strings.Split(line, ":") if len(splitString) != 2 { log.Fatalf("Unparsable colon separated key/value pair: '%s'\n", line) @@ -62,7 +42,7 @@ func ParseLineIntoKeyValueMap(line string, theMap map[string]string) { theMap[key] = value } -type ParserState struct { +type parserState struct { currentState string inputVariables map[string]string headerDescription map[string]string @@ -70,7 +50,7 @@ type ParserState struct { csvFieldMap map[string]int } -func ParseVarOutLine(varOutLine string, result *map[string]int) error { +func parseVarOutLine(varOutLine string, result *map[string]int) error { varOutSplit := strings.Split(varOutLine, ":") if len(varOutSplit) != 2 { @@ -107,7 +87,7 @@ func ParseOutputFile(filename string) OutputFileRecord { defer file.Close() - state := ParserState{ + state := parserState{ currentState: INITIAL, inputVariables: make(map[string]string), headerDescription: make(map[string]string), @@ -139,13 +119,13 @@ func ParseOutputFile(filename string) OutputFileRecord { switch state.currentState { case HEADER_DESCRIPTION: - ParseLineIntoKeyValueMap(line, state.headerDescription) + parseLineIntoKeyValueMap(line, state.headerDescription) case INPUT_VARIABLES: if line == "var_In:" || line == "Var_In_List:" || strings.TrimSpace(line) == "" { continue } - ParseLineIntoKeyValueMap(line, state.inputVariables) + parseLineIntoKeyValueMap(line, state.inputVariables) case OUTPUT_VARIABLES: line = strings.TrimSpace(line) @@ -155,7 +135,7 @@ func ParseOutputFile(filename string) OutputFileRecord { if len(state.csvFieldMap) != 0 { log.Fatal("Parsing multiple 'var_out' lines can't be right") } - if err := ParseVarOutLine(line, &(state.csvFieldMap)); err != nil { + if err := parseVarOutLine(line, &(state.csvFieldMap)); err != nil { log.Fatalf("Couldn't parse output variable declaration '%s'\n", err) } continue @@ -225,34 +205,34 @@ func ParseOutputFile(filename string) OutputFileRecord { return result } -func getOutputFileName(state ParserState) string { +func getOutputFileName(state parserState) string { return "" + getCustomer(state) + "_" + getProfileType(state) + "_" + getBatchNo(state) } -func getBatchNo(state ParserState) string { +func getBatchNo(state parserState) string { return state.headerDescription["Batch No"] } -func getProfileType(state ParserState) string { +func getProfileType(state parserState) string { return state.headerDescription["ProfileType"] } -func getCustomer(state ParserState) string { +func getCustomer(state parserState) string { // TODO: Maker safe, so that it fails reliably if Customer is not in map. // also use constant, not magic string return state.headerDescription["Customer"] } -func parseOutputLine(state ParserState, s string) (string, string, string) { +func parseOutputLine(state parserState, s string) (string, string, string) { parsedString := strings.Split(s, " ") return parsedString[state.csvFieldMap["ICCID"]], parsedString[state.csvFieldMap["IMSI"]], parsedString[state.csvFieldMap["KI"]] } -func transitionMode(state *ParserState, targetState string) { +func transitionMode(state *parserState, targetState string) { state.currentState = targetState } -// TODO: Consider replacing this thing with a map lookup. + func modeFromSectionHeader(s string) string { sectionName := strings.Trim(s, "* ") switch sectionName { diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index d64bb9a0d..759a531a3 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -8,7 +8,7 @@ import ( func TestKeywordValueParser(t *testing.T) { theMap := make(map[string]string) - ParseLineIntoKeyValueMap("ProfileType : BAR_FOOTEL_STD", theMap) + parseLineIntoKeyValueMap("ProfileType : BAR_FOOTEL_STD", theMap) assert.Equal(t, "BAR_FOOTEL_STD", theMap["ProfileType"]) } @@ -48,7 +48,7 @@ func TestParseOutputVariablesLine(t *testing.T) { varOutLine := "var_out:ICCID/IMSI/PIN1/PUK1/PIN2/PUK2/ADM1/KI/Access_Control/Code Retailer/Code ADM/ADM2/ADM3/ADM4" m := make(map[string]int) - if err := ParseVarOutLine(varOutLine, &m); err != nil { + if err := parseVarOutLine(varOutLine, &m); err != nil { t.Error("Couldn't parse var_out line:", err) } From 459fa20b46d2f2b8bddf14fd2a9f4017f601c85c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 09:03:10 +0100 Subject: [PATCH 254/309] Fix leftovers from yesterday's quick hack --- .../sim-batch-management/outfileparser/outfileparser.go | 2 +- sim-administration/sim-batch-management/sim-batch-mgt.go | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index c9aebf86b..46799843e 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -159,7 +159,7 @@ func ParseOutputFile(filename string) OutputFileRecord { } var iccidWithoutChecksum = loltelutils.TrimSuffix(iccidWithChecksum, 1) - // TODO: Enable this!! checkICCIDSyntax(iccidWithChecksum) + // TODO: Check syntax of iccid with checksum. entry := model.SimEntry{ RawIccid: rawIccid, IccidWithChecksum: iccidWithChecksum, diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index df05595ea..0fe2fffc2 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -323,10 +323,7 @@ func parseCommandLine() error { // TODO: Do all of this in a transaction! for _, e := range outRecord.Entries { - // TODO: The ICCIDs may be paddec with F values, and I don't want to - // deal with that now, so I'm - // simProfile, err := db.GetSimProfileByIccid(e.Iccid) - simProfile, err := db.GetSimProfileByImsi(e.Imsi) + simProfile, err := db.GetSimProfileByIccid(e.IccidWithChecksum) if err != nil {return err} if simProfile == nil { return fmt.Errorf("couldn't find profile enty for IMSI=%s", e.Imsi)} if simProfile.Imsi != e.Imsi{ From 8a7ff3ab8e1b63148a5ae049639b55f72e4f7f70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 09:07:31 +0100 Subject: [PATCH 255/309] Wrap reading and processing of outfile in a transaction, so that if any single thing fails, the whole reading fails. --- sim-administration/sim-batch-management/sim-batch-mgt.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 0fe2fffc2..d21c28d09 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -322,6 +322,7 @@ func parseCommandLine() error { // TODO: Do all of this in a transaction! + tx := db.Begin() for _, e := range outRecord.Entries { simProfile, err := db.GetSimProfileByIccid(e.IccidWithChecksum) if err != nil {return err} @@ -331,6 +332,7 @@ func parseCommandLine() error { } db.UpdateSimEntryKi(simProfile.Id, e.Ki) } + tx.Commit() case "batch-write-hss": From 906cd9e56161f3276344a2d2081942cfeaf5c582 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Tue, 19 Nov 2019 09:28:23 +0100 Subject: [PATCH 256/309] Cleaning up. Adding rollback to transaction, fixing error strings. --- .../outfileparser/outfileparser.go | 23 ++++++------- .../outfileparser/outfileparser_test.go | 11 ++++-- .../sim-batch-management/sim-batch-mgt.go | 34 ++++++++++++++----- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 46799843e..08f56b74c 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -31,7 +31,6 @@ type OutputFileRecord struct { OutputFileName string } - func parseLineIntoKeyValueMap(line string, theMap map[string]string) { var splitString = strings.Split(line, ":") if len(splitString) != 2 { @@ -70,13 +69,13 @@ func parseVarOutLine(varOutLine string, result *map[string]int) error { } // Implement a state machine that parses an output file. -func ParseOutputFile(filename string) OutputFileRecord { +func ParseOutputFile(filename string) (*OutputFileRecord, error) { if _, err := os.Stat(filename); err != nil { if os.IsNotExist(err) { - log.Fatalf("Couldn't find file '%s'\n", filename) + return nil, fmt.Errorf("couldn't find file '%s'", filename) } else { - log.Fatalf("Couldn't stat file '%s'\n", filename) + return nil, fmt.Errorf("couldn't stat file '%s'", filename) } } @@ -133,17 +132,16 @@ func ParseOutputFile(filename string) OutputFileRecord { if strings.HasPrefix(lowercaseLine, "var_out:") { if len(state.csvFieldMap) != 0 { - log.Fatal("Parsing multiple 'var_out' lines can't be right") + return nil, fmt.Errorf("parsing multiple 'var_out' lines can't be right") } if err := parseVarOutLine(line, &(state.csvFieldMap)); err != nil { - log.Fatalf("Couldn't parse output variable declaration '%s'\n", err) + return nil, fmt.Errorf("couldn't parse output variable declaration '%s'", err) } continue } if len(state.csvFieldMap) == 0 { - fmt.Println("Line = ", line) - log.Fatal("Cannot parse CSV part of input file without having first parsed a CSV header.") + return nil, fmt.Errorf("cannot parse CSV part of input file without having first parsed a CSV header, failed when processing line '%s'", line) } line = strings.TrimSpace(line) @@ -172,7 +170,7 @@ func ParseOutputFile(filename string) OutputFileRecord { continue default: - log.Fatalf("Unknown parser state '%s'\n", state.currentState) + return nil, fmt.Errorf("unknown parser state '%s'", state.currentState) } } @@ -184,11 +182,11 @@ func ParseOutputFile(filename string) OutputFileRecord { declaredNoOfEntities, err := strconv.Atoi(state.headerDescription["Quantity"]) if err != nil { - log.Fatal("Could not find 'Quantity' field while parsing file '", filename, "'") + return nil, fmt.Errorf("could not find 'Quantity' field while parsing file '%s'", filename) } if countedNoOfEntries != declaredNoOfEntities { - log.Fatalf("Declared no of entities = %d, counted nunber of entities = %d. Mismatch!", + return nil, fmt.Errorf("mismatch between no of entities = %d, counted number of entities = %d", declaredNoOfEntities, countedNoOfEntries) } @@ -202,7 +200,7 @@ func ParseOutputFile(filename string) OutputFileRecord { OutputFileName: getOutputFileName(state), } - return result + return &result, nil } func getOutputFileName(state parserState) string { @@ -232,7 +230,6 @@ func transitionMode(state *parserState, targetState string) { state.currentState = targetState } - func modeFromSectionHeader(s string) string { sectionName := strings.Trim(s, "* ") switch sectionName { diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 759a531a3..407e1158b 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -15,7 +15,10 @@ func TestKeywordValueParser(t *testing.T) { func TestReadingSimpleOutputFile(t *testing.T) { sample_output_file_name := "sample_out_file_for_testing.out" - record := ParseOutputFile(sample_output_file_name) + record, err := ParseOutputFile(sample_output_file_name) + if err != nil { + t.Error(t) + } // First parameter to check assert.Equal(t, sample_output_file_name, record.Filename) @@ -38,7 +41,11 @@ func TestReadingSimpleOutputFile(t *testing.T) { func TestReadingComplexOutputFile(t *testing.T) { sample_output_file_name := "sample-out-2.out" - record := ParseOutputFile(sample_output_file_name) + record, err := ParseOutputFile(sample_output_file_name) + if err != nil { + t.Error(t) + } + fmt.Println("Record = ", record) // TODO: Check that we got all the fields diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index d21c28d09..8da1cc0fb 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -303,9 +303,23 @@ func parseCommandLine() error { case "batch-read-out-file": + tx := db.Begin() + + // By default we're not cool to commit. Need to prove something first. + weCool := false + + defer func() { + if weCool { + tx.Commit() + } else { + tx.Rollback() + } + }() + batch, err := db.GetBatchByName(*spBatchName) if err != nil { + tx.Rollback() return err } @@ -313,16 +327,17 @@ func parseCommandLine() error { return fmt.Errorf("no batch found with name '%s'", *spBatchName) } + outRecord, err := outfileparser.ParseOutputFile(*spUploadInputFile) + if err != nil { + return err + } - // TODO handle error values - outRecord := outfileparser.ParseOutputFile(*spUploadInputFile) - - // TODO: Handle inconsistencies in the outRecord to ensure that - // we're not reading an incongruent output file. + if outRecord.NoOfEntries != batch.Quantity { + return fmt.Errorf("number of records returned from outfile (%d) does not match number of profiles (%d) in batch '%s'", + outRecord.NoOfEntries, batch.Quantity, batch.Name) + } - // TODO: Do all of this in a transaction! - tx := db.Begin() for _, e := range outRecord.Entries { simProfile, err := db.GetSimProfileByIccid(e.IccidWithChecksum) if err != nil {return err} @@ -332,7 +347,10 @@ func parseCommandLine() error { } db.UpdateSimEntryKi(simProfile.Id, e.Ki) } - tx.Commit() + + // Signal to defered transaction cleanup that we're cool + // with a commit. + weCool = true case "batch-write-hss": From 910d0bbeb08b75cb05461f573de789143c21f803 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 19 Nov 2019 20:01:19 +0100 Subject: [PATCH 257/309] Adding dummy project to remove github integration with snyk.io since it does not support multi-module gradle kotlin script project. --- build.gradle | 45 +++++++++++++++++++ settings.gradle.kts | 2 + .../es2plus4dropwizard/build.gradle.kts | 6 +++ .../ostelco-dropwizard-utils/build.gradle.kts | 12 +++-- snyk/build.gradle | 0 5 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 build.gradle create mode 100644 snyk/build.gradle diff --git a/build.gradle b/build.gradle new file mode 100644 index 000000000..b0d5ce37f --- /dev/null +++ b/build.gradle @@ -0,0 +1,45 @@ +plugins { + id("base") + id("java") + id("project-report") + id("com.github.ben-manes.versions") version "0.27.0" + id("jacoco") + id("org.jetbrains.kotlin.jvm") version "1.3.50" apply false + id("com.google.protobuf") version "0.8.10" apply false + id("com.github.johnrengelman.shadow") version "5.2.0" apply false + id("idea") +} + +allprojects { + apply plugin: "jacoco" + + group = "org.ostelco" + version = "1.0.0-SNAPSHOT" + + repositories { + mavenCentral() + maven { url = uri("https://dl.bintray.com/palantir/releases") } // docker-compose-rule is published on bintray + jcenter() + maven { url = uri("http://repository.jboss.org/nexus/content/repositories/releases/") } + maven { url = uri("https://maven.repository.redhat.com/ga/") } + maven { url = uri("http://clojars.org/repo/") } + maven { url = uri("https://jitpack.io") } + } + + jacoco { + toolVersion = "0.8.4" + } +} + +java { + sourceCompatibility = JavaVersion.VERSION_12 + targetCompatibility = JavaVersion.VERSION_12 +} + +subprojects { + tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).forEach { + kotlinOptions { + jvmTarget = JavaVersion.VERSION_12.majorVersion + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index fcc6319e3..ffb7c67a7 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -40,6 +40,7 @@ include(":scaninfo-datastore") include(":scaninfo-shredder") include(":secure-archive") include(":slack") +include(":snyk") include(":tracing") include(":sim-administration:es2plus4dropwizard") @@ -90,6 +91,7 @@ project(":scaninfo-datastore").projectDir = File("$rootDir/scaninfo-datastore") project(":scaninfo-shredder").projectDir = File("$rootDir/scaninfo-shredder") project(":secure-archive").projectDir = File("$rootDir/secure-archive") project(":slack").projectDir = File("$rootDir/slack") +project(":snyk").projectDir = File("$rootDir/snyk") project(":tracing").projectDir = File("$rootDir/tracing") project(":sim-administration:es2plus4dropwizard").projectDir = File("$rootDir/sim-administration/es2plus4dropwizard") diff --git a/sim-administration/es2plus4dropwizard/build.gradle.kts b/sim-administration/es2plus4dropwizard/build.gradle.kts index 22561d5c5..8206b7e35 100644 --- a/sim-administration/es2plus4dropwizard/build.gradle.kts +++ b/sim-administration/es2plus4dropwizard/build.gradle.kts @@ -17,4 +17,10 @@ dependencies { testImplementation("org.mockito:mockito-core:${Version.mockito}") } +tasks.withType { + kotlinOptions { + jvmTarget = JavaVersion.VERSION_12.majorVersion + } +} + apply(from = "../../gradle/jacoco.gradle.kts") \ No newline at end of file diff --git a/sim-administration/ostelco-dropwizard-utils/build.gradle.kts b/sim-administration/ostelco-dropwizard-utils/build.gradle.kts index 9dc256d31..00130a512 100644 --- a/sim-administration/ostelco-dropwizard-utils/build.gradle.kts +++ b/sim-administration/ostelco-dropwizard-utils/build.gradle.kts @@ -7,13 +7,11 @@ plugins { dependencies { implementation(kotlin("stdlib-jdk8")) + implementation(kotlin("reflect")) + implementation("io.dropwizard:dropwizard-core:${Version.dropwizard}") implementation("io.swagger.core.v3:swagger-jaxrs2:${Version.swagger}") - - implementation(kotlin("reflect")) - implementation(kotlin("stdlib-jdk8")) - implementation("io.dropwizard:dropwizard-client:${Version.dropwizard}") implementation("io.dropwizard:dropwizard-core:${Version.dropwizard}") @@ -29,4 +27,10 @@ dependencies { testImplementation("io.dropwizard:dropwizard-testing:${Version.dropwizard}") } +tasks.withType { + kotlinOptions { + jvmTarget = JavaVersion.VERSION_12.majorVersion + } +} + apply(from = "../../gradle/jacoco.gradle.kts") \ No newline at end of file diff --git a/snyk/build.gradle b/snyk/build.gradle new file mode 100644 index 000000000..e69de29bb From 0b47ec0007dfc267b1d79b04065685187a1f1772 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 19 Nov 2019 22:26:25 +0100 Subject: [PATCH 258/309] Removed temporary changes done to remove snyk.io integration with github --- build.gradle | 45 ------------------- settings.gradle.kts | 2 - .../es2plus4dropwizard/build.gradle.kts | 6 --- .../ostelco-dropwizard-utils/build.gradle.kts | 6 --- snyk/build.gradle | 0 5 files changed, 59 deletions(-) delete mode 100644 build.gradle delete mode 100644 snyk/build.gradle diff --git a/build.gradle b/build.gradle deleted file mode 100644 index b0d5ce37f..000000000 --- a/build.gradle +++ /dev/null @@ -1,45 +0,0 @@ -plugins { - id("base") - id("java") - id("project-report") - id("com.github.ben-manes.versions") version "0.27.0" - id("jacoco") - id("org.jetbrains.kotlin.jvm") version "1.3.50" apply false - id("com.google.protobuf") version "0.8.10" apply false - id("com.github.johnrengelman.shadow") version "5.2.0" apply false - id("idea") -} - -allprojects { - apply plugin: "jacoco" - - group = "org.ostelco" - version = "1.0.0-SNAPSHOT" - - repositories { - mavenCentral() - maven { url = uri("https://dl.bintray.com/palantir/releases") } // docker-compose-rule is published on bintray - jcenter() - maven { url = uri("http://repository.jboss.org/nexus/content/repositories/releases/") } - maven { url = uri("https://maven.repository.redhat.com/ga/") } - maven { url = uri("http://clojars.org/repo/") } - maven { url = uri("https://jitpack.io") } - } - - jacoco { - toolVersion = "0.8.4" - } -} - -java { - sourceCompatibility = JavaVersion.VERSION_12 - targetCompatibility = JavaVersion.VERSION_12 -} - -subprojects { - tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).forEach { - kotlinOptions { - jvmTarget = JavaVersion.VERSION_12.majorVersion - } - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index ffb7c67a7..fcc6319e3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -40,7 +40,6 @@ include(":scaninfo-datastore") include(":scaninfo-shredder") include(":secure-archive") include(":slack") -include(":snyk") include(":tracing") include(":sim-administration:es2plus4dropwizard") @@ -91,7 +90,6 @@ project(":scaninfo-datastore").projectDir = File("$rootDir/scaninfo-datastore") project(":scaninfo-shredder").projectDir = File("$rootDir/scaninfo-shredder") project(":secure-archive").projectDir = File("$rootDir/secure-archive") project(":slack").projectDir = File("$rootDir/slack") -project(":snyk").projectDir = File("$rootDir/snyk") project(":tracing").projectDir = File("$rootDir/tracing") project(":sim-administration:es2plus4dropwizard").projectDir = File("$rootDir/sim-administration/es2plus4dropwizard") diff --git a/sim-administration/es2plus4dropwizard/build.gradle.kts b/sim-administration/es2plus4dropwizard/build.gradle.kts index 8206b7e35..22561d5c5 100644 --- a/sim-administration/es2plus4dropwizard/build.gradle.kts +++ b/sim-administration/es2plus4dropwizard/build.gradle.kts @@ -17,10 +17,4 @@ dependencies { testImplementation("org.mockito:mockito-core:${Version.mockito}") } -tasks.withType { - kotlinOptions { - jvmTarget = JavaVersion.VERSION_12.majorVersion - } -} - apply(from = "../../gradle/jacoco.gradle.kts") \ No newline at end of file diff --git a/sim-administration/ostelco-dropwizard-utils/build.gradle.kts b/sim-administration/ostelco-dropwizard-utils/build.gradle.kts index 00130a512..80cb4adc0 100644 --- a/sim-administration/ostelco-dropwizard-utils/build.gradle.kts +++ b/sim-administration/ostelco-dropwizard-utils/build.gradle.kts @@ -27,10 +27,4 @@ dependencies { testImplementation("io.dropwizard:dropwizard-testing:${Version.dropwizard}") } -tasks.withType { - kotlinOptions { - jvmTarget = JavaVersion.VERSION_12.majorVersion - } -} - apply(from = "../../gradle/jacoco.gradle.kts") \ No newline at end of file diff --git a/snyk/build.gradle b/snyk/build.gradle deleted file mode 100644 index e69de29bb..000000000 From 08fb015ced3093099e5a4932ccaf7909e1156d4c Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 19 Nov 2019 22:26:47 +0100 Subject: [PATCH 259/309] Updated versions of dependencies --- README.md | 2 +- build.gradle.kts | 2 +- .../kotlin/org/ostelco/prime/gradle/Version.kt | 16 ++++++++-------- gradle/wrapper/gradle-wrapper.properties | 2 +- prime-customer-api/build.gradle.kts | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 99f4a7f49..f7c9f7870 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.3.50-blue.svg)](http://kotlinlang.org/) +[![Kotlin version badge](https://img.shields.io/badge/kotlin-1.3.60-blue.svg)](http://kotlinlang.org/) [![Prime version](https://img.shields.io/github/tag/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/tags) [![GitHub license](https://img.shields.io/github/license/ostelco/ostelco-core.svg)](https://github.com/ostelco/ostelco-core/blob/master/LICENSE) diff --git a/build.gradle.kts b/build.gradle.kts index 9d2174c51..1d43ce22f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,7 +10,7 @@ plugins { id("project-report") id("com.github.ben-manes.versions") version "0.27.0" jacoco - kotlin("jvm") version "1.3.50" apply false + kotlin("jvm") version "1.3.60" apply false id("com.google.protobuf") version "0.8.10" apply false id("com.github.johnrengelman.shadow") version "5.2.0" apply false idea diff --git a/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt b/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt index a22521cc1..138716a32 100644 --- a/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt +++ b/buildSrc/src/main/kotlin/org/ostelco/prime/gradle/Version.kt @@ -14,10 +14,10 @@ object Version { const val firebase = "6.11.0" const val googleCloud = "1.91.3" - const val googleCloudDataStore = "1.100.0" + const val googleCloudDataStore = "1.101.0" const val googleCloudLogging = "0.116.0-alpha" - const val googleCloudPubSub = "1.100.0" - const val googleCloudStorage = "1.100.0" + const val googleCloudPubSub = "1.101.0" + const val googleCloudStorage = "1.101.0" const val gson = "2.8.6" const val grpc = "1.25.0" @@ -30,10 +30,10 @@ object Version { // Keeping it version 1.16.1 to be consistent with grpc via PubSub client lib // Keeping it version 1.16.1 to be consistent with netty via Firebase lib const val jaxb = "2.3.1" - const val jdbi3 = "3.10.1" + const val jdbi3 = "3.11.1" const val jjwt = "0.10.7" const val junit5 = "5.5.2" - const val kotlin = "1.3.50" + const val kotlin = "1.3.60" const val kotlinXCoroutines = "1.3.2" const val mockito = "3.1.0" const val mockitoKotlin = "2.2.0" @@ -46,9 +46,9 @@ object Version { const val slf4j = "1.7.29" // IMPORTANT: When Stripe SDK library version is updated, check if the Stripe API version has changed. // If so, then update API version in Stripe Web Console for callback Webhooks. - const val stripe = "15.3.0" - const val swagger = "2.0.10" - const val swaggerCodegen = "2.4.9" + const val stripe = "15.4.0" + const val swagger = "2.1.0" + const val swaggerCodegen = "2.4.10" const val testcontainers = "1.12.3" const val tink = "1.2.2" const val zxing = "3.4.0" diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 0140ada96..53170a22f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon May 27 14:33:41 CEST 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.0.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/prime-customer-api/build.gradle.kts b/prime-customer-api/build.gradle.kts index e5751b0b7..80d0e25c5 100644 --- a/prime-customer-api/build.gradle.kts +++ b/prime-customer-api/build.gradle.kts @@ -30,7 +30,7 @@ dependencies { implementation("javax.annotation:javax.annotation-api:${Version.javaxAnnotation}") // taken from build/swagger-code-java-client/build.gradle - implementation("io.swagger:swagger-annotations:1.5.24") + implementation("io.swagger:swagger-annotations:1.6.0") implementation("com.google.code.gson:gson:${Version.gson}") implementation("com.squareup.okhttp:okhttp:2.7.5") implementation("com.squareup.okhttp:logging-interceptor:2.7.5") From 1978e29d609a0fe670622c94c695d30e7cb99cc8 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Tue, 19 Nov 2019 22:27:52 +0100 Subject: [PATCH 260/309] Updated prime version --- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index 3489ac4aa..f37e365a6 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.72.7" +version = "1.73.0" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 422d3f80e..881782bbc 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.72.7,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.73.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From 8cda8ca365fa171449bb347280b3a9b7c6378876 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 07:31:22 +0100 Subject: [PATCH 261/309] Bigger machine for CircleCI --- .circleci/config.yml | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5c1c24fe..7bf9ccd99 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,11 @@ -version: 2 +version: 2.1 jobs: ### JOBS FOR on-feature-branch-commit PIPELINE build-test-repo: # machine is needed to run Gradle build and to run docker compose tests machine: image: ubuntu-1604:201903-01 + resource_class: large environment: JAVA_HOME: /usr/lib/jvm/zulu13.28.11-ca-jdk13.0.1-linux_x64 @@ -18,7 +19,7 @@ jobs: sudo tar -zxf /tmp/zulu13.28.11-ca-jdk13.0.1-linux_x64.tar.gz -C /usr/lib/jvm - run: # checking for merge conflicts and merging locally if none exist - name: merging ${CIRCLE_BRANCH} into develop locally + name: merging into develop locally command: | git config --global user.email "${GIT_USER_EMAIL}" git config --global user.name "${GIT_USER_NAME}" @@ -65,13 +66,14 @@ jobs: - . # generating selfsigned certs. Needed for docker compose tests - - run: - name: Generate self signed certs - command: | - scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org - cp certs/ocs.dev.ostelco.org/nginx.crt ocsgw/cert/ocs.crt - scripts/generate-selfsigned-ssl-certs.sh metrics.dev.ostelco.org - cp certs/metrics.dev.ostelco.org/nginx.crt ocsgw/cert/metrics.crt +# - run: +# name: Generate self signed certs +# command: | +# scripts/generate-selfsigned-ssl-certs.sh ocs.dev.ostelco.org +# cp certs/ocs.dev.ostelco.org/nginx.crt ocsgw/cert/ocs.crt +# scripts/generate-selfsigned-ssl-certs.sh metrics.dev.ostelco.org +# cp certs/metrics.dev.ostelco.org/nginx.crt ocsgw/cert/metrics.crt + - run: name: Acceptance Tests command: docker-compose up --build --abort-on-container-exit @@ -116,6 +118,7 @@ jobs: build-code: machine: image: ubuntu-1604:201903-01 + resource_class: large environment: JAVA_HOME: /usr/lib/jvm/zulu13.28.11-ca-jdk13.0.1-linux_x64 From 374f119b4fd59be1081e34da8a7bec74fd540997 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 07:59:16 +0100 Subject: [PATCH 262/309] Updated datastore emulator docker images --- docker-compose.esp.yaml | 2 +- docker-compose.ocs.yaml | 2 +- docker-compose.seagull.yaml | 2 +- docker-compose.yaml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.esp.yaml b/docker-compose.esp.yaml index db2967f11..bf20cb83b 100644 --- a/docker-compose.esp.yaml +++ b/docker-compose.esp.yaml @@ -143,7 +143,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk:218.0.0 + image: google/cloud-sdk:272.0.0 expose: - "8081" environment: diff --git a/docker-compose.ocs.yaml b/docker-compose.ocs.yaml index c2846eedd..4507520a8 100644 --- a/docker-compose.ocs.yaml +++ b/docker-compose.ocs.yaml @@ -61,7 +61,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk:218.0.0 + image: google/cloud-sdk:272.0.0 expose: - "8081" environment: diff --git a/docker-compose.seagull.yaml b/docker-compose.seagull.yaml index 903e1b0a6..398e572af 100644 --- a/docker-compose.seagull.yaml +++ b/docker-compose.seagull.yaml @@ -90,7 +90,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk:218.0.0 + image: google/cloud-sdk:272.0.0 expose: - "8081" environment: diff --git a/docker-compose.yaml b/docker-compose.yaml index 95737c45e..f29034793 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -118,7 +118,7 @@ services: datastore-emulator: container_name: datastore-emulator - image: google/cloud-sdk:218.0.0 + image: google/cloud-sdk:272.0.0 expose: - "8081" environment: @@ -142,7 +142,7 @@ services: build: context: sim-administration/postgres dockerfile: Dockerfile - tmpfs: "//var/lib/postgresql/data" + tmpfs: "/var/lib/postgresql/data" ports: - "55432:5432" From 5aa45c07c97b6e74f22d2d5b00edffff5300034e Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 07:59:43 +0100 Subject: [PATCH 263/309] Enable docker layer caching in CircleCI --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7bf9ccd99..29cdfbe6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,7 +198,8 @@ jobs: at: ~/project # starts a remote docker environment to run docker commands - - setup_remote_docker + - setup_remote_docker: + docker_layer_caching: true - run: name: build Prime docker image and push image to GCR @@ -234,7 +235,8 @@ jobs: at: ~/project # starts a remote docker environment to run docker commands - - setup_remote_docker + - setup_remote_docker: + docker_layer_caching: true - run: name: build scaninfo shredder docker image and push image to GCR From ec5274acda172064299a8e28f34722c8706f4c24 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 10:53:31 +0100 Subject: [PATCH 264/309] Verbose logging of Apple ID Auth error response --- .../kotlin/org/ostelco/prime/auth/apple/Model.kt | 16 ++++++++-------- .../prime/auth/resources/AppleIdAuthResource.kt | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/apple/Model.kt b/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/apple/Model.kt index 9dab2e0bb..0b7afe9b1 100644 --- a/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/apple/Model.kt +++ b/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/apple/Model.kt @@ -15,13 +15,13 @@ data class TokenResponse( data class ErrorResponse(val error: Error) -enum class Error { - invalid_request, - invalid_client, - invalid_grant, - unauthorized_client, - unsupported_grant_type, - invalid_scope, +enum class Error(val cause: String) { + invalid_request("The request is malformed, normally due to a missing parameter, contains an unsupported parameter, includes multiple credentials, or uses more than one mechanism for authenticating the client."), + invalid_client("The client authentication failed."), + invalid_grant("The authorization grant or refresh token is invalid."), + unauthorized_client("The client is not authorized to use this authorization grant type."), + unsupported_grant_type("The authenticated client is not authorized to use the grant type."), + invalid_scope("The requested scope is invalid."), } data class JWKKey( @@ -33,4 +33,4 @@ data class JWKKey( val use: String ) -data class JWKSet(val keys: Collection) \ No newline at end of file +data class JWKSet(val keys: Collection) diff --git a/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/resources/AppleIdAuthResource.kt b/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/resources/AppleIdAuthResource.kt index 28cf4f061..adbdc4a61 100644 --- a/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/resources/AppleIdAuthResource.kt +++ b/appleid-auth-service/src/main/kotlin/org/ostelco/prime/auth/resources/AppleIdAuthResource.kt @@ -29,7 +29,7 @@ class AppleIdAuthResource { return AppleIdAuthClient.authorize(authCode.authCode) .fold( { - logger.warn("error: {}", it.error) + logger.warn("AppleId Auth Error Response: {}, cause: {}", it.error, it.error.error.cause) Response.status(it.status).entity(asJson(it)) }, { tokenResponse -> From e95531b73e9792504096a3b317544258fd89ef0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 12:49:12 +0100 Subject: [PATCH 265/309] Refactor, fix readability, use more idiomatic code. --- .../ostelco/sim/es2plus/Es2PlusEntities.kt | 42 +++++++++---------- .../ostelco/sim/es2plus/Es2PlusResources.kt | 10 ++--- .../sim/es2plus/Es2PlusServiceInterfaces.kt | 5 +-- .../EncryptedRemoteEs2ClientOnlyTest.kt | 8 ++-- .../ostelco/sim/es2plus/Es2plusApplication.kt | 7 ++-- 5 files changed, 34 insertions(+), 38 deletions(-) diff --git a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt index b707f62cf..73ef44489 100644 --- a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt +++ b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusEntities.kt @@ -77,7 +77,7 @@ data class Es2PlusDownloadOrder( data class Es2DownloadOrderResponse( @JsonProperty("header") val header: ES2ResponseHeader = eS2SuccessResponseHeader(), @JsonProperty("iccid") val iccid: String? = null -): Es2Response(header) +) : Es2Response(header) /// @@ -118,7 +118,7 @@ data class IccidListEntry( @JsonInclude(JsonInclude.Include.NON_NULL) data class Es2ProfileStatusCommand( @JsonProperty("header") val header: ES2RequestHeader, - @JsonProperty("iccidList") val iccidList: List = listOf()) + @JsonProperty("iccidList") val iccidList: List = listOf()) @JsonInclude(JsonInclude.Include.NON_NULL) @@ -126,11 +126,11 @@ data class Es2ProfileStatusResponse( @JsonProperty("header") val header: ES2ResponseHeader = eS2SuccessResponseHeader(), @JsonProperty("profileStatusList") val profileStatusList: List? = listOf(), @JsonProperty("completionTimestamp") val completionTimestamp: String? = getNowAsDatetime() -): Es2Response(myHeader = header) +) : Es2Response(myHeader = header) @JsonInclude(JsonInclude.Include.NON_NULL) data class ProfileStatus( - @JsonProperty("status_last_update_timestamp") val lastUpdateTimestamp:String? = null, + @JsonProperty("status_last_update_timestamp") val lastUpdateTimestamp: String? = null, @JsonProperty("profileStatusList") val profileStatusList: List? = listOf(), @JsonProperty("acToken") val acToken: String? = null, @JsonProperty("state") val state: String? = null, @@ -166,7 +166,7 @@ data class Es2ConfirmOrderResponse( @JsonProperty("eid") val eid: String? = null, @JsonProperty("matchingId") val matchingId: String? = null, @JsonProperty("smdpAddress") val smdsAddress: String? = null -): Es2Response(myHeader = header) +) : Es2Response(myHeader = header) /// /// The CancelOrder function @@ -176,10 +176,10 @@ data class Es2ConfirmOrderResponse( // XXX CXHeck @JsonSchema("ES2+CancelOrder-def") data class Es2CancelOrder( @JsonProperty("header") val header: ES2RequestHeader, - @JsonProperty("eid") val eid: String?=null, + @JsonProperty("eid") val eid: String? = null, @JsonProperty("profileStatusList") val profileStatusList: String? = null, @JsonProperty("matchingId") val matchingId: String? = null, - @JsonProperty("iccid") val iccid: String?=null, + @JsonProperty("iccid") val iccid: String? = null, @JsonProperty("finalProfileStatusIndicator") val finalProfileStatusIndicator: String? = null ) @@ -219,7 +219,7 @@ data class Es2HandleDownloadProgressInfo( val imei: String? = null, // This field is added to ensure that the function signature of the primary and the actual // constructors are not confused by the JVM. It is ignored by all business logic. - private val ignoreThisField : String? = null) { + private val ignoreThisField: String? = null) { // If the stored ICCID contains a trailing "F", which it may because some vendors insist @@ -232,16 +232,16 @@ data class Es2HandleDownloadProgressInfo( // class. @JsonCreator - constructor (@JsonProperty("header") header: ES2RequestHeader, - @JsonProperty("eid") eid: String? = null, - @JsonProperty("iccid") iccid: String, - @JsonProperty("profileType") profileType: String, - @JsonProperty("timestamp") timestamp: String = getNowAsDatetime(), - @JsonProperty("tac") tac: String? = null, - @JsonProperty("notificationPointId") notificationPointId: Int, - @JsonProperty("notificationPointStatus") notificationPointStatus: ES2NotificationPointStatus, - @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("resultData") resultData: String? = null, - @JsonProperty("imei") imei: String? = null) : this( + constructor (@JsonProperty("header") header: ES2RequestHeader, + @JsonProperty("eid") eid: String? = null, + @JsonProperty("iccid") iccid: String, + @JsonProperty("profileType") profileType: String, + @JsonProperty("timestamp") timestamp: String = getNowAsDatetime(), + @JsonProperty("tac") tac: String? = null, + @JsonProperty("notificationPointId") notificationPointId: Int, + @JsonProperty("notificationPointStatus") notificationPointStatus: ES2NotificationPointStatus, + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("resultData") resultData: String? = null, + @JsonProperty("imei") imei: String? = null) : this( header = header, eid = eid, iccid = if (!iccid.endsWith("F")) { // Rewrite input value if necessary @@ -263,13 +263,13 @@ data class Es2HandleDownloadProgressInfo( @JsonInclude(JsonInclude.Include.NON_NULL) data class ES2NotificationPointStatus( @JsonProperty("status") val status: FunctionExecutionStatusType = FunctionExecutionStatusType.ExecutedSuccess, - @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("statusCodeData") val statusCodeData: ES2StatusCodeData? = null + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("statusCodeData") val statusCodeData: ES2StatusCodeData? = null ) @JsonInclude(JsonInclude.Include.NON_NULL) data class ES2StatusCodeData( @JsonProperty("subjectCode") val subjectCode: String, // "Executed-Success, Executed-WithWarning, Failed or - @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("reasonCode") val statusCodeData: String, + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty("reasonCode") val statusCodeData: String, @JsonProperty("subjectIdentifier") val subjectIdentifier: String? = null, @JsonProperty("message") val message: String? = null ) @@ -286,6 +286,6 @@ fun newErrorHeader(exception: SmDpPlusException): ES2ResponseHeader { statusCodeData = exception.statusCodeData)) } -fun eS2SuccessResponseHeader() = +fun eS2SuccessResponseHeader(): ES2ResponseHeader = ES2ResponseHeader(functionExecutionStatus = FunctionExecutionStatus(status = FunctionExecutionStatusType.ExecutedSuccess)) \ No newline at end of file diff --git a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusResources.kt b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusResources.kt index 66e3ddca1..95952eb2b 100644 --- a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusResources.kt +++ b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusResources.kt @@ -88,7 +88,7 @@ class SmdpExceptionMapper : ExceptionMapper { override fun toResponse(ex: SmDpPlusException): Response { // First we log the event. - logger.error("SM-DP+ processing failed: {}" , ex.statusCodeData) + logger.error("SM-DP+ processing failed: {}", ex.statusCodeData) // Then we prepare a response that will be returned to // whoever invoked the resource. @@ -113,7 +113,7 @@ class SmDpPlusServerResource(private val smDpPlus: SmDpPlusService) { private val logger = getLogger() companion object { - const val ES2PLUS_PATH_PREFIX : String = "gsma/rsp2/es2plus/" + const val ES2PLUS_PATH_PREFIX: String = "gsma/rsp2/es2plus/" } /** @@ -136,12 +136,12 @@ class SmDpPlusServerResource(private val smDpPlus: SmDpPlusService) { @POST fun confirmOrder(order: Es2ConfirmOrder): Es2ConfirmOrderResponse { return smDpPlus.confirmOrder( - eid=order.eid, + eid = order.eid, iccid = order.iccid, confirmationCode = order.confirmationCode, smdsAddress = order.smdpAddress, machingId = order.matchingId, - releaseFlag = order.releaseFlag + releaseFlag = order.releaseFlag ) } @@ -177,7 +177,7 @@ class SmDpPlusServerResource(private val smDpPlus: SmDpPlusService) { @POST fun getProfileStatus(order: Es2ProfileStatusCommand): Es2ProfileStatusResponse { logger.value.info("Logging getProfileStatusOrder with order = $order") - return smDpPlus.getProfileStatus(iccidList = order.iccidList.map {it.iccid}.filterNotNull()) + return smDpPlus.getProfileStatus(iccidList = order.iccidList.mapNotNull { it.iccid }) } } diff --git a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusServiceInterfaces.kt b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusServiceInterfaces.kt index 98e2a7240..2b52fe087 100644 --- a/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusServiceInterfaces.kt +++ b/sim-administration/es2plus4dropwizard/src/main/kotlin/org/ostelco/sim/es2plus/Es2PlusServiceInterfaces.kt @@ -1,7 +1,6 @@ package org.ostelco.sim.es2plus - class SmDpPlusException(val statusCodeData: StatusCodeData) : Exception() @@ -11,7 +10,7 @@ interface SmDpPlusService { fun downloadOrder(eid: String?, iccid: String?, profileType: String?): Es2DownloadOrderResponse @Throws(SmDpPlusException::class) - fun confirmOrder(eid: String?, iccid: String?, smdsAddress: String?, machingId: String?, confirmationCode: String?, releaseFlag:Boolean): Es2ConfirmOrderResponse + fun confirmOrder(eid: String?, iccid: String?, smdsAddress: String?, machingId: String?, confirmationCode: String?, releaseFlag: Boolean): Es2ConfirmOrderResponse @Throws(SmDpPlusException::class) fun cancelOrder(eid: String?, iccid: String?, matchingId: String?, finalProfileStatusIndicator: String?) @@ -23,7 +22,7 @@ interface SmDpPlusService { fun releaseProfile(iccid: String) } -interface SmDpPlusCallbackService { +interface SmDpPlusCallbackService { @Throws(SmDpPlusException::class) fun handleDownloadProgressInfo( diff --git a/sim-administration/es2plus4dropwizard/src/test/kotlin/org/ostelco/sim/es2plus/EncryptedRemoteEs2ClientOnlyTest.kt b/sim-administration/es2plus4dropwizard/src/test/kotlin/org/ostelco/sim/es2plus/EncryptedRemoteEs2ClientOnlyTest.kt index 4498defc0..f2cb5a5c4 100644 --- a/sim-administration/es2plus4dropwizard/src/test/kotlin/org/ostelco/sim/es2plus/EncryptedRemoteEs2ClientOnlyTest.kt +++ b/sim-administration/es2plus4dropwizard/src/test/kotlin/org/ostelco/sim/es2plus/EncryptedRemoteEs2ClientOnlyTest.kt @@ -105,7 +105,7 @@ class EncryptedRemoteEs2ClientOnlyTest { } companion object { - val SUPPORT = DropwizardTestSupport( + val SUPPORT: DropwizardTestSupport = DropwizardTestSupport( DummyAppUsingSmDpPlusClient::class.java, "src/test/resources/config-external-smdp.yml" ) @@ -113,7 +113,6 @@ class EncryptedRemoteEs2ClientOnlyTest { } - class DummyAppUsingSmDpPlusClient : Application() { override fun getName(): String { @@ -151,7 +150,6 @@ class DummyAppUsingSmDpPlusClient : Application() { class PlaceholderSmDpPlusService : SmDpPlusService { override fun getProfileStatus(iccidList: List): Es2ProfileStatusResponse { - val statuses : List = iccidList.map { - iccid ->ProfileStatus(iccid = iccid, state = "ALLOCATED")} + val statuses: List = iccidList.map { iccid -> ProfileStatus(iccid = iccid, state = "ALLOCATED") } return Es2ProfileStatusResponse( profileStatusList = statuses) } @Throws(SmDpPlusException::class) override fun downloadOrder(eid: String?, iccid: String?, profileType: String?): Es2DownloadOrderResponse { - return Es2DownloadOrderResponse(eS2SuccessResponseHeader(), iccid="01234567890123456789") + return Es2DownloadOrderResponse(eS2SuccessResponseHeader(), iccid = "01234567890123456789") } override fun confirmOrder(eid: String?, iccid: String?, smdsAddress: String?, machingId: String?, confirmationCode: String?, releaseFlag: Boolean): Es2ConfirmOrderResponse { - return Es2ConfirmOrderResponse(eS2SuccessResponseHeader(), eid="1234567890123456789012", matchingId = "foo", smdsAddress = "localhost") + return Es2ConfirmOrderResponse(eS2SuccessResponseHeader(), eid = "1234567890123456789012", matchingId = "foo", smdsAddress = "localhost") } @Throws(SmDpPlusException::class) From 67c6ee4897372f7100d322b270073f3fde0a101d Mon Sep 17 00:00:00 2001 From: Prasanth Ullattil Date: Wed, 20 Nov 2019 12:53:19 +0100 Subject: [PATCH 266/309] Differentiate callback errors --- .../prime/admin/resources/KYCResource.kt | 67 ++++++++++--------- .../ostelco/prime/apierror/ApiErrorCodes.kt | 2 + 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt b/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt index f4babc616..f510de80e 100644 --- a/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt +++ b/admin-endpoint/src/main/kotlin/org/ostelco/prime/admin/resources/KYCResource.kt @@ -1,6 +1,8 @@ package org.ostelco.prime.admin.resources import arrow.core.Either +import arrow.core.left +import arrow.core.right import com.fasterxml.jackson.module.kotlin.readValue import org.ostelco.prime.apierror.ApiError import org.ostelco.prime.apierror.ApiErrorCode @@ -77,7 +79,7 @@ class KYCResource { return null } - private fun toScanInformation(dataMap: Map): ScanInformation? { + private fun toScanInformation(dataMap: Map): Either { try { val vendorScanReference: String = dataMap[JumioScanData.JUMIO_SCAN_ID.s]!! var status: ScanStatus = toScanStatus(dataMap[JumioScanData.SCAN_STATUS.s]!!) @@ -110,29 +112,30 @@ class KYCResource { } } } - return getCountryCodeForScan(scanId) - ?.let { countryCode -> - ScanInformation( - scanId, - countryCode, - status, - ScanResult( - vendorScanReference = vendorScanReference, - verificationStatus = verificationStatus, - time = time, - type = type, - country = country, - firstName = firstName, - lastName = lastName, - dob = dob, - expiry = expiry, - rejectReason = rejectReason - ) - ) - } + val countryCode = getCountryCodeForScan(scanId) + if (countryCode == null) { + return NotFoundError("Cannot find country for scan $scanId", ApiErrorCode.FAILED_TO_GET_COUNTRY_FOR_SCAN).left() + } else { + return ScanInformation( + scanId, + countryCode, + status, + ScanResult( + vendorScanReference = vendorScanReference, + verificationStatus = verificationStatus, + time = time, + type = type, + country = country, + firstName = firstName, + lastName = lastName, + dob = dob, + expiry = expiry, + rejectReason = rejectReason + )).right() + } } catch (e: NullPointerException) { logger.error("Missing mandatory fields in scan result $dataMap", e) - return null + return BadRequestError("Missing mandatory fields in scan result", ApiErrorCode.FAILED_TO_CONVERT_SCAN_RESULT).left() } } @@ -144,16 +147,16 @@ class KYCResource { @Context httpHeaders: HttpHeaders, formData: MultivaluedMap): Response { dumpRequestInfo(request, httpHeaders, formData) - val scanInformation = toScanInformation(toRegularMap(formData)) - if (scanInformation == null) { - logger.info("Unable to convert scan information from form data") - val reqError = BadRequestError("Missing mandatory fields in scan result", ApiErrorCode.FAILED_TO_UPDATE_SCAN_RESULTS) - return Response.status(reqError.status).entity(asJson(reqError)).build() - } - logger.info("Updating scan information ${scanInformation.scanId} jumioIdScanReference ${scanInformation.scanResult?.vendorScanReference}") - return updateScanInformation(scanInformation, formData).fold( - { apiError -> Response.status(apiError.status).entity(asJson(apiError)) }, - { Response.status(Response.Status.OK).entity(asJson(scanInformation)) }).build() + return toScanInformation(toRegularMap(formData)) + .fold({ + logger.info("Unable to convert scan information from form data") + Response.status(it.status).entity(asJson(it)) + }, { scanInformation -> + logger.info("Updating scan information ${scanInformation.scanId} jumioIdScanReference ${scanInformation.scanResult?.vendorScanReference}") + updateScanInformation(scanInformation, formData).fold( + { Response.status(it.status).entity(asJson(it)) }, + { Response.status(Response.Status.OK).entity(asJson(scanInformation)) }) + }).build() } private fun getCountryCodeForScan(scanId: String): String? { diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt index 4c0f69461..42835ebb1 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiErrorCodes.kt @@ -91,6 +91,8 @@ enum class ApiErrorCode { // Jumio FAILED_TO_CREATE_SCANID, FAILED_TO_FETCH_SCAN_INFORMATION, + FAILED_TO_GET_COUNTRY_FOR_SCAN, + FAILED_TO_CONVERT_SCAN_RESULT, FAILED_TO_UPDATE_SCAN_RESULTS, FAILED_TO_FETCH_SUBSCRIBER_STATE, From ecb6894a78024c0cb15acaf4686edfcc17e24a15 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 13:01:01 +0100 Subject: [PATCH 267/309] Fixing Class Not Found exception --- neo4j-store/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neo4j-store/build.gradle.kts b/neo4j-store/build.gradle.kts index e675bd518..59dfba558 100644 --- a/neo4j-store/build.gradle.kts +++ b/neo4j-store/build.gradle.kts @@ -14,6 +14,8 @@ dependencies { implementation("org.neo4j.driver:neo4j-java-driver:${Version.neo4jDriver}") implementation("org.neo4j:neo4j-slf4j:${Version.neo4j}") + implementation("com.sun.jna:jna:3.0.9") + testImplementation("com.palantir.docker.compose:docker-compose-rule-junit4:${Version.dockerComposeJunitRule}") testImplementation(kotlin("test")) From 6540a76e251e54ac87fcb1ce4e1572d1666251f3 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 13:01:31 +0100 Subject: [PATCH 268/309] Fixing incorrect grouping of errors in GCP Error Reporting --- .../src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt index 7bd14b9ec..eebe0ac44 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt @@ -32,7 +32,7 @@ object ApiErrorMapper { val logger by getLogger() fun mapPaymentErrorToApiError(description: String, errorCode: ApiErrorCode, paymentError: PaymentError) : ApiError { - logger.error("description: $description, errorCode: $errorCode, paymentError: ${asJson(paymentError)}") + logger.error("{}: {}, paymentError: {}", errorCode, description, asJson(paymentError)) return when(paymentError) { is org.ostelco.prime.paymentprocessor.core.PlanAlredyPurchasedError -> ForbiddenError(description, errorCode, paymentError) is org.ostelco.prime.paymentprocessor.core.ForbiddenError -> ForbiddenError(description, errorCode, paymentError) @@ -44,7 +44,7 @@ object ApiErrorMapper { } fun mapStorageErrorToApiError(description: String, errorCode: ApiErrorCode, storeError: StoreError) : ApiError { - logger.error("description: $description, errorCode: $errorCode, storeError: ${asJson(storeError)}") + logger.error("{}: {}, storeError: {}", errorCode, description, asJson(storeError)) return when(storeError) { is org.ostelco.prime.storage.NotFoundError -> NotFoundError(description, errorCode, storeError) is org.ostelco.prime.storage.AlreadyExistsError -> ForbiddenError(description, errorCode, storeError) @@ -61,7 +61,7 @@ object ApiErrorMapper { } fun mapSimManagerErrorToApiError(description: String, errorCode: ApiErrorCode, simManagerError: SimManagerError) : ApiError { - logger.error("description: $description, errorCode: $errorCode, simManagerError: ${asJson(simManagerError)}") + logger.error("{}: {}, simManagerError: {}", errorCode, description, asJson(simManagerError)) return when (simManagerError) { is org.ostelco.prime.simmanager.NotFoundError -> NotFoundError(description, errorCode, simManagerError) is org.ostelco.prime.simmanager.NotUpdatedError -> BadRequestError(description, errorCode, simManagerError) From 41a260bb16cbc040e3da01ab1b135388a88d8b95 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 13:01:48 +0100 Subject: [PATCH 269/309] Updated prime version --- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index f37e365a6..b6601122a 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.73.0" +version = "1.73.1" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 881782bbc..63cc2dbf0 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.73.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.73.1,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From d522de754a88e4112d4e8044cbc53d7c88cefd72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 14:32:36 +0100 Subject: [PATCH 270/309] Add todo about accessing HSS. --- sim-administration/sim-batch-management/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index d7b9e2d0a..5d1f1f5d1 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -95,6 +95,7 @@ TODO 1. Create a very clean PR for future code review. 1. Write up a nice markdown documentation describing common usecases. 1. Add crypto resources so that the program can talk to external parties. +1. Add code to activate profiles in HSS (if API is known) 1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) 1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external From 84d9e0fb85c705536ba07e4cf8030725686aa9f3 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 14 Nov 2019 12:54:46 +0100 Subject: [PATCH 271/309] Refactors the payment API (cosmetic) - rename parameters containing the 'stripe...' prefix - replaces Either.{right,left}() with just .{right,left}() --- .../org/ostelco/at/common/StripePayment.kt | 4 +- .../ostelco/prime/storage/graph/Neo4jStore.kt | 2 +- .../StripePaymentProcessorTest.kt | 72 +++++++++---------- .../StripePaymentProcessor.kt | 67 +++++++++-------- .../paymentprocessor/PaymentProcessor.kt | 30 ++++---- 5 files changed, 90 insertions(+), 85 deletions(-) diff --git a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt index 5d6a9bd3f..2c376fb49 100644 --- a/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt +++ b/acceptance-tests/src/main/kotlin/org/ostelco/at/common/StripePayment.kt @@ -83,7 +83,7 @@ object StripePayment { * verify that the correspondng 'setDefaultSource' API works as * intended. */ - fun getDefaultSourceForCustomer(stripeCustomerId: String) : String { + fun getDefaultSourceForCustomer(customerId: String) : String { // https://stripe.com/docs/api/java#create_source Stripe.apiKey = System.getenv("STRIPE_API_KEY") @@ -92,7 +92,7 @@ object StripePayment { (0..MAX_TRIES).forEach { try { - return Customer.retrieve(stripeCustomerId).defaultSource + return Customer.retrieve(customerId).defaultSource } catch (e: Exception) { error = e } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 2ca5625e0..97ed83a4c 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1385,7 +1385,7 @@ object Neo4jStoreSingleton : GraphStore { val subscriptionDetailsInfo = paymentProcessor.createSubscription( planId = planStripeId, - stripeCustomerId = profileInfo.id, + customerId = profileInfo.id, trialEnd = trialEnd, taxRegionId = taxRegionId) .mapLeft { diff --git a/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 24ea53c0a..58df31f52 100644 --- a/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -24,7 +24,7 @@ class StripePaymentProcessorTest { private val testCustomer = UUID.randomUUID().toString() private val emailTestCustomer = "test@internet.org" - private var stripeCustomerId = "" + private var customerId = "" private fun createPaymentTokenId(): String { @@ -64,7 +64,7 @@ class StripePaymentProcessorTest { val resultAdd = paymentProcessor.createPaymentProfile(customerId = testCustomer, email = emailTestCustomer) resultAdd.isRight() - stripeCustomerId = resultAdd.fold({ "" }, { it.id }) + customerId = resultAdd.fold({ "" }, { it.id }) } @Before @@ -75,7 +75,7 @@ class StripePaymentProcessorTest { @After fun cleanUp() { - val resultDelete = paymentProcessor.deletePaymentProfile(stripeCustomerId) + val resultDelete = paymentProcessor.deletePaymentProfile(customerId) assertNotFailure(resultDelete) } @@ -118,7 +118,7 @@ class StripePaymentProcessorTest { @Test fun unknownCustomerGetSavedSources() { - val result = paymentProcessor.getSavedSources(stripeCustomerId = "unknown") + val result = paymentProcessor.getSavedSources(customerId = "unknown") assertFailure(result) } @@ -127,7 +127,7 @@ class StripePaymentProcessorTest { fun getPaymentProfile() { val result = paymentProcessor.getPaymentProfile(testCustomer) assertNotFailure(result) - assertEquals(stripeCustomerId, result.fold({ "" }, { it.id })) + assertEquals(customerId, result.fold({ "" }, { it.id })) } @Test @@ -140,14 +140,14 @@ class StripePaymentProcessorTest { fun ensureSourcesSorted() { run { - paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + paymentProcessor.addSource(customerId, createPaymentTokenId()) // Ensure that not all sources falls within the same second. Thread.sleep(1_001) - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + paymentProcessor.addSource(customerId, createPaymentSourceId()) } // Should be in descending sorted order by the "created" timestamp. - val sources = paymentProcessor.getSavedSources(stripeCustomerId) + val sources = paymentProcessor.getSavedSources(customerId) val createdTimestamps = sources.getOrElse { fail("The 'created' field is missing from the list of sources: ${sources}") @@ -163,12 +163,12 @@ class StripePaymentProcessorTest { fun addAndRemoveMultipleSources() { val sources = listOf( - paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()), - paymentProcessor.addSource(stripeCustomerId, createPaymentSourceId()) + paymentProcessor.addSource(customerId, createPaymentTokenId()), + paymentProcessor.addSource(customerId, createPaymentSourceId()) ) val sourcesRemoved = sources.map { - paymentProcessor.removeSource(stripeCustomerId, it.getOrElse { + paymentProcessor.removeSource(customerId, it.getOrElse { fail("Failed to remove source ${it}") }.id) } @@ -196,28 +196,28 @@ class StripePaymentProcessorTest { @Test fun addSourceToCustomerAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) - val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) + val resultStoredSources = paymentProcessor.getSavedSources(customerId) checkthatStoredResourcesMatchAddedResources(resultAddSource, resultStoredSources) - val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId, right(resultAddSource).id) + val resultDeleteSource = paymentProcessor.removeSource(customerId, right(resultAddSource).id) assertNotFailure(resultDeleteSource) } @Test fun addSourceToCustomerTwice() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) - val resultStoredSources = paymentProcessor.getSavedSources(stripeCustomerId) + val resultStoredSources = paymentProcessor.getSavedSources(customerId) checkthatStoredResourcesMatchAddedResources(resultAddSource, resultStoredSources) - val resultAddSecondSource = paymentProcessor.addSource(stripeCustomerId, right(resultStoredSources).first().id) + val resultAddSecondSource = paymentProcessor.addSource(customerId, right(resultStoredSources).first().id) assertFailure(resultAddSecondSource) - val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId, right(resultAddSource).id) + val resultDeleteSource = paymentProcessor.removeSource(customerId, right(resultAddSource).id) assertNotFailure(resultDeleteSource) } @@ -227,55 +227,55 @@ class StripePaymentProcessorTest { @Test fun addDefaultSourceAndRemove() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) assertNotFailure(resultAddSource) - val resultAddDefault = paymentProcessor.setDefaultSource(stripeCustomerId, right(resultAddSource).id) + val resultAddDefault = paymentProcessor.setDefaultSource(customerId, right(resultAddSource).id) assertNotFailure(resultAddDefault) - val resultGetDefault = paymentProcessor.getDefaultSource(stripeCustomerId) + val resultGetDefault = paymentProcessor.getDefaultSource(customerId) assertNotFailure(resultGetDefault) assertEquals(resultAddDefault.fold({ "" }, { it.id }), right(resultGetDefault).id) - val resultRemoveDefault = paymentProcessor.removeSource(stripeCustomerId, right(resultAddDefault).id) + val resultRemoveDefault = paymentProcessor.removeSource(customerId, right(resultAddDefault).id) assertNotFailure(resultRemoveDefault) } @Test fun createAuthorizeChargeAndRefund() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) assertNotFailure(resultAddSource) val amount = 1000 val currency = "NOK" - val resultAuthorizeCharge = paymentProcessor.authorizeCharge(stripeCustomerId, right(resultAddSource).id, amount, currency) + val resultAuthorizeCharge = paymentProcessor.authorizeCharge(customerId, right(resultAddSource).id, amount, currency) assertNotFailure(resultAuthorizeCharge) val resultRefundCharge = paymentProcessor.refundCharge(right(resultAuthorizeCharge), amount) assertNotFailure(resultRefundCharge) - val resultRemoveSource = paymentProcessor.removeSource(stripeCustomerId, right(resultAddSource).id) + val resultRemoveSource = paymentProcessor.removeSource(customerId, right(resultAddSource).id) assertNotFailure(resultRemoveSource) } @Test fun createAuthorizeChargeAndRefundWithZeroAmount() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) assertNotFailure(resultAddSource) val amount = 0 val currency = "NOK" - val resultAuthorizeCharge = paymentProcessor.authorizeCharge(stripeCustomerId, right(resultAddSource).id, amount, currency) + val resultAuthorizeCharge = paymentProcessor.authorizeCharge(customerId, right(resultAddSource).id, amount, currency) assertNotFailure(resultAuthorizeCharge) val resultRefundCharge = paymentProcessor.refundCharge(right(resultAuthorizeCharge), amount) assertNotFailure(resultRefundCharge) assertEquals(resultAuthorizeCharge.fold({ "" }, { it }), right(resultRefundCharge)) - val resultRemoveSource = paymentProcessor.removeSource(stripeCustomerId, right(resultAddSource).id) + val resultRemoveSource = paymentProcessor.removeSource(customerId, right(resultAddSource).id) assertNotFailure(resultRemoveSource) } @@ -291,7 +291,7 @@ class StripePaymentProcessorTest { @Test fun subscribeAndUnsubscribePlan() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) assertNotFailure(resultAddSource) val resultCreateProduct = paymentProcessor.createProduct("TestSku") @@ -300,7 +300,7 @@ class StripePaymentProcessorTest { val resultCreatePlan = paymentProcessor.createPlan(right(resultCreateProduct).id, 1000, "NOK", PaymentProcessor.Interval.MONTH) assertNotFailure(resultCreatePlan) - val resultSubscribePlan = paymentProcessor.createSubscription(right(resultCreatePlan).id, stripeCustomerId) + val resultSubscribePlan = paymentProcessor.createSubscription(right(resultCreatePlan).id, customerId) assertNotFailure(resultSubscribePlan) val resultUnsubscribePlan = paymentProcessor.cancelSubscription(right(resultSubscribePlan).id, false) @@ -315,19 +315,19 @@ class StripePaymentProcessorTest { assertNotFailure(resultRemoveProduct) assertEquals(resultCreateProduct.fold({ "" }, { it.id }), right(resultRemoveProduct).id) - val resultDeleteSource = paymentProcessor.removeSource(stripeCustomerId, right(resultAddSource).id) + val resultDeleteSource = paymentProcessor.removeSource(customerId, right(resultAddSource).id) assertNotFailure(resultDeleteSource) } @Test fun createAndDeleteInvoiceItem() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) assertNotFailure(resultAddSource) val amount = 5000 val currency = "SGD" - val addedInvoiceItem = paymentProcessor.createInvoiceItem(stripeCustomerId, amount, currency, "SGD") + val addedInvoiceItem = paymentProcessor.createInvoiceItem(customerId, amount, currency, "SGD") assertNotFailure(addedInvoiceItem) val removedInvoiceItem = paymentProcessor.removeInvoiceItem(right(addedInvoiceItem).id) @@ -336,13 +336,13 @@ class StripePaymentProcessorTest { @Test fun createAndDeleteInvoiceWithTaxes() { - val resultAddSource = paymentProcessor.addSource(stripeCustomerId, createPaymentTokenId()) + val resultAddSource = paymentProcessor.addSource(customerId, createPaymentTokenId()) assertNotFailure(resultAddSource) val amount = 5000 val currency = "SGD" - val addedInvoiceItem = paymentProcessor.createInvoiceItem(stripeCustomerId, amount, currency, "SGD") + val addedInvoiceItem = paymentProcessor.createInvoiceItem(customerId, amount, currency, "SGD") assertNotFailure(addedInvoiceItem) val taxRegionId = "sg" @@ -350,7 +350,7 @@ class StripePaymentProcessorTest { val taxRates = paymentProcessor.getTaxRatesForTaxRegionId(taxRegionId) assertNotFailure(taxRates) - val addedInvoice = paymentProcessor.createInvoice(stripeCustomerId, right(taxRates)) + val addedInvoice = paymentProcessor.createInvoice(customerId, right(taxRates)) assertNotFailure(addedInvoice) val payedInvoice = paymentProcessor.payInvoice(right(addedInvoice).id) diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index d4d2480b0..aa8cf5953 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -48,9 +48,9 @@ class StripePaymentProcessor : PaymentProcessor { private val logger by getLogger() - override fun getSavedSources(stripeCustomerId: String): Either> = - either("Failed to retrieve sources for customer $stripeCustomerId") { - val customer = Customer.retrieve(stripeCustomerId) + override fun getSavedSources(customerId: String): Either> = + either("Failed to retrieve sources for customer $customerId") { + val customer = Customer.retrieve(customerId) val sources: List = customer.sources.data.map { val details = getAccountDetails(it) SourceDetailsInfo(it.id, getAccountType(details), details) @@ -153,12 +153,15 @@ class StripePaymentProcessor : PaymentProcessor { }.fold( ifSuccess = { customer -> when { - customer.deleted == true -> Either.left(NotFoundError("Payment profile for user $customerId was previously deleted")) - else -> Either.right(ProfileInfo(customer.id)) + customer.deleted == true -> NotFoundError("Payment profile for user $customerId was previously deleted") + .left() + else -> ProfileInfo(customer.id) + .right() } }, ifFailure = { - Either.left(NotFoundError("Could not find a payment profile for user $customerId")) + NotFoundError("Could not find a payment profile for user $customerId") + .left() } ) @@ -196,38 +199,38 @@ class StripePaymentProcessor : PaymentProcessor { ProductInfo(product.delete().id) } - override fun addSource(stripeCustomerId: String, stripeSourceId: String): Either = - either("Failed to add source $stripeSourceId to customer $stripeCustomerId") { - val customer = Customer.retrieve(stripeCustomerId) - val sourceParams = mapOf("source" to stripeSourceId, + override fun addSource(customerId: String, sourceId: String): Either = + either("Failed to add source $sourceId to customer $customerId") { + val customer = Customer.retrieve(customerId) + val sourceParams = mapOf("source" to sourceId, "metadata" to mapOf("created" to ofEpochMilliToSecond(Instant.now().toEpochMilli()))) SourceInfo(customer.sources.create(sourceParams).id) } - override fun setDefaultSource(stripeCustomerId: String, sourceId: String): Either = - either("Failed to set default source $sourceId for customer $stripeCustomerId") { - val customer = Customer.retrieve(stripeCustomerId) + override fun setDefaultSource(customerId: String, sourceId: String): Either = + either("Failed to set default source $sourceId for customer $customerId") { + val customer = Customer.retrieve(customerId) val updateParams = mapOf("default_source" to sourceId) val customerUpdated = customer.update(updateParams) SourceInfo(customerUpdated.defaultSource) } - override fun getDefaultSource(stripeCustomerId: String): Either = - either("Failed to get default source for customer $stripeCustomerId") { - SourceInfo(Customer.retrieve(stripeCustomerId).defaultSource) + override fun getDefaultSource(customerId: String): Either = + either("Failed to get default source for customer $customerId") { + SourceInfo(Customer.retrieve(customerId).defaultSource) } - override fun deletePaymentProfile(stripeCustomerId: String): Either = - either("Failed to delete customer $stripeCustomerId") { - val customer = Customer.retrieve(stripeCustomerId) + override fun deletePaymentProfile(customerId: String): Either = + either("Failed to delete customer $customerId") { + val customer = Customer.retrieve(customerId) ProfileInfo(customer.delete().id) } /* The 'expand' part will cause an immediate attempt at charging for the subscription when creating it. For interpreting the result see: https://stripe.com/docs/billing/subscriptions/payment#signup-3b */ - override fun createSubscription(planId: String, stripeCustomerId: String, trialEnd: Long, taxRegionId: String?): Either = - either("Failed to subscribe customer $stripeCustomerId to plan $planId") { + override fun createSubscription(planId: String, customerId: String, trialEnd: Long, taxRegionId: String?): Either = + either("Failed to subscribe customer $customerId to plan $planId") { val item = mapOf("plan" to planId) val taxRates = getTaxRatesForTaxRegionId(taxRegionId) .fold( @@ -235,7 +238,7 @@ class StripePaymentProcessor : PaymentProcessor { { it } ) val subscriptionParams = mapOf( - "customer" to stripeCustomerId, + "customer" to customerId, "items" to mapOf("0" to item), *(if (trialEnd > Instant.now().epochSecond) arrayOf("trial_end" to trialEnd.toString()) @@ -294,7 +297,8 @@ class StripePaymentProcessor : PaymentProcessor { override fun authorizeCharge(customerId: String, sourceId: String?, amount: Int, currency: String): Either { val errorMessage = "Failed to authorize the charge for customerId $customerId sourceId $sourceId amount $amount currency $currency" return when (amount) { - 0 -> Either.right("ZERO_CHARGE_${UUID.randomUUID()}") + 0 -> "ZERO_CHARGE_${UUID.randomUUID()}" + .right() else -> either(errorMessage) { val chargeParams = mutableMapOf( "amount" to amount, @@ -319,7 +323,7 @@ class StripePaymentProcessor : PaymentProcessor { override fun captureCharge(chargeId: String, customerId: String, amount: Int, currency: String): Either { val errorMessage = "Failed to capture charge for customerId $customerId chargeId $chargeId" return when (amount) { - 0 -> Either.right(chargeId) + 0 -> chargeId.right() else -> either(errorMessage) { Charge.retrieve(chargeId) }.flatMap { charge: Charge -> @@ -332,10 +336,10 @@ class StripePaymentProcessor : PaymentProcessor { }.flatMap { charge -> try { charge.capture() - Either.right(charge.id) + charge.id.right() } catch (e: Exception) { logger.warn(errorMessage, e) - Either.left(BadGatewayError(errorMessage)) + BadGatewayError(errorMessage).left() } } } @@ -349,7 +353,7 @@ class StripePaymentProcessor : PaymentProcessor { override fun refundCharge(chargeId: String, amount: Int): Either = when (amount) { - 0 -> Either.right(chargeId) + 0 -> chargeId.right() else -> either("Failed to refund charge $chargeId") { val refundParams = mapOf( "charge" to chargeId, @@ -358,14 +362,15 @@ class StripePaymentProcessor : PaymentProcessor { } } - override fun removeSource(stripeCustomerId: String, sourceId: String): Either = - either("Failed to remove source $sourceId for stripeCustomerId $stripeCustomerId") { - val accountInfo = Customer.retrieve(stripeCustomerId).sources.retrieve(sourceId) + override fun removeSource(customerId: String, sourceId: String): Either = + either("Failed to remove source $sourceId for customerId $customerId") { + val accountInfo = Customer.retrieve(customerId).sources.retrieve(sourceId) when (accountInfo) { is Card -> accountInfo.delete() is Source -> accountInfo.detach() else -> - Either.left(BadGatewayError("Attempt to remove unsupported account-type $accountInfo")) + BadGatewayError("Attempt to remove unsupported account-type $accountInfo") + .left() } SourceInfo(sourceId) } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt index bcf5ec186..d91988279 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt @@ -26,11 +26,11 @@ interface PaymentProcessor { } /** - * @param stripeCustomerId Stripe customer id - * @param stripeSourceId Stripe source id + * @param customerId Stripe customer id + * @param sourceId Stripe source id * @return Stripe sourceId if created */ - fun addSource(stripeCustomerId: String, stripeSourceId: String): Either + fun addSource(customerId: String, sourceId: String): Either /** * @param customerId: Prime unique identifier for customer @@ -40,10 +40,10 @@ interface PaymentProcessor { fun createPaymentProfile(customerId: String, email: String): Either /** - * @param stripeCustomerId Stripe customer id + * @param customerId Stripe customer id * @return Stripe customerId if deleted */ - fun deletePaymentProfile(stripeCustomerId: String): Either + fun deletePaymentProfile(customerId: String): Either /** * @param customerId: user email (Prime unique identifier for customer) @@ -69,12 +69,12 @@ interface PaymentProcessor { /** * @param Stripe Plan Id - * @param stripeCustomerId Stripe Customer Id + * @param customerId Stripe Customer Id * @param trielEnd Epoch timestamp for when the trial period ends * @param taxRegion An identifier representing the taxes to be applied to a region * @return Stripe SubscriptionId if subscribed */ - fun createSubscription(planId: String, stripeCustomerId: String, trialEnd: Long = 0L, taxRegionId: String? = null): Either + fun createSubscription(planId: String, customerId: String, trialEnd: Long = 0L, taxRegionId: String? = null): Either /** * @param Stripe Subscription Id @@ -96,23 +96,23 @@ interface PaymentProcessor { fun removeProduct(productId: String): Either /** - * @param stripeCustomerId Stripe customer id + * @param customerId Stripe customer id * @return List of Stripe sourceId */ - fun getSavedSources(stripeCustomerId: String): Either> + fun getSavedSources(customerId: String): Either> /** - * @param stripeCustomerId Stripe customer id + * @param customerId Stripe customer id * @return Stripe default sourceId */ - fun getDefaultSource(stripeCustomerId: String): Either + fun getDefaultSource(customerId: String): Either /** - * @param stripeCustomerId Stripe customer id + * @param customerId Stripe customer id * @param sourceId Stripe source id * @return SourceInfo if created */ - fun setDefaultSource(stripeCustomerId: String, sourceId: String): Either + fun setDefaultSource(customerId: String, sourceId: String): Either /** * @param customerId Customer id in the payment system @@ -144,11 +144,11 @@ interface PaymentProcessor { fun refundCharge(chargeId: String, amount: Int): Either /** - * @param stripeCustomerId Customer id in the payment system + * @param customerId Customer id in the payment system * @param sourceId id of the payment source * @return id if removed */ - fun removeSource(stripeCustomerId: String, sourceId: String): Either + fun removeSource(customerId: String, sourceId: String): Either fun getStripeEphemeralKey(customerId: String, email: String, apiVersion: String): Either From 7093af20d915f83f91a7cc31efca3640f0b71022 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 14 Nov 2019 13:13:07 +0100 Subject: [PATCH 272/309] Refactors delete-payment-profile in payment API --- .../StripePaymentProcessorTest.kt | 2 +- .../StripePaymentProcessor.kt | 23 +++++++++++++------ .../paymentprocessor/PaymentProcessor.kt | 4 ++-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt b/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt index 58df31f52..c227578a6 100644 --- a/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt +++ b/payment-processor/src/integration-test/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessorTest.kt @@ -75,7 +75,7 @@ class StripePaymentProcessorTest { @After fun cleanUp() { - val resultDelete = paymentProcessor.deletePaymentProfile(customerId) + val resultDelete = paymentProcessor.removePaymentProfile(customerId) assertNotFailure(resultDelete) } diff --git a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt index aa8cf5953..3f2f5d2ab 100644 --- a/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt +++ b/payment-processor/src/main/kotlin/org/ostelco/prime/paymentprocessor/StripePaymentProcessor.kt @@ -148,6 +148,14 @@ class StripePaymentProcessor : PaymentProcessor { } override fun getPaymentProfile(customerId: String): Either = + getCustomer(customerId) + .flatMap { customer -> + ProfileInfo(customer.id) + .right() + } + + /* Fetch customer from Stripe with result checks. */ + private fun getCustomer(customerId: String): Either = Try { Customer.retrieve(customerId) }.fold( @@ -155,12 +163,12 @@ class StripePaymentProcessor : PaymentProcessor { when { customer.deleted == true -> NotFoundError("Payment profile for user $customerId was previously deleted") .left() - else -> ProfileInfo(customer.id) + else -> customer .right() } }, ifFailure = { - NotFoundError("Could not find a payment profile for user $customerId") + NotFoundError("Could not find a payment profile for customer $customerId") .left() } ) @@ -220,11 +228,12 @@ class StripePaymentProcessor : PaymentProcessor { SourceInfo(Customer.retrieve(customerId).defaultSource) } - override fun deletePaymentProfile(customerId: String): Either = - either("Failed to delete customer $customerId") { - val customer = Customer.retrieve(customerId) - ProfileInfo(customer.delete().id) - } + override fun removePaymentProfile(customerId: String): Either = + getCustomer(customerId) + .flatMap { customer -> + ProfileInfo(customer.delete().id) + .right() + } /* The 'expand' part will cause an immediate attempt at charging for the subscription when creating it. For interpreting the result see: diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt index d91988279..2756ee057 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/paymentprocessor/PaymentProcessor.kt @@ -41,9 +41,9 @@ interface PaymentProcessor { /** * @param customerId Stripe customer id - * @return Stripe customerId if deleted + * @return Stripe customerId if removed */ - fun deletePaymentProfile(customerId: String): Either + fun removePaymentProfile(customerId: String): Either /** * @param customerId: user email (Prime unique identifier for customer) From 3a2b52d5aa58c743564629cdaa18feb23415afdc Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 14 Nov 2019 14:32:04 +0100 Subject: [PATCH 273/309] Remove corresponding payment profile when removing a customer --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 24 +++++++++++++++++++ .../prime/storage/graph/Neo4jStoreTest.kt | 4 ++++ .../org/ostelco/prime/apierror/ApiError.kt | 1 + .../org/ostelco/prime/storage/StoreError.kt | 8 +++++++ 4 files changed, 37 insertions(+) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 97ed83a4c..5260336dc 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -99,6 +99,7 @@ import org.ostelco.prime.storage.NotCreatedError import org.ostelco.prime.storage.NotDeletedError import org.ostelco.prime.storage.NotFoundError import org.ostelco.prime.storage.NotUpdatedError +import org.ostelco.prime.storage.PartiallyNotDeletedError import org.ostelco.prime.storage.ScanInformationStore import org.ostelco.prime.storage.StoreError import org.ostelco.prime.storage.SystemError @@ -488,6 +489,29 @@ object Neo4jStoreSingleton : GraphStore { ifTrue = {}, ifFalse = { NotFoundError(type = identityEntity.name, id = identity.id) }) }.bind() + + /* If removal of payment profile fails, then the customer will be deleted + in neo4j but will still be present in payment backend. In that case the + profile must be removed from the payment backend manually. */ + paymentProcessor.removePaymentProfile(customerId) + .fold( + { + if (it is org.ostelco.prime.paymentprocessor.core.NotFoundError) { + /* Ignore. Customer has not bought products yet. */ + Unit.right() + } else { + logger.error(NOTIFY_OPS_MARKER, + "Removing corresponding payment profile when removing customer $customerId " + + "failed with error ${it.message} : ${it.description}") + PartiallyNotDeletedError(type = customerEntity.name, + id = "Failed to remove corresponding payment profile when removing customer $customerId", + error = it).left() + } + }, + { + Unit.right() + } + ).bind() }.fix() }.unsafeRunSync() .ifFailedThenRollback(transaction) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt index c360822f8..be5112319 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt @@ -1165,6 +1165,10 @@ class Neo4jStoreTest { @Test fun `test delete customer`() { + // mock + `when`(mockPaymentProcessor.removePaymentProfile(customerId = CUSTOMER.id)) + .thenReturn(ProfileInfo(EMAIL).right()) + // setup job { create { Region("sg", "Singapore") } diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt index eebe0ac44..ef4091475 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt @@ -52,6 +52,7 @@ object ApiErrorMapper { is org.ostelco.prime.storage.NotCreatedError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.NotUpdatedError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.NotDeletedError -> InternalServerError(description, errorCode) + is org.ostelco.prime.storage.PartiallyNotDeletedError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.ValidationError -> ForbiddenError(description, errorCode, storeError) is org.ostelco.prime.storage.FileDownloadError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.FileDeleteError -> InternalServerError(description, errorCode) diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt index 3c8a6c926..21051e2f8 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt @@ -51,6 +51,14 @@ class NotDeletedError(type: String, message = "$type - $id not deleted.", error = error) +class PartiallyNotDeletedError(type: String, + id: String, + error: InternalError? = null) : + StoreError(type = type, + id = id, + message = "$type - $id not deleted.", + error = error) + class ValidationError(type: String, id: String, message: String, From f85ec1e9dd69114960862485040556ebb48698ee Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Wed, 20 Nov 2019 17:31:33 +0100 Subject: [PATCH 274/309] Fix class not found --- kts-engine/build.gradle.kts | 2 ++ neo4j-store/build.gradle.kts | 2 -- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kts-engine/build.gradle.kts b/kts-engine/build.gradle.kts index fd49560a8..bcc210486 100644 --- a/kts-engine/build.gradle.kts +++ b/kts-engine/build.gradle.kts @@ -18,6 +18,8 @@ dependencies { implementation(project(":prime-modules")) implementation("com.fasterxml.jackson.core:jackson-databind:${Version.jacksonDatabind}") + implementation("net.java.dev.jna:jna:5.5.0") + testImplementation(kotlin("test")) testImplementation(kotlin("test-junit")) diff --git a/neo4j-store/build.gradle.kts b/neo4j-store/build.gradle.kts index 59dfba558..e675bd518 100644 --- a/neo4j-store/build.gradle.kts +++ b/neo4j-store/build.gradle.kts @@ -14,8 +14,6 @@ dependencies { implementation("org.neo4j.driver:neo4j-java-driver:${Version.neo4jDriver}") implementation("org.neo4j:neo4j-slf4j:${Version.neo4j}") - implementation("com.sun.jna:jna:3.0.9") - testImplementation("com.palantir.docker.compose:docker-compose-rule-junit4:${Version.dockerComposeJunitRule}") testImplementation(kotlin("test")) diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index b6601122a..b70787748 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.73.1" +version = "1.73.2" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 63cc2dbf0..23410f1a1 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.73.1,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.73.2,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml From 967c8eda95c5d833eee63e1f9b06c0703fe43840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 18:24:13 +0100 Subject: [PATCH 275/309] An attempt at breaking a large method into several smaller ones. --- .../admin/PeriodicProvisioningTask.kt | 47 ++++++++++--------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt index 518b60668..1f397379e 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt @@ -76,36 +76,37 @@ class PreallocateProfilesTask( } } - - // TODO: This method must be refactored. It is still _way_ too complex. private fun preProvisionSimProfile(hssEntry: HssEntry, simEntry: SimEntry): Either = if (simEntry.id == null) { // TODO: This idiom is _bad_, find something better! AdapterError("simEntry.id == null for simEntry = '$simEntry'.").left() } else getProfileVendorAdapterForProfileVendorId(simEntry.profileVendorId) - .flatMap { profileVendorAdapter -> - when { - simEntry.hssState == HssState.NOT_ACTIVATED -> { - logger.debug("Preallocating (HSS not activated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") - - profileVendorAdapter.activate(simEntry = simEntry) - .flatMap { - hssAdapterProxy.activate(simEntry) - } - .flatMap { - simInventoryDAO.setHssState(simEntry.id, HssState.ACTIVATED) - } - - } - else -> { - // TODO: THis looks like bug! It looks like the preallocation will _either_ run against the HSS, _or_ against the profile vendor adapter. - // This is clearly wrong, it should run against both. - logger.debug("Preallocating (HSS preactivated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") - profileVendorAdapter.activate(simEntry = simEntry) - } - } + .flatMap { preProvisionProfileAdapter(simEntry, hssEntry, simEntry.id, it) } + + + private fun preProvisionProfileAdapter(simEntry: SimEntry, hssEntry: HssEntry, id: Long, profileVendorAdapter: ProfileVendorAdapter): Either { + when { + simEntry.hssState == HssState.NOT_ACTIVATED -> { + logger.debug("Preallocating (HSS not activated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") + + profileVendorAdapter.activate(simEntry = simEntry) + .flatMap { + hssAdapterProxy.activate(simEntry) + } + .flatMap { + simInventoryDAO.setHssState(id, HssState.ACTIVATED) } + } + else -> { + // TODO: THis looks like bug! It looks like the preallocation will _either_ run against the HSS, _or_ against the profile vendor adapter. + // This is clearly wrong, it should run against both. + logger.debug("Preallocating (HSS preactivated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") + profileVendorAdapter.activate(simEntry = simEntry) + } + } + } + private fun batchPreprovisionSimProfiles(hssEntry: HssEntry, From add49c638ebaebd9bd1b2c0a5b9b6aef4ff42e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 18:33:56 +0100 Subject: [PATCH 276/309] Only use the OYA_LOLTEL_STD_ACB profiles for loltel --- prime/config/config.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/prime/config/config.yaml b/prime/config/config.yaml index df48ee342..1ca12ecb3 100644 --- a/prime/config/config.yaml +++ b/prime/config/config.yaml @@ -156,10 +156,8 @@ modules: profile: OYA_M1_STANDARD_ACB - regex: "Digi.*" profile: OYA_DIGI_STANDARD_ACB - - regex: "Loltel.android" - profile: Loltel_ANDROID_1 - regex: "Loltel.*" - profile: LOLTEL_IPHONE_1 + profile: OYA_LOLTEL_STD_ACB database: driverClass: org.postgresql.Driver user: ${DB_USER} From 31acb9d1f05a959544075595764bef50cd59b475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 21:56:35 +0100 Subject: [PATCH 277/309] Making it cwork again --- .../admin/PeriodicProvisioningTask.kt | 47 +++++++++---------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt index 1f397379e..518b60668 100644 --- a/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt +++ b/sim-administration/simmanager/src/main/kotlin/org/ostelco/simcards/admin/PeriodicProvisioningTask.kt @@ -76,37 +76,36 @@ class PreallocateProfilesTask( } } + + // TODO: This method must be refactored. It is still _way_ too complex. private fun preProvisionSimProfile(hssEntry: HssEntry, simEntry: SimEntry): Either = if (simEntry.id == null) { // TODO: This idiom is _bad_, find something better! AdapterError("simEntry.id == null for simEntry = '$simEntry'.").left() } else getProfileVendorAdapterForProfileVendorId(simEntry.profileVendorId) - .flatMap { preProvisionProfileAdapter(simEntry, hssEntry, simEntry.id, it) } - - - private fun preProvisionProfileAdapter(simEntry: SimEntry, hssEntry: HssEntry, id: Long, profileVendorAdapter: ProfileVendorAdapter): Either { - when { - simEntry.hssState == HssState.NOT_ACTIVATED -> { - logger.debug("Preallocating (HSS not activated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") - - profileVendorAdapter.activate(simEntry = simEntry) - .flatMap { - hssAdapterProxy.activate(simEntry) - } - .flatMap { - simInventoryDAO.setHssState(id, HssState.ACTIVATED) + .flatMap { profileVendorAdapter -> + when { + simEntry.hssState == HssState.NOT_ACTIVATED -> { + logger.debug("Preallocating (HSS not activated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") + + profileVendorAdapter.activate(simEntry = simEntry) + .flatMap { + hssAdapterProxy.activate(simEntry) + } + .flatMap { + simInventoryDAO.setHssState(simEntry.id, HssState.ACTIVATED) + } + + } + else -> { + // TODO: THis looks like bug! It looks like the preallocation will _either_ run against the HSS, _or_ against the profile vendor adapter. + // This is clearly wrong, it should run against both. + logger.debug("Preallocating (HSS preactivated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") + profileVendorAdapter.activate(simEntry = simEntry) + } + } } - } - else -> { - // TODO: THis looks like bug! It looks like the preallocation will _either_ run against the HSS, _or_ against the profile vendor adapter. - // This is clearly wrong, it should run against both. - logger.debug("Preallocating (HSS preactivated) for HSS with ID/metricName ${hssEntry.id}/${hssEntry.name} simEntry with ICCID=${simEntry.iccid}") - profileVendorAdapter.activate(simEntry = simEntry) - } - } - } - private fun batchPreprovisionSimProfiles(hssEntry: HssEntry, From c38d2f9b1c795941a4ba3c5f134c3b9f927566dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 22:24:41 +0100 Subject: [PATCH 278/309] Make names shorter --- .../sim-batch-management/es2plus/es2plus.go | 26 +++++++++---------- .../sim-batch-management/sim-batch-mgt.go | 4 +-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index d89c387bc..b1164f156 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -14,7 +14,7 @@ import ( /// /// External interface /// -type Es2PlusClient interface { +type Client interface { GetStatus(iccid string) (*ProfileStatus, error) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) @@ -146,7 +146,7 @@ type ES2PlusConfirmOrderResponse struct { // Generating new ES2Plus clients // -type Es2PlusClientState struct { +type ClientState struct { httpClient *http.Client hostport string requesterId string @@ -154,8 +154,8 @@ type Es2PlusClientState struct { logHeaders bool } -func NewClient(certFilePath string, keyFilePath string, hostport string, requesterId string) *Es2PlusClientState { - return &Es2PlusClientState{ +func NewClient(certFilePath string, keyFilePath string, hostport string, requesterId string) *ClientState { + return &ClientState{ httpClient: newHttpClient(certFilePath, keyFilePath), hostport: hostport, requesterId: requesterId, @@ -227,7 +227,7 @@ func newUuid() (string, error) { return uuid.URN(), nil } -func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { +func newEs2plusHeader(client Client) (*ES2PlusHeader, error) { functionCallIdentifier, err := newUuid() if err != nil { @@ -237,7 +237,7 @@ func newEs2plusHeader(client Es2PlusClient) (*ES2PlusHeader, error) { return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } -func (client *Es2PlusClientState) execute( +func (client *ClientState) execute( es2plusCommand string, payload interface{}, result interface{}) error { @@ -281,7 +281,7 @@ func (client *Es2PlusClientState) execute( /// Externally visible API for Es2Plus protocol /// -func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error) { +func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" header, err := newEs2plusHeader(client) @@ -305,7 +305,7 @@ func (client *Es2PlusClientState) GetStatus(iccid string) (*ProfileStatus, error } } -func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { +func (client *ClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { result := new(ES2PlusRecoverProfileResponse) es2plusCommand := "recoverProfile" header, err := newEs2plusHeader(client) @@ -320,7 +320,7 @@ func (client *Es2PlusClientState) RecoverProfile(iccid string, targetState strin return result, client.execute(es2plusCommand, payload, result) } -func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { +func (client *ClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { result := new(ES2PlusCancelOrderResponse) es2plusCommand := "cancelOrder" header, err := newEs2plusHeader(client) @@ -335,7 +335,7 @@ func (client *Es2PlusClientState) CancelOrder(iccid string, targetState string) return result, client.execute(es2plusCommand, payload, result) } -func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { +func (client *ClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { result := new(ES2PlusDownloadOrderResponse) es2plusCommand := "downloadOrder" header, err := newEs2plusHeader(client) @@ -360,7 +360,7 @@ func (client *Es2PlusClientState) DownloadOrder(iccid string) (*ES2PlusDownloadO } } -func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { +func (client *ClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { result := new(ES2PlusConfirmOrderResponse) es2plusCommand := "confirmOrder" header, err := newEs2plusHeader(client) @@ -389,7 +389,7 @@ func (client *Es2PlusClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrd } } -func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { +func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { result, err := client.GetStatus(iccid) if err != nil { @@ -418,6 +418,6 @@ func (client *Es2PlusClientState) ActivateIccid(iccid string) (*ProfileStatus, e } // TODO: This shouldn't have to be public, but how can it be avoided? -func (clientState *Es2PlusClientState) RequesterId() string { +func (clientState *ClientState) RequesterId() string { return clientState.requesterId } diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sim-batch-mgt.go index 8da1cc0fb..1867a31ae 100755 --- a/sim-administration/sim-batch-management/sim-batch-mgt.go +++ b/sim-administration/sim-batch-management/sim-batch-mgt.go @@ -919,7 +919,7 @@ func GenerateInputFile(batch *model.Batch) string { return result } -func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Es2PlusClient, error) { +func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Client, error) { vendor, err := db.GetProfileVendorByName(vendorName) if err != nil { return nil, err @@ -933,7 +933,7 @@ func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Es2PlusCl return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId), nil } -func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Es2PlusClient, *model.Batch, error) { +func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Client, *model.Batch, error) { batch, err := db.GetBatchByName(batchName) if err != nil { From 80364421e21ad5889653925724fb7dd5f70e5e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Wed, 20 Nov 2019 22:25:48 +0100 Subject: [PATCH 279/309] rename file to the command name we want to use --- .../sim-batch-management/{sim-batch-mgt.go => sbm.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sim-administration/sim-batch-management/{sim-batch-mgt.go => sbm.go} (100%) diff --git a/sim-administration/sim-batch-management/sim-batch-mgt.go b/sim-administration/sim-batch-management/sbm.go similarity index 100% rename from sim-administration/sim-batch-management/sim-batch-mgt.go rename to sim-administration/sim-batch-management/sbm.go From 4caa4ec18868754fac539032c222e0055a212b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 07:22:06 +0100 Subject: [PATCH 280/309] Making es2+ header struct non-exported --- .../sim-batch-management/es2plus/es2plus.go | 18 +++---- .../sim-batch-management/sbm.go | 50 ++++++++----------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index b1164f156..b53920ed2 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -12,7 +12,7 @@ import ( ) /// -/// External interface +/// External interface for ES2+ client /// type Client interface { GetStatus(iccid string) (*ProfileStatus, error) @@ -27,13 +27,13 @@ type Client interface { /// /// Generic headers for invocations and responses /// -type ES2PlusHeader struct { +type es2PlusHeader struct { FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` FunctionCallIdentifier string `json:"functionCallIdentifier"` } type ES2PlusGetProfileStatusRequest struct { - Header ES2PlusHeader `json:"header"` + Header es2PlusHeader `json:"header"` IccidList []ES2PlusIccid `json:"iccidList"` } @@ -81,7 +81,7 @@ type ProfileStatus struct { // type ES2PlusRecoverProfileRequest struct { - Header ES2PlusHeader `json:"header"` + Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` ProfileStatus string `json:"profileStatus"` } @@ -95,7 +95,7 @@ type ES2PlusRecoverProfileResponse struct { // type ES2PlusCancelOrderRequest struct { - Header ES2PlusHeader `json:"header"` + Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` } @@ -109,7 +109,7 @@ type ES2PlusCancelOrderResponse struct { // type ES2PlusDownloadOrderRequest struct { - Header ES2PlusHeader `json:"header"` + Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` Profiletype string `json:"profiletype,omitempty"` @@ -125,7 +125,7 @@ type ES2PlusDownloadOrderResponse struct { // type ES2PlusConfirmOrderRequest struct { - Header ES2PlusHeader `json:"header"` + Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` MatchingId string `json:"matchingId,omitempty"` @@ -227,14 +227,14 @@ func newUuid() (string, error) { return uuid.URN(), nil } -func newEs2plusHeader(client Client) (*ES2PlusHeader, error) { +func newEs2plusHeader(client Client) (*es2PlusHeader, error) { functionCallIdentifier, err := newUuid() if err != nil { return nil, err } - return &ES2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil + return &es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil } func (client *ClientState) execute( diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index 1867a31ae..02cb99471 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -26,8 +26,6 @@ var ( // TODO: Global flags can be added to Kingpin, but also make it have an effect. // debug = kingpin.Flag("debug", "enable debug mode").Default("false").Bool() - - /// /// Profile-vendor - centric commands /// @@ -102,25 +100,20 @@ var ( addMsisdnFromFileCsvfile = addMsisdnFromFile.Flag("csv-file", "The CSV file to read from").Required().ExistingFile() addMsisdnFromFileAddLuhn = addMsisdnFromFile.Flag("add-luhn-checksums", "Assume that the checksums for the ICCIDs are not present, and add them").Default("false").Bool() + bwBatch = kingpin.Command("batch-write-hss", "Generate a batch upload script") + bwBatchName = bwBatch.Arg("batch-name", "The batch to generate upload script from").String() + bwOutputDirName = bwBatch.Arg("output-dir-name", "The directory in which to place the output file.").String() - bwBatch = kingpin.Command("batch-write-hss", "Generate a batch upload script") - bwBatchName = bwBatch.Arg("batch-name", "The batch to generate upload script from").String() - bwOutputDirName = bwBatch.Arg("output-dir-name", "The directory in which to place the output file.").String() - - - spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") - spBatchName = spUpload.Arg("batch-name", "The batch to augment").Required().String() - spUploadInputFile = spUpload.Arg("input-file", "path to .out file used as input file").Required().String() + spUpload = kingpin.Command("batch-read-out-file", "Convert an output (.out) file from an sim profile producer into an input file for an HSS.") + spBatchName = spUpload.Arg("batch-name", "The batch to augment").Required().String() + spUploadInputFile = spUpload.Arg("input-file", "path to .out file used as input file").Required().String() - - generateUploadBatch = kingpin.Command("batch-generate-upload-script", "Write a file that can be used by an HSS to insert profiles.") + generateUploadBatch = kingpin.Command("batch-generate-upload-script", "Write a file that can be used by an HSS to insert profiles.") generateUploadBatchBatch = generateUploadBatch.Arg("batch", "The batch to output from").Required().String() - -// TODO: Delete this asap! -// spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", -// "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() - + // TODO: Delete this asap! + // spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", + // "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() generateActivationCodeSql = kingpin.Command("batch-generate-activation-code-updating-sql", "Generate SQL code to update access codes") generateActivationCodeSqlBatch = generateActivationCodeSql.Arg("batch-name", "The batch to generate sql coce for").String() @@ -147,7 +140,7 @@ var ( dbHssVendor = bd.Flag("hss-vendor", "The HSS vendor").Default("M1").String() dbUploadHostname = bd.Flag("upload-hostname", "host to upload batch to").Default("localhost").String() dbUploadPortnumber = bd.Flag("upload-portnumber", "port to upload to").Default("8080").String() - dbProfileVendor = bd.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() + dbProfileVendor = bd.Flag("profile-vendor", "Vendor of SIM profiles").Default("Idemia").String() dbInitialHlrActivationStatusOfProfiles = bd.Flag( "initial-hlr-activation-status-of-profiles", @@ -327,7 +320,7 @@ func parseCommandLine() error { return fmt.Errorf("no batch found with name '%s'", *spBatchName) } - outRecord, err := outfileparser.ParseOutputFile(*spUploadInputFile) + outRecord, err := outfileparser.ParseOutputFile(*spUploadInputFile) if err != nil { return err } @@ -337,13 +330,16 @@ func parseCommandLine() error { outRecord.NoOfEntries, batch.Quantity, batch.Name) } - for _, e := range outRecord.Entries { simProfile, err := db.GetSimProfileByIccid(e.IccidWithChecksum) - if err != nil {return err} - if simProfile == nil { return fmt.Errorf("couldn't find profile enty for IMSI=%s", e.Imsi)} - if simProfile.Imsi != e.Imsi{ - return fmt.Errorf("profile enty for ICCID=%s has IMSI (%s), but we expected (%s)",e.Iccid, e.Imsi, simProfile.Imsi) + if err != nil { + return err + } + if simProfile == nil { + return fmt.Errorf("couldn't find profile enty for IMSI=%s", e.Imsi) + } + if simProfile.Imsi != e.Imsi { + return fmt.Errorf("profile enty for ICCID=%s has IMSI (%s), but we expected (%s)", e.Iccid, e.Imsi, simProfile.Imsi) } db.UpdateSimEntryKi(simProfile.Id, e.Ki) } @@ -352,7 +348,6 @@ func parseCommandLine() error { // with a commit. weCool = true - case "batch-write-hss": batch, err := db.GetBatchByName(*bwBatchName) @@ -365,14 +360,13 @@ func parseCommandLine() error { return fmt.Errorf("no batch found with name '%s'", *bwBatchName) } - outputFile := fmt.Sprintf("%s/%s.csv", *bwOutputDirName, batch.Name) + outputFile := fmt.Sprintf("%s/%s.csv", *bwOutputDirName, batch.Name) log.Println("outputFile = ", outputFile) if err := outfileparser.WriteHssCsvFile(outputFile, db, batch); err != nil { return fmt.Errorf("couldn't write hss output to file '%s', . Error = '%v'", outputFile, err) } - case "batches-list": allBatches, err := db.GetAllBatches() if err != nil { @@ -584,14 +578,12 @@ func parseCommandLine() error { *dbProfileVendor, *dbInitialHlrActivationStatusOfProfiles) - if err != nil { return err } log.Printf("Declared batch '%s'", batch.Name) return nil - case "iccid-get-status": client, err := ClientForVendor(db, *getStatusProfileVendor) if err != nil { From 7e13b3810ae2b36e30f6ef58284b7093cba46ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 07:23:52 +0100 Subject: [PATCH 281/309] Making es2plus iccid fields package local --- sim-administration/sim-batch-management/es2plus/es2plus.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index b53920ed2..e914a1272 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -34,10 +34,10 @@ type es2PlusHeader struct { type ES2PlusGetProfileStatusRequest struct { Header es2PlusHeader `json:"header"` - IccidList []ES2PlusIccid `json:"iccidList"` + IccidList []es2PlusIccid `json:"iccidList"` } -type ES2PlusIccid struct { +type es2PlusIccid struct { Iccid string `json:"iccid"` } @@ -290,7 +290,7 @@ func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { } payload := &ES2PlusGetProfileStatusRequest{ Header: *header, - IccidList: []ES2PlusIccid{ES2PlusIccid{Iccid: iccid}}, + IccidList: []es2PlusIccid{es2PlusIccid{Iccid: iccid}}, } if err = client.execute(es2plusCommand, payload, result); err != nil { return nil, err From 3c3fc20578f717f31f3d9d2bfa024da5e78e0894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 07:25:09 +0100 Subject: [PATCH 282/309] Making ES2PlusConfirmOrderResponse package local --- .../sim-batch-management/es2plus/es2plus.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index e914a1272..7d841ef44 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -46,7 +46,7 @@ type FunctionExecutionStatus struct { StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` } -type ES2PlusResponseHeader struct { +type es2PlusResponseHeader struct { FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` } @@ -62,7 +62,7 @@ type ES2PlusStatusCodeData struct { } type ES2ProfileStatusResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header es2PlusResponseHeader `json:"header"` ProfileStatusList []ProfileStatus `json:"profileStatusList"` CompletionTimestamp string `json:"completionTimestamp"` } @@ -87,7 +87,7 @@ type ES2PlusRecoverProfileRequest struct { } type ES2PlusRecoverProfileResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header es2PlusResponseHeader `json:"header"` } // @@ -101,7 +101,7 @@ type ES2PlusCancelOrderRequest struct { } type ES2PlusCancelOrderResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header es2PlusResponseHeader `json:"header"` } // @@ -116,7 +116,7 @@ type ES2PlusDownloadOrderRequest struct { } type ES2PlusDownloadOrderResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header es2PlusResponseHeader `json:"header"` Iccid string `json:"iccid"` } @@ -135,7 +135,7 @@ type ES2PlusConfirmOrderRequest struct { } type ES2PlusConfirmOrderResponse struct { - Header ES2PlusResponseHeader `json:"header"` + Header es2PlusResponseHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` MatchingId string `json:"matchingId,omitempty"` From 559f1eff1f1fe882befbb6868a4d8fb23e4405e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 07:45:07 +0100 Subject: [PATCH 283/309] Refactoring more, adding comments and fixing entity names --- .../sim-batch-management/es2plus/es2plus.go | 104 ++++++++++-------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 7d841ef44..a9c7f0f93 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -16,19 +16,19 @@ import ( /// type Client interface { GetStatus(iccid string) (*ProfileStatus, error) - RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) - CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) - DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) - ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) + RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) + CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) + DownloadOrder(iccid string) (*DownloadOrderResponse, error) + ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) ActivateIccid(iccid string) (*ProfileStatus, error) - RequesterId() string + RequesterID() string } /// /// Generic headers for invocations and responses /// type es2PlusHeader struct { - FunctionRequesterIdentifier string `json:"functionRequesterIdentifier"` + FunctionrequesterIDentifier string `json:"functionrequesterIDentifier"` FunctionCallIdentifier string `json:"functionCallIdentifier"` } @@ -43,7 +43,7 @@ type es2PlusIccid struct { type FunctionExecutionStatus struct { FunctionExecutionStatusType string `json:"status"` - StatusCodeData ES2PlusStatusCodeData `json:"statusCodeData"` + StatusCodeData es2PlusStatusCodeData `json:"statusCodeData"` } type es2PlusResponseHeader struct { @@ -54,7 +54,7 @@ type es2PlusResponseHeader struct { // Status code invocation. // -type ES2PlusStatusCodeData struct { +type es2PlusStatusCodeData struct { SubjectCode string `json:"subjectCode"` ReasonCode string `json:"reasonCode"` SubjectIdentifier string `json:"subjectIdentifier"` @@ -67,6 +67,9 @@ type ES2ProfileStatusResponse struct { CompletionTimestamp string `json:"completionTimestamp"` } +/// +/// The status of a profile as retrieved from the SM-DP+ +/// type ProfileStatus struct { StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` ACToken string `json:"acToken"` @@ -80,13 +83,13 @@ type ProfileStatus struct { // Profile reset invocation // -type ES2PlusRecoverProfileRequest struct { +type es2PlusRecoverProfileRequest struct { Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` ProfileStatus string `json:"profileStatus"` } -type ES2PlusRecoverProfileResponse struct { +type RecoverProfileResponse struct { Header es2PlusResponseHeader `json:"header"` } @@ -94,13 +97,13 @@ type ES2PlusRecoverProfileResponse struct { // Cancel order invocation // -type ES2PlusCancelOrderRequest struct { +type es2PlusCancelOrderRequest struct { Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` } -type ES2PlusCancelOrderResponse struct { +type CancelOrderResponse struct { Header es2PlusResponseHeader `json:"header"` } @@ -108,14 +111,14 @@ type ES2PlusCancelOrderResponse struct { // Download order invocation // -type ES2PlusDownloadOrderRequest struct { +type downloadOrderRequest struct { Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` Profiletype string `json:"profiletype,omitempty"` } -type ES2PlusDownloadOrderResponse struct { +type DownloadOrderResponse struct { Header es2PlusResponseHeader `json:"header"` Iccid string `json:"iccid"` } @@ -124,21 +127,21 @@ type ES2PlusDownloadOrderResponse struct { // ConfirmOrder invocation // -type ES2PlusConfirmOrderRequest struct { +type confirmOrderRequest struct { Header es2PlusHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` - MatchingId string `json:"matchingId,omitempty"` + MatchingID string `json:"matchingId,omitempty"` ConfirmationCode string `json:"confirmationCode,omitempty"` SmdpAddress string `json:"smdpAddress,omitempty"` ReleaseFlag bool `json:"releaseFlag"` } -type ES2PlusConfirmOrderResponse struct { +type ConfirmOrderResponse struct { Header es2PlusResponseHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` - MatchingId string `json:"matchingId,omitempty"` + MatchingID string `json:"matchingId,omitempty"` SmdpAddress string `json:"smdpAddress,omitempty"` } @@ -149,22 +152,22 @@ type ES2PlusConfirmOrderResponse struct { type ClientState struct { httpClient *http.Client hostport string - requesterId string + requesterID string logPayload bool logHeaders bool } -func NewClient(certFilePath string, keyFilePath string, hostport string, requesterId string) *ClientState { +func NewClient(certFilePath string, keyFilePath string, hostport string, requesterID string) *ClientState { return &ClientState{ - httpClient: newHttpClient(certFilePath, keyFilePath), + httpClient: newHTTPClient(certFilePath, keyFilePath), hostport: hostport, - requesterId: requesterId, + requesterID: requesterID, logPayload: false, logHeaders: false, } } -func newHttpClient(certFilePath string, keyFilePath string) *http.Client { +func newHTTPClient(certFilePath string, keyFilePath string) *http.Client { cert, err := tls.LoadX509KeyPair( certFilePath, keyFilePath) @@ -219,7 +222,7 @@ func formatRequest(r *http.Request) string { return strings.Join(request, "\n") } -func newUuid() (string, error) { +func newUUID() (string, error) { uuid, err := uuid.NewRandom() if err != nil { return "", err @@ -229,12 +232,12 @@ func newUuid() (string, error) { func newEs2plusHeader(client Client) (*es2PlusHeader, error) { - functionCallIdentifier, err := newUuid() + functionCallIdentifier, err := newUUID() if err != nil { return nil, err } - return &es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionRequesterIdentifier: client.RequesterId()}, nil + return &es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionrequesterIDentifier: client.RequesterID()}, nil } func (client *ClientState) execute( @@ -281,6 +284,8 @@ func (client *ClientState) execute( /// Externally visible API for Es2Plus protocol /// + +// Get the status of a profile with a specific ICCID. func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(ES2ProfileStatusResponse) es2plusCommand := "getProfileStatus" @@ -305,14 +310,16 @@ func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { } } -func (client *ClientState) RecoverProfile(iccid string, targetState string) (*ES2PlusRecoverProfileResponse, error) { - result := new(ES2PlusRecoverProfileResponse) +// Recover the state of the profile with a particular ICCID, +// by setting it to the target state. +func (client *ClientState) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) { + result := new(RecoverProfileResponse) es2plusCommand := "recoverProfile" header, err := newEs2plusHeader(client) if err != nil { return nil, err } - payload := &ES2PlusRecoverProfileRequest{ + payload := &es2PlusRecoverProfileRequest{ Header: *header, Iccid: iccid, ProfileStatus: targetState, @@ -320,14 +327,16 @@ func (client *ClientState) RecoverProfile(iccid string, targetState string) (*ES return result, client.execute(es2plusCommand, payload, result) } -func (client *ClientState) CancelOrder(iccid string, targetState string) (*ES2PlusCancelOrderResponse, error) { - result := new(ES2PlusCancelOrderResponse) +// Cancel an order by setting the state of the profile with a particular ICCID, +// the target state. +func (client *ClientState) CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) { + result := new(CancelOrderResponse) es2plusCommand := "cancelOrder" header, err := newEs2plusHeader(client) if err != nil { return nil, err } - payload := &ES2PlusCancelOrderRequest{ + payload := &es2PlusCancelOrderRequest{ Header: *header, Iccid: iccid, FinalProfileStatusIndicator: targetState, @@ -335,14 +344,16 @@ func (client *ClientState) CancelOrder(iccid string, targetState string) (*ES2Pl return result, client.execute(es2plusCommand, payload, result) } -func (client *ClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderResponse, error) { - result := new(ES2PlusDownloadOrderResponse) +// Prepare the profile to be downloaded (first of two steps, the +// ConfirmDownload is also necessary). +func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, error) { + result := new(DownloadOrderResponse) es2plusCommand := "downloadOrder" header, err := newEs2plusHeader(client) if err != nil { return nil, err } - payload := &ES2PlusDownloadOrderRequest{ + payload := &downloadOrderRequest{ Header: *header, Iccid: iccid, Eid: "", @@ -360,19 +371,21 @@ func (client *ClientState) DownloadOrder(iccid string) (*ES2PlusDownloadOrderRes } } -func (client *ClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderResponse, error) { - result := new(ES2PlusConfirmOrderResponse) +// Second of the two steps that are necessary to prepare a profile for +// to be downloaded. +func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) { + result := new(ConfirmOrderResponse) es2plusCommand := "confirmOrder" header, err := newEs2plusHeader(client) if err != nil { return nil, err } - payload := &ES2PlusConfirmOrderRequest{ + payload := &confirmOrderRequest{ Header: *header, Iccid: iccid, Eid: "", ConfirmationCode: "", - MatchingId: "", + MatchingID: "", SmdpAddress: "", ReleaseFlag: true, } @@ -384,11 +397,16 @@ func (client *ClientState) ConfirmOrder(iccid string) (*ES2PlusConfirmOrderRespo executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType if executionStatus != "Executed-Success" { return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) - } else { - return result, nil } + + return result, nil } + +// Take a profile to the state "READY" where it can be downloaded. +// This function will if poll the current status of the profile, and if +// necessary advance the state by executing the DownloadOrder and +// ConfirmOrder functions. func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { result, err := client.GetStatus(iccid) @@ -418,6 +436,6 @@ func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { } // TODO: This shouldn't have to be public, but how can it be avoided? -func (clientState *ClientState) RequesterId() string { - return clientState.requesterId +func (clientState *ClientState) RequesterID() string { + return clientState.requesterID } From 43358929704129dce273c39d440029775923faae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 07:58:11 +0100 Subject: [PATCH 284/309] Fixing capitalization etc (this is btw a breaking change for existing databases) --- .../sim-batch-management/model/model.go | 4 +- .../outfileparser/outfileparser.go | 2 +- .../sim-batch-management/sbm.go | 8 ++-- .../sim-batch-management/store/store.go | 46 ++++++++++--------- .../sim-batch-management/store/store_test.go | 14 +++--- .../uploadtoprime/uploadtoprime.go | 2 +- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index e87490978..71ac1aea9 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -5,7 +5,7 @@ package model // about three of them. type SimEntry struct { Id int64 `db:"id" json:"id"` - BatchID int64 `db:"batchId" json:"batchId"` + BatchID int64 `db:"batchID" json:"batchID"` RawIccid string `db:"rawIccid" json:"rawIccid"` IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` IccidWithoutChecksum string `db:"iccidWithoutChecksum" json:"iccidWithoutChecksum"` @@ -18,7 +18,7 @@ type SimEntry struct { type Batch struct { - BatchId int64 `db:"id" json:"id"` // TODO: SHould this be called 'Id' + BatchID int64 `db:"id" json:"id"` // TODO: SHould this be called 'Id' Name string `db:"name" json:"name"` // TODO: Customer is a misnomer: This is the customer name used when diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 08f56b74c..bd1fc7968 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -282,7 +282,7 @@ func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) } - entries, err := sdb.GetAllSimEntriesForBatch(batch.BatchId) + entries, err := sdb.GetAllSimEntriesForBatch(batch.BatchID) if err != nil { return err } diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index 02cb99471..4d198d23e 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -236,7 +236,7 @@ func parseCommandLine() error { return err } - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + entries, err := db.GetAllSimEntriesForBatch(batch.BatchID) if err != nil { return err } @@ -402,7 +402,7 @@ func parseCommandLine() error { return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) } - simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchID) if err != nil { return err } @@ -518,7 +518,7 @@ func parseCommandLine() error { recordMap[record.iccid] = record } - simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchID) if err != nil { return err } @@ -777,7 +777,7 @@ func parseCommandLine() error { return err } - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + entries, err := db.GetAllSimEntriesForBatch(batch.BatchID) if err != nil { return err } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 87cae4ded..5cef34efb 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -15,10 +15,14 @@ import ( "strings" ) + +/// Holding database abstraction for the sim batch management system. type SimBatchDB struct { Db *sqlx.DB } +// An interface used to abstract the CRUD operations on the +// various entities in the sim batch management database. type Store interface { GenerateTables() error DropTables() error @@ -29,10 +33,10 @@ type Store interface { GetBatchByName(id string) (*model.Batch, error) CreateSimEntry(simEntry *model.SimEntry) error - UpdateSimEntryMsisdn(simId int64, msisdn string) - UpdateActivationCode(simId int64, activationCode string) error - UpdateSimEntryKi(simId int64, ki string) error - GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) + UpdateSimEntryMsisdn(simID int64, msisdn string) + UpdateActivationCode(simID int64, activationCode string) error + UpdateSimEntryKi(simID int64, ki string) error + GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, error) GetSimProfileByIccid(msisdn string) (*model.SimEntry, error) CreateProfileVendor(*model.ProfileVendor) error @@ -133,7 +137,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { if err != nil { return fmt.Errorf("getting last inserted id failed '%s'", err) } - theBatch.BatchId = id + theBatch.BatchID = id _, err = sdb.Db.NamedExec("UPDATE BATCH SET firstIccid = :firstIccid, firstImsi = :firstImsi, firstMsisdn = :firstMsisdn, msisdnIncrement = :msisdnIncrement, iccidIncrement = :iccidIncrement, imsiIncrement = :imsiIncrement, url=:url WHERE id = :id", theBatch) @@ -166,7 +170,7 @@ func (sdb *SimBatchDB) GenerateTables() error { sql = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( id INTEGER PRIMARY KEY AUTOINCREMENT, - batchId INTEGER NOT NULL, + batchID INTEGER NOT NULL, activationCode VARCHAR NOT NULL, imsi VARCHAR NOT NULL, rawIccid VARCHAR NOT NULL, @@ -247,7 +251,7 @@ func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { - res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchId, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", + res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchID, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", theEntry.BatchID, theEntry.ActivationCode, theEntry.RawIccid, @@ -267,9 +271,9 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { return err } -func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { +func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { result := []model.SimEntry{} - if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", simId); err != nil { + if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", simID); err != nil { return nil, err } @@ -280,9 +284,9 @@ func (sdb SimBatchDB) GetSimEntryById(simId int64) (*model.SimEntry, error) { } } -func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchId int64) ([]model.SimEntry, error) { +func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, error) { result := []model.SimEntry{} - if err := sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchId = ?", batchId) ; err != nil { + if err := sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchID = ?", batchID) ; err != nil { return nil, err } @@ -323,29 +327,29 @@ func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) } -func (sdb SimBatchDB) UpdateSimEntryMsisdn(simId int64, msisdn string) error { - _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simId", +func (sdb SimBatchDB) UpdateSimEntryMsisdn(simID int64, msisdn string) error { + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simID", map[string]interface{}{ - "simId": simId, + "simID": simID, "msisdn": msisdn, }) return err } -func (sdb SimBatchDB) UpdateSimEntryKi(simId int64, ki string) error { - _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET ki=:ki WHERE id = :simId", +func (sdb SimBatchDB) UpdateSimEntryKi(simID int64, ki string) error { + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET ki=:ki WHERE id = :simID", map[string]interface{}{ - "simId": simId, + "simID": simID, "ki": ki, }) return err } -func (sdb SimBatchDB) UpdateActivationCode(simId int64, activationCode string) error { - _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE id = :simId", +func (sdb SimBatchDB) UpdateActivationCode(simID int64, activationCode string) error { + _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE id = :simID", map[string]interface{}{ - "simId": simId, + "simID": simID, "activationCode": activationCode, }) return err @@ -528,7 +532,7 @@ func (sdb SimBatchDB) DeclareBatch( iccidWithLuhnChecksum := fmt.Sprintf("%d%d", iccidWithoutLuhnChecksum, fieldsyntaxchecks.LuhnChecksum(iccidWithoutLuhnChecksum)) simEntry := &model.SimEntry{ - BatchID: batch.BatchId, + BatchID: batch.BatchID, ActivationCode: "", RawIccid: fmt.Sprintf("%d", iccidWithoutLuhnChecksum), IccidWithChecksum: iccidWithLuhnChecksum, diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 8bf1e601f..3538d231c 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -129,7 +129,7 @@ func TestGetBatchById(t *testing.T) { injectTestprofileVendor(t) theBatch := injectTestBatch() - batchById, _ := sdb.GetBatchById(theBatch.BatchId) + batchById, _ := sdb.GetBatchById(theBatch.BatchID) if !reflect.DeepEqual(batchById, theBatch) { t.Logf("theBatch = %v\n", theBatch) @@ -196,7 +196,7 @@ func TestDeclareBatch(t *testing.T) { injectTestprofileVendor(t) theBatch := declareTestBatch(t) - retrievedValue, _ := sdb.GetBatchById(theBatch.BatchId) + retrievedValue, _ := sdb.GetBatchById(theBatch.BatchID) if retrievedValue == nil { t.Fatalf("Null retrievedValue") } @@ -204,7 +204,7 @@ func TestDeclareBatch(t *testing.T) { t.Fatal("getBatchById failed, stored batch not equal to retrieved batch") } - retrievedEntries, err := sdb.GetAllSimEntriesForBatch(theBatch.BatchId) + retrievedEntries, err := sdb.GetAllSimEntriesForBatch(theBatch.BatchID) if err != nil { t.Fatal(err) } @@ -260,10 +260,10 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { cleanTables() injectTestprofileVendor(t) theBatch := declareTestBatch(t) - batchId := theBatch.BatchId + batchID := theBatch.BatchID entry := model.SimEntry{ - BatchID: batchId, + BatchID: batchID, RawIccid: "1", IccidWithChecksum: "2", IccidWithoutChecksum: "3", @@ -291,10 +291,10 @@ func TestSimBatchDB_UpdateSimEntryKi(t *testing.T) { cleanTables() injectTestprofileVendor(t) theBatch := declareTestBatch(t) - batchId := theBatch.BatchId + batchID := theBatch.BatchID entry := model.SimEntry{ - BatchID: batchId, + BatchID: batchID, RawIccid: "1", IccidWithChecksum: "2", IccidWithoutChecksum: "3", diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index 05cf9a3e6..b62303ece 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -19,7 +19,7 @@ func GenerateCsvPayload(db *store.SimBatchDB, batch model.Batch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") - entries, err := db.GetAllSimEntriesForBatch(batch.BatchId) + entries, err := db.GetAllSimEntriesForBatch(batch.BatchID) if err != nil { panic(err) } From 56d6e9b3fbbd81ffb4aa6753be25b8945f6c7735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 08:02:26 +0100 Subject: [PATCH 285/309] Refactoring: Even more renaming of id to ID --- .../sim-batch-management/store/store.go | 16 ++++++++++++---- .../sim-batch-management/store/store_test.go | 8 ++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 5cef34efb..fcf47f743 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -29,7 +29,7 @@ type Store interface { CreateBatch(theBatch *model.Batch) error GetAllBatches(id string) ([]model.Batch, error) - GetBatchById(id int64) (*model.Batch, error) + GetBatchByID(id int64) (*model.Batch, error) GetBatchByName(id string) (*model.Batch, error) CreateSimEntry(simEntry *model.SimEntry) error @@ -40,7 +40,7 @@ type Store interface { GetSimProfileByIccid(msisdn string) (*model.SimEntry, error) CreateProfileVendor(*model.ProfileVendor) error - GetProfileVendorById(id int64) (*model.ProfileVendor, error) + GetProfileVendorByID(id int64) (*model.ProfileVendor, error) GetProfileVendorByName(name string) (*model.ProfileVendor, error) Begin() @@ -54,6 +54,8 @@ func (sdb *SimBatchDB) Begin() *sql.Tx { return tx } + +// Create a new in-memory instance of an SQLIte database func NewInMemoryDatabase() (*SimBatchDB, error) { db, err := sqlx.Connect("sqlite3", ":memory:") if err != nil { @@ -68,6 +70,10 @@ func NewInMemoryDatabase() (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } +// Create a new instance of an SQLIte database backed by a file whose path +// is found in the named environment variable. If the +// file doesn't exist, then it is created. If the environment variable is not +// defined or empty, an error is returned. func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (*SimBatchDB, error) { variableValue := strings.TrimSpace(os.Getenv(variablename)) if variableValue == "" { @@ -77,6 +83,8 @@ func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (* return db, err } +// Create a new instance of an SQLIte database backed by a file. If the +// file doesn't exist, then it is created. func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { /* TODO: Introduce 'debug' flag, and let that flag light up this code. @@ -99,7 +107,7 @@ func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { return result, sdb.Db.Select(&result, "SELECT * from BATCH") } -func (sdb SimBatchDB) GetBatchById(id int64) (*model.Batch, error) { +func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { result := []model.Batch{} if err := sdb.Db.Select(&result, "SELECT * FROM BATCH WHERE id = ?", id); err != nil { return nil, err @@ -221,7 +229,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { return nil } -func (sdb SimBatchDB) GetProfileVendorById(id int64) (*model.ProfileVendor, error) { +func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, error) { result := []model.ProfileVendor{} if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where id = ?", id); err != nil { return nil, err diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 3538d231c..bddb8a902 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -129,12 +129,12 @@ func TestGetBatchById(t *testing.T) { injectTestprofileVendor(t) theBatch := injectTestBatch() - batchById, _ := sdb.GetBatchById(theBatch.BatchID) + batchById, _ := sdb.GetBatchByID(theBatch.BatchID) if !reflect.DeepEqual(batchById, theBatch) { t.Logf("theBatch = %v\n", theBatch) t.Logf("batchById = %v\n", batchById) - t.Errorf("getBatchById failed") + t.Errorf("getBatchByID failed") } } @@ -196,7 +196,7 @@ func TestDeclareBatch(t *testing.T) { injectTestprofileVendor(t) theBatch := declareTestBatch(t) - retrievedValue, _ := sdb.GetBatchById(theBatch.BatchID) + retrievedValue, _ := sdb.GetBatchByID(theBatch.BatchID) if retrievedValue == nil { t.Fatalf("Null retrievedValue") } @@ -246,7 +246,7 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { t.Fatalf("name retrieved and stored profile vendor entries are different, %v v.s. %v", nameRetrievedVendor, v) } - idRetrievedVendor,err := sdb.GetProfileVendorById(v.Id) + idRetrievedVendor,err := sdb.GetProfileVendorByID(v.Id) if err != nil { t.Fatal(err) } From 58d403bfd54a2c96759ec113bd886f87daf07813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 08:38:30 +0100 Subject: [PATCH 286/309] upcasing url, adding comments, fixing whitespace bloopers --- .../sim-batch-management/README.md | 55 +++++++++---------- .../sim-batch-management/model/model.go | 18 +++--- .../sim-batch-management/sbm.go | 2 +- .../sim-batch-management/store/store.go | 33 ++++++----- .../sim-batch-management/store/store_test.go | 30 +++++----- 5 files changed, 68 insertions(+), 70 deletions(-) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index 5d1f1f5d1..456153735 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -21,16 +21,13 @@ go programmes here, both designed to be run from the command line. For both of these programmes, see the source code, in particular the comments near the top of the files for instructions on how to use them. - -To build everythning -== - +##To build everythning ### Prerequisites -* Go has to be installed on the system being run. -* Prime needs to be accessible via ssh tunnel or otherwise from the host - where the script is being run. + * Go has to be installed on the system being run. + * Prime needs to be accessible via ssh tunnel or otherwise from the host + where the script is being run. ### Building @@ -44,14 +41,11 @@ To build everythning ... will compile and test the program, then if you're running bash extend your shell with command line extensions for the sbm program. - -Some common usecases -== - +## Some common usecases ### How to upload batch information to prime -### Introduction +#### Introduction Prime has REST endpoint for uploading sim batches. This is an interface with little error checking (beyond the bare miniumums) @@ -80,34 +74,37 @@ then the script will terminate with an error message. (these are reasonable things to check btw, errors have been made that justifies adding these checks). - - -##A typical invocation looks like this: +###A typical invocation looks like this: TBD - - -TODO -== +##TODO + 1. Create a very clean PR for future code review. -1. Create a very clean PR for future code review. -1. Write up a nice markdown documentation describing common usecases. -1. Add crypto resources so that the program can talk to external parties. -1. Add code to activate profiles in HSS (if API is known) -1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) -1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that + 1. Write up a nice markdown documentation describing common usecases. + + 1. Add crypto resources so that the program can talk to external parties. + + 1. Add code to activate profiles in HSS (if API is known) + + 1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) + + 1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external components accessed from gocode. -1. Figure out how to handle workflows. Be explicit! -1. The interfaces to external parties will be + + 1. Figure out how to handle workflows. Be explicit! + + 1. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determined) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or directly from the script later. In either case it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. -1. Declare legal hss/dpv combinations, batches must use legal combos. -1. Declare prime instances (should make sense to have both prod and dev defined + + 1. Declare legal hss/dpv combinations, batches must use legal combos. + + 1. Declare prime instances (should make sense to have both prod and dev defined with different constraints on them). diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 71ac1aea9..dca4694b3 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -1,6 +1,5 @@ package model - // TODO: Delete all the ICCID entries that are not necessary, that would be at // about three of them. type SimEntry struct { @@ -16,7 +15,6 @@ type SimEntry struct { ActivationCode string `db:"activationCode" json:"activationCode"` } - type Batch struct { BatchID int64 `db:"id" json:"id"` // TODO: SHould this be called 'Id' Name string `db:"name" json:"name"` @@ -34,7 +32,7 @@ type Batch struct { Quantity int `db:"quantity" json:"quantity"` FirstIccid string `db:"firstIccid" json:"firstIccid"` FirstImsi string `db:"firstImsi" json:"firstImsi"` - Url string `db:"url" json:"url"` + URL string `db:"url" json:"url"` MsisdnIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` IccidIncrement int `db:"iccidIncrement" json:"msisdnIncrement"` ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` @@ -43,11 +41,11 @@ type Batch struct { } type ProfileVendor struct { - Id int64 `db:"id" json:"id"` - Name string `db:"name" json:"name"` - Es2PlusCert string `db:"es2PlusCertPath" json:"es2plusCertPath"` - Es2PlusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` - Es2PlusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` - Es2PlusPort int `db:"es2PlusPort" json:"es2plusPort"` - Es2PlusRequesterId string `db:"es2PlusRequesterId" json:"es2PlusRequesterId"` + Id int64 `db:"id" json:"id"` + Name string `db:"name" json:"name"` + Es2PlusCert string `db:"es2PlusCertPath" json:"es2plusCertPath"` + Es2PlusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` + Es2PlusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` + Es2PlusPort int `db:"es2PlusPort" json:"es2plusPort"` + Es2PlusRequesterId string `db:"es2PlusRequesterId" json:"es2PlusRequesterId"` } diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index 4d198d23e..244dce7e3 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -424,7 +424,7 @@ func parseCommandLine() error { return fmt.Errorf("no batch found with name '%s'", *describeBatchBatch) } else { var csvPayload = uploadtoprime.GenerateCsvPayload(db, *batch) - uploadtoprime.GeneratePostingCurlscript(batch.Url, csvPayload) + uploadtoprime.GeneratePostingCurlscript(batch.URL, csvPayload) } case "batch-generate-input-file": diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index fcf47f743..6c96adf63 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -15,7 +15,6 @@ import ( "strings" ) - /// Holding database abstraction for the sim batch management system. type SimBatchDB struct { Db *sqlx.DB @@ -54,7 +53,6 @@ func (sdb *SimBatchDB) Begin() *sql.Tx { return tx } - // Create a new in-memory instance of an SQLIte database func NewInMemoryDatabase() (*SimBatchDB, error) { db, err := sqlx.Connect("sqlite3", ":memory:") @@ -93,7 +91,7 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { } else { log.Printf("No databasefile found at '%s', will create one", path) } - */ + */ db, err := sqlx.Open("sqlite3", path) if err != nil { @@ -102,11 +100,14 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } +// Get a slice containing all the batches in the database func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { result := []model.Batch{} return result, sdb.Db.Select(&result, "SELECT * from BATCH") } +// Get a batch identified by its datbase ID number. If nothing is found +// a nil value is returned. func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { result := []model.Batch{} if err := sdb.Db.Select(&result, "SELECT * FROM BATCH WHERE id = ?", id); err != nil { @@ -119,6 +120,8 @@ func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { } } +// Get a batch identified by its name. If nothing is found +// a nil value is returned. func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { result := []model.Batch{} if err := sdb.Db.Select(&result, "select * from BATCH where name = ?", name); err != nil { @@ -130,6 +133,10 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { } } +// Create an instance of a batch in the database. Populate it with +// fields from the parameter "theBatch". No error checking +// (except uniqueness of name field) is performed on the the +// "theBatch" parameter. func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { // TODO: mutex? @@ -153,6 +160,8 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { return err } +// If they don't already exist, then generate the tables used by the +// store package. func (sdb *SimBatchDB) GenerateTables() error { sql := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, @@ -242,8 +251,6 @@ func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, erro } } - - func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { result := []model.ProfileVendor{} if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where name = ?", name); err != nil { @@ -294,7 +301,7 @@ func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, error) { result := []model.SimEntry{} - if err := sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchID = ?", batchID) ; err != nil { + if err := sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchID = ?", batchID); err != nil { return nil, err } @@ -307,7 +314,6 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, // TODO: Add unit test for this method. - func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error) { result := []model.SimEntry{} if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where iccid = ?", iccid); err != nil { @@ -334,7 +340,6 @@ func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) } } - func (sdb SimBatchDB) UpdateSimEntryMsisdn(simID int64, msisdn string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simID", map[string]interface{}{ @@ -344,7 +349,7 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simID int64, msisdn string) error { return err } -func (sdb SimBatchDB) UpdateSimEntryKi(simID int64, ki string) error { +func (sdb SimBatchDB) UpdateSimEntryKi(simID int64, ki string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET ki=:ki WHERE id = :simID", map[string]interface{}{ "simID": simID, @@ -353,7 +358,6 @@ func (sdb SimBatchDB) UpdateSimEntryKi(simID int64, ki string) error { return err } - func (sdb SimBatchDB) UpdateActivationCode(simID int64, activationCode string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE id = :simID", map[string]interface{}{ @@ -363,6 +367,7 @@ func (sdb SimBatchDB) UpdateActivationCode(simID int64, activationCode string) e return err } +// Drop all tables used by the store package. func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` _, err := sdb.Db.Exec(foo) @@ -397,7 +402,7 @@ func (sdb SimBatchDB) DeclareBatch( profileVendor string, initialHlrActivationStatusOfProfiles string) (*model.Batch, error) { - log.Println("Declaring batch ...") + log.Println("Declaring batch ...") vendor, err := sdb.GetProfileVendorByName(profileVendor) if err != nil { @@ -438,10 +443,10 @@ func (sdb SimBatchDB) DeclareBatch( return nil, fmt.Errorf("OutputBatch Quantity must be positive, but was '%d'", batchLength) } - uploadUrl := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", + uploadURL := fmt.Sprintf("http://%s:%s/ostelco/sim-inventory/%s/import-batch/profilevendor/%s?initialHssState=%s", uploadHostname, uploadPortnumber, hssVendor, profileVendor, initialHlrActivationStatusOfProfiles) - fieldsyntaxchecks.CheckURLSyntax("uploadUrl", uploadUrl) + fieldsyntaxchecks.CheckURLSyntax("uploadURL", uploadURL) fieldsyntaxchecks.CheckProfileType("profile-type", profileType) // Convert to integers, and get lengths @@ -490,7 +495,7 @@ func (sdb SimBatchDB) DeclareBatch( Name: name, BatchNo: batchNo, ProfileType: profileType, - Url: uploadUrl, + URL: uploadURL, Quantity: loltelutils.Abs(iccidlen), FirstIccid: firstIccid, IccidIncrement: loltelutils.Sign(iccidlen), diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index bddb8a902..758b8136b 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -11,7 +11,7 @@ import ( ) var ( - sdb *SimBatchDB + sdb *SimBatchDB sdbSetupError error ) @@ -42,7 +42,7 @@ func setup() { panic("Returned null database object") } - if sdbSetupError = sdb.GenerateTables(); sdbSetupError != nil { + if sdbSetupError = sdb.GenerateTables(); sdbSetupError != nil { panic(fmt.Sprintf("Couldn't generate tables '%s'", sdbSetupError)) } @@ -97,7 +97,7 @@ func injectTestBatch() *model.Batch { ProfileType: "banana", BatchNo: "100", Quantity: 100, - Url: "http://vg.no", + URL: "http://vg.no", FirstIccid: "1234567890123456789", FirstImsi: "123456789012345", ProfileVendor: "Durian", @@ -121,7 +121,7 @@ func TestGetBatchById(t *testing.T) { fmt.Print("TestGetBatchById starts") cleanTables() batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") - if batch != nil { + if batch != nil { t.Errorf("Duplicate detected, error in test setup") } @@ -183,7 +183,7 @@ func declareTestBatch(t *testing.T) *model.Batch { "LOL", // hssVendor string, "localhost", // uploadHostname string, "8088", // uploadPortnumber string, - "Durian", // profileVendor string, + "Durian", // profileVendor string, "ACTIVE") // initialHlrActivationStatusOfProfiles string if err != nil { @@ -213,15 +213,14 @@ func TestDeclareBatch(t *testing.T) { // TODO: Add check for content of retrieved entity } - -func injectTestprofileVendor( t *testing.T) *model.ProfileVendor { +func injectTestprofileVendor(t *testing.T) *model.ProfileVendor { v := &model.ProfileVendor{ - Name: "Durian", - Es2PlusCert: "cert", - Es2PlusKey: "key", - Es2PlusHost: "host", - Es2PlusPort: 4711, - Es2PlusRequesterId: "1.2.3", + Name: "Durian", + Es2PlusCert: "cert", + Es2PlusKey: "key", + Es2PlusHost: "host", + Es2PlusPort: 4711, + Es2PlusRequesterId: "1.2.3", } if err := sdb.CreateProfileVendor(v); err != nil { @@ -236,8 +235,7 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { v := injectTestprofileVendor(t) - - nameRetrievedVendor,err := sdb.GetProfileVendorByName(v.Name) + nameRetrievedVendor, err := sdb.GetProfileVendorByName(v.Name) if err != nil { t.Fatal(err) } @@ -246,7 +244,7 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { t.Fatalf("name retrieved and stored profile vendor entries are different, %v v.s. %v", nameRetrievedVendor, v) } - idRetrievedVendor,err := sdb.GetProfileVendorByID(v.Id) + idRetrievedVendor, err := sdb.GetProfileVendorByID(v.Id) if err != nil { t.Fatal(err) } From 7f0a540cc0f8107d98cb38903f15863ebf767b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 08:52:19 +0100 Subject: [PATCH 287/309] Fixing minor errors, supressing some other errors --- .../outfileparser/outfileparser.go | 3 +- .../outfileparser/outfileparser_test.go | 10 +++--- .../sim-batch-management/sbm.go | 8 ++--- .../sim-batch-management/store/store.go | 31 ++++++++++++++----- .../sim-batch-management/store/store_test.go | 21 +++++++++---- 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index bd1fc7968..cb522b3d7 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -14,6 +14,7 @@ import ( "strings" ) +//noinspection GoSnakeCaseUsage const ( INITIAL = "initial" HEADER_DESCRIPTION = "header_description" @@ -56,7 +57,7 @@ func parseVarOutLine(varOutLine string, result *map[string]int) error { return fmt.Errorf("syntax error in var_out line, more than two colon separated fields") } - varOutToken := strings.TrimSpace(string(varOutSplit[0])) + varOutToken := strings.TrimSpace(varOutSplit[0]) if strings.ToLower(varOutToken) != "var_out" { return fmt.Errorf("syntax error in var_out line. Does not start with 'var_out', was '%s'", varOutToken) } diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go index 407e1158b..c7d918225 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser_test.go @@ -14,14 +14,14 @@ func TestKeywordValueParser(t *testing.T) { } func TestReadingSimpleOutputFile(t *testing.T) { - sample_output_file_name := "sample_out_file_for_testing.out" - record, err := ParseOutputFile(sample_output_file_name) + sampleOutputFileName := "sample_out_file_for_testing.out" + record, err := ParseOutputFile(sampleOutputFileName) if err != nil { t.Error(t) } // First parameter to check - assert.Equal(t, sample_output_file_name, record.Filename) + assert.Equal(t, sampleOutputFileName, record.Filename) // Check that all the header variables are there assert.Equal(t, record.HeaderDescription["Customer"], "Footel") @@ -40,8 +40,8 @@ func TestReadingSimpleOutputFile(t *testing.T) { } func TestReadingComplexOutputFile(t *testing.T) { - sample_output_file_name := "sample-out-2.out" - record, err := ParseOutputFile(sample_output_file_name) + sampleOutputFileName := "sample-out-2.out" + record, err := ParseOutputFile(sampleOutputFileName) if err != nil { t.Error(t) } diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index 244dce7e3..f08b08a6f 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -492,11 +492,11 @@ func parseCommandLine() error { // Read all the lines into the record map. for { - line, error := reader.Read() - if error == io.EOF { + line, err := reader.Read() + if err == io.EOF { break - } else if error != nil { - log.Fatal(error) + } else if err != nil { + log.Fatal(err) } iccid := line[columnMap["Iccid"]] diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 6c96adf63..4933a75a6 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -102,6 +102,7 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { // Get a slice containing all the batches in the database func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { + //noinspection GoPreferNilSlice result := []model.Batch{} return result, sdb.Db.Select(&result, "SELECT * from BATCH") } @@ -109,6 +110,7 @@ func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { // Get a batch identified by its datbase ID number. If nothing is found // a nil value is returned. func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { + //noinspection GoPreferNilSlice result := []model.Batch{} if err := sdb.Db.Select(&result, "SELECT * FROM BATCH WHERE id = ?", id); err != nil { return nil, err @@ -123,6 +125,7 @@ func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { // Get a batch identified by its name. If nothing is found // a nil value is returned. func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { + //noinspection GoPreferNilSlice result := []model.Batch{} if err := sdb.Db.Select(&result, "select * from BATCH where name = ?", name); err != nil { return nil, err @@ -163,7 +166,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { // If they don't already exist, then generate the tables used by the // store package. func (sdb *SimBatchDB) GenerateTables() error { - sql := `CREATE TABLE IF NOT EXISTS BATCH ( + s := `CREATE TABLE IF NOT EXISTS BATCH ( id integer primary key autoincrement, name VARCHAR NOT NULL UNIQUE, profileVendor VARCHAR NOT NULL, @@ -180,12 +183,12 @@ func (sdb *SimBatchDB) GenerateTables() error { imsiIncrement INTEGER, iccidIncrement INTEGER, url VARCHAR)` - _, err := sdb.Db.Exec(sql) + _, err := sdb.Db.Exec(s) if err != nil { return err } - sql = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( + s = `CREATE TABLE IF NOT EXISTS SIM_PROFILE ( id INTEGER PRIMARY KEY AUTOINCREMENT, batchID INTEGER NOT NULL, activationCode VARCHAR NOT NULL, @@ -196,12 +199,12 @@ func (sdb *SimBatchDB) GenerateTables() error { iccid VARCHAR NOT NULL, ki VARCHAR NOT NULL, msisdn VARCHAR NOT NULL)` - _, err = sdb.Db.Exec(sql) + _, err = sdb.Db.Exec(s) if err != nil { return err } - sql = `CREATE TABLE IF NOT EXISTS PROFILE_VENDOR ( + s = `CREATE TABLE IF NOT EXISTS PROFILE_VENDOR ( id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR NOT NULL UNIQUE, es2PlusCertPath VARCHAR, @@ -209,7 +212,7 @@ func (sdb *SimBatchDB) GenerateTables() error { es2PlusHostPath VARCHAR, es2PlusPort VARCHAR, es2PlusRequesterId VARCHAR)` - _, err = sdb.Db.Exec(sql) + _, err = sdb.Db.Exec(s) return err } @@ -239,6 +242,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { } func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, error) { + //noinspection GoPreferNilSlice result := []model.ProfileVendor{} if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where id = ?", id); err != nil { return nil, err @@ -252,6 +256,7 @@ func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, erro } func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { + //noinspection GoPreferNilSlice result := []model.ProfileVendor{} if err := sdb.Db.Select(&result, "select * from PROFILE_VENDOR where name = ?", name); err != nil { return nil, err @@ -287,6 +292,7 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { } func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { + //noinspection GoPreferNilSlice result := []model.SimEntry{} if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", simID); err != nil { return nil, err @@ -300,6 +306,7 @@ func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { } func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, error) { + //noinspection GoPreferNilSlice result := []model.SimEntry{} if err := sdb.Db.Select(&result, "SELECT * from SIM_PROFILE WHERE batchID = ?", batchID); err != nil { return nil, err @@ -315,6 +322,7 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, // TODO: Add unit test for this method. func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error) { + //noinspection GoPreferNilSlice result := []model.SimEntry{} if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where iccid = ?", iccid); err != nil { return nil, err @@ -328,6 +336,7 @@ func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error } func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) { + //noinspection GoPreferNilSlice result := []model.SimEntry{} if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where imsi = ?", imsi); err != nil { return nil, err @@ -514,9 +523,15 @@ func (sdb SimBatchDB) DeclareBatch( defer func() { if weCool { - tx.Commit() + err := tx.Commit() + if err != nil { + panic(err) + } } else { - tx.Rollback() + err := tx.Rollback() + if err != nil { + panic(err) + } } }() diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 758b8136b..6fafa7ad7 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -29,7 +29,7 @@ func setup() { filename := "bazunka.db" // delete file, ignore any errors - os.Remove(filename) + _ = os.Remove(filename) sdb, sdbSetupError = OpenFileSqliteDatabase(filename) @@ -79,7 +79,7 @@ func shutdown() { if err := sdb.DropTables(); err != nil { panic(fmt.Sprintf("Couldn't drop tables '%s'", err)) } - sdb.Db.Close() + _ = sdb.Db.Close() } // ... just to know that everything is sane. @@ -106,7 +106,7 @@ func injectTestBatch() *model.Batch { batch, _ := sdb.GetBatchByName(theBatch.Name) if batch != nil { - panic(fmt.Errorf("Duplicate batch detected '%s'", theBatch.Name)) + panic(fmt.Errorf("duplicate batch detected '%s'", theBatch.Name)) } err := sdb.CreateBatch(&theBatch) @@ -164,6 +164,7 @@ func TestGetAllBatches(t *testing.T) { } } +//noinspection GoUnusedParameter func declareTestBatch(t *testing.T) *model.Batch { theBatch, err := sdb.DeclareBatch( @@ -213,6 +214,7 @@ func TestDeclareBatch(t *testing.T) { // TODO: Add check for content of retrieved entity } +//noinspection GoUnusedParameter func injectTestprofileVendor(t *testing.T) *model.ProfileVendor { v := &model.ProfileVendor{ Name: "Durian", @@ -272,7 +274,10 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { ActivationCode: "8", } - sdb.CreateSimEntry(&entry) + err := sdb.CreateSimEntry(&entry) + if err != nil { + t.Fatal(err) + } assert.Assert(t, entry.Id != 0) retrivedEntry, err := sdb.GetSimEntryById(entry.Id) @@ -303,11 +308,15 @@ func TestSimBatchDB_UpdateSimEntryKi(t *testing.T) { ActivationCode: "8", } - sdb.CreateSimEntry(&entry) + err := sdb.CreateSimEntry(&entry) + if err != nil { + t.Fatal(err) + } + assert.Assert(t, entry.Id != 0) newKi := "12" - err := sdb.UpdateSimEntryKi(entry.Id, newKi) + err = sdb.UpdateSimEntryKi(entry.Id, newKi) if err != nil { t.Fatal(err) From a817f5270cd71103a6d9332874e36b5707f41501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 09:22:22 +0100 Subject: [PATCH 288/309] Fixing typographic issues --- .../sim-batch-management/README.md | 18 +++++++++--------- .../sim-batch-management/store/store_test.go | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index 456153735..1ab6eac8f 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -82,21 +82,21 @@ that justifies adding these checks). 1. Create a very clean PR for future code review. - 1. Write up a nice markdown documentation describing common usecases. + 2. Write up a nice markdown documentation describing common usecases. - 1. Add crypto resources so that the program can talk to external parties. + 3. Add crypto resources so that the program can talk to external parties. - 1. Add code to activate profiles in HSS (if API is known) + 4. Add code to activate profiles in HSS (if API is known) - 1. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) + 5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) - 1. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that + 6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external components accessed from gocode. - 1. Figure out how to handle workflows. Be explicit! + 7. Figure out how to handle workflows. Be explicit! - 1. The interfaces to external parties will be + 8. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determined) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or @@ -104,7 +104,7 @@ that justifies adding these checks). it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. - 1. Declare legal hss/dpv combinations, batches must use legal combos. + 9. Declare legal hss/dpv combinations, batches must use legal combos. - 1. Declare prime instances (should make sense to have both prod and dev defined + 10. Declare prime instances (should make sense to have both prod and dev defined with different constraints on them). diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 6fafa7ad7..d4f9f93ba 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -116,9 +116,9 @@ func injectTestBatch() *model.Batch { return &theBatch } -func TestGetBatchById(t *testing.T) { +func TestGetBatchByID(t *testing.T) { - fmt.Print("TestGetBatchById starts") + fmt.Print("TestGetBatchByID starts") cleanTables() batch, _ := sdb.GetBatchByName("SOME UNIQUE NAME") if batch != nil { @@ -129,11 +129,11 @@ func TestGetBatchById(t *testing.T) { injectTestprofileVendor(t) theBatch := injectTestBatch() - batchById, _ := sdb.GetBatchByID(theBatch.BatchID) - if !reflect.DeepEqual(batchById, theBatch) { + batchByID, _ := sdb.GetBatchByID(theBatch.BatchID) + if !reflect.DeepEqual(batchByID, theBatch) { t.Logf("theBatch = %v\n", theBatch) - t.Logf("batchById = %v\n", batchById) + t.Logf("batchByID = %v\n", batchByID) t.Errorf("getBatchByID failed") } } From 796eb0e33242417c8322288aa23facf3d7a9ff8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 11:57:53 +0100 Subject: [PATCH 289/309] Refactor based on suggestions from codacy --- .../sim-batch-management/README.md | 25 +++++---- .../sim-batch-management/es2plus/es2plus.go | 55 ++++++++++--------- .../sim-batch-management/store/store.go | 3 + 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index 1ab6eac8f..885669cb5 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -25,8 +25,9 @@ comments near the top of the files for instructions on how to use them. ### Prerequisites - * Go has to be installed on the system being run. - * Prime needs to be accessible via ssh tunnel or otherwise from the host +* Go has to be installed on the system being run. + +* Prime needs to be accessible via ssh tunnel or otherwise from the host where the script is being run. ### Building @@ -80,23 +81,23 @@ that justifies adding these checks). ##TODO - 1. Create a very clean PR for future code review. +1. Create a very clean PR for future code review. - 2. Write up a nice markdown documentation describing common usecases. +2. Write up a nice markdown documentation describing common usecases. - 3. Add crypto resources so that the program can talk to external parties. +3. Add crypto resources so that the program can talk to external parties. - 4. Add code to activate profiles in HSS (if API is known) +4. Add code to activate profiles in HSS (if API is known) - 5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) +5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) - 6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that +6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external components accessed from gocode. - 7. Figure out how to handle workflows. Be explicit! +7. Figure out how to handle workflows. Be explicit! - 8. The interfaces to external parties will be +8. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determined) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or @@ -104,7 +105,7 @@ that justifies adding these checks). it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. - 9. Declare legal hss/dpv combinations, batches must use legal combos. +9. Declare legal hss/dpv combinations, batches must use legal combos. - 10. Declare prime instances (should make sense to have both prod and dev defined +10. Declare prime instances (should make sense to have both prod and dev defined with different constraints on them). diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index a9c7f0f93..565a41016 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -32,7 +32,7 @@ type es2PlusHeader struct { FunctionCallIdentifier string `json:"functionCallIdentifier"` } -type ES2PlusGetProfileStatusRequest struct { +type es2PlusGetProfileStatusRequest struct { Header es2PlusHeader `json:"header"` IccidList []es2PlusIccid `json:"iccidList"` } @@ -41,13 +41,13 @@ type es2PlusIccid struct { Iccid string `json:"iccid"` } -type FunctionExecutionStatus struct { - FunctionExecutionStatusType string `json:"status"` +type functionExecutionStatus struct { + functionExecutionStatusType string `json:"status"` StatusCodeData es2PlusStatusCodeData `json:"statusCodeData"` } type es2PlusResponseHeader struct { - FunctionExecutionStatus FunctionExecutionStatus `json:"functionExecutionStatus"` + functionExecutionStatus functionExecutionStatus `json:"functionExecutionStatus"` } // @@ -61,7 +61,7 @@ type es2PlusStatusCodeData struct { Message string `json:"message"` } -type ES2ProfileStatusResponse struct { +type es2ProfileStatusResponse struct { Header es2PlusResponseHeader `json:"header"` ProfileStatusList []ProfileStatus `json:"profileStatusList"` CompletionTimestamp string `json:"completionTimestamp"` @@ -149,7 +149,7 @@ type ConfirmOrderResponse struct { // Generating new ES2Plus clients // -type ClientState struct { +type clientState struct { httpClient *http.Client hostport string requesterID string @@ -157,8 +157,8 @@ type ClientState struct { logHeaders bool } -func NewClient(certFilePath string, keyFilePath string, hostport string, requesterID string) *ClientState { - return &ClientState{ +func NewClient(certFilePath string, keyFilePath string, hostport string, requesterID string) *clientState { + return &clientState{ httpClient: newHTTPClient(certFilePath, keyFilePath), hostport: hostport, requesterID: requesterID, @@ -240,7 +240,10 @@ func newEs2plusHeader(client Client) (*es2PlusHeader, error) { return &es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionrequesterIDentifier: client.RequesterID()}, nil } -func (client *ClientState) execute( +// execute is an internal function that will package a payload +// by json serializing it, then execute an ES2+ command +// and unmarshal the result into the result object. +func (client *clientState) execute( es2plusCommand string, payload interface{}, result interface{}) error { @@ -285,15 +288,15 @@ func (client *ClientState) execute( /// -// Get the status of a profile with a specific ICCID. -func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { - result := new(ES2ProfileStatusResponse) +// GetStatus will return the status of a profile with a specific ICCID. +func (client *clientState) GetStatus(iccid string) (*ProfileStatus, error) { + result := new(es2ProfileStatusResponse) es2plusCommand := "getProfileStatus" header, err := newEs2plusHeader(client) if err != nil { return nil, err } - payload := &ES2PlusGetProfileStatusRequest{ + payload := &es2PlusGetProfileStatusRequest{ Header: *header, IccidList: []es2PlusIccid{es2PlusIccid{Iccid: iccid}}, } @@ -310,9 +313,9 @@ func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { } } -// Recover the state of the profile with a particular ICCID, +// RecoverProfile will recover the state of the profile with a particular ICCID, // by setting it to the target state. -func (client *ClientState) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) { +func (client *clientState) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) { result := new(RecoverProfileResponse) es2plusCommand := "recoverProfile" header, err := newEs2plusHeader(client) @@ -327,9 +330,9 @@ func (client *ClientState) RecoverProfile(iccid string, targetState string) (*Re return result, client.execute(es2plusCommand, payload, result) } -// Cancel an order by setting the state of the profile with a particular ICCID, +// CancelOrder will cancel an order by setting the state of the profile with a particular ICCID, // the target state. -func (client *ClientState) CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) { +func (client *clientState) CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) { result := new(CancelOrderResponse) es2plusCommand := "cancelOrder" header, err := newEs2plusHeader(client) @@ -344,9 +347,9 @@ func (client *ClientState) CancelOrder(iccid string, targetState string) (*Cance return result, client.execute(es2plusCommand, payload, result) } -// Prepare the profile to be downloaded (first of two steps, the +// DownloadOrder will prepare the profile to be downloaded (first of two steps, the // ConfirmDownload is also necessary). -func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, error) { +func (client *clientState) DownloadOrder(iccid string) (*DownloadOrderResponse, error) { result := new(DownloadOrderResponse) es2plusCommand := "downloadOrder" header, err := newEs2plusHeader(client) @@ -363,7 +366,7 @@ func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, return nil, err } - executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + executionStatus := result.Header.functionExecutionStatus.functionExecutionStatusType if executionStatus == "Executed-Success" { return result, nil } else { @@ -371,9 +374,9 @@ func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, } } -// Second of the two steps that are necessary to prepare a profile for +// ConfirmOrder will execute the second of the two steps that are necessary to prepare a profile for // to be downloaded. -func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) { +func (client *clientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) { result := new(ConfirmOrderResponse) es2plusCommand := "confirmOrder" header, err := newEs2plusHeader(client) @@ -394,7 +397,7 @@ func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, er return nil, err } - executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType + executionStatus := result.Header.functionExecutionStatus.functionExecutionStatusType if executionStatus != "Executed-Success" { return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) } @@ -403,11 +406,11 @@ func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, er } -// Take a profile to the state "READY" where it can be downloaded. +// ActivateIccid will take a profile to the state "READY" where it can be downloaded. // This function will if poll the current status of the profile, and if // necessary advance the state by executing the DownloadOrder and // ConfirmOrder functions. -func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { +func (client *clientState) ActivateIccid(iccid string) (*ProfileStatus, error) { result, err := client.GetStatus(iccid) if err != nil { @@ -436,6 +439,6 @@ func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { } // TODO: This shouldn't have to be public, but how can it be avoided? -func (clientState *ClientState) RequesterID() string { +func (clientState *clientState) RequesterID() string { return clientState.requesterID } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 4933a75a6..180ff71d7 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -217,6 +217,7 @@ func (sdb *SimBatchDB) GenerateTables() error { return err } +// func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { // TODO: This insert string can be made through reflection, and at some point should be. @@ -241,6 +242,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { return nil } +// Find a profile vendor in the database by looking it up by name. func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, error) { //noinspection GoPreferNilSlice result := []model.ProfileVendor{} @@ -255,6 +257,7 @@ func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, erro } } +// Find a profile vendor in the database by looking it up by name. func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { //noinspection GoPreferNilSlice result := []model.ProfileVendor{} From 3ee91a4d49c3b3983af6bf0d893a337873e17c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 12:21:21 +0100 Subject: [PATCH 290/309] Making some structs public again since that's necessary for using json --- .../sim-batch-management/README.md | 26 +++---- .../sim-batch-management/es2plus/es2plus.go | 69 +++++++++---------- 2 files changed, 47 insertions(+), 48 deletions(-) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index 885669cb5..f67700f2d 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -21,13 +21,13 @@ go programmes here, both designed to be run from the command line. For both of these programmes, see the source code, in particular the comments near the top of the files for instructions on how to use them. -##To build everythning +##To build everything ### Prerequisites -* Go has to be installed on the system being run. + * Go has to be installed on the system being run. -* Prime needs to be accessible via ssh tunnel or otherwise from the host + * Prime needs to be accessible via ssh tunnel or otherwise from the host where the script is being run. ### Building @@ -81,23 +81,23 @@ that justifies adding these checks). ##TODO -1. Create a very clean PR for future code review. + 1. Create a very clean PR for future code review. -2. Write up a nice markdown documentation describing common usecases. + 2. Write up a nice markdown documentation describing common usecases. -3. Add crypto resources so that the program can talk to external parties. + 3. Add crypto resources so that the program can talk to external parties. -4. Add code to activate profiles in HSS (if API is known) + 4. Add code to activate profiles in HSS (if API is known) -5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) + 5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) -6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that + 6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external components accessed from gocode. -7. Figure out how to handle workflows. Be explicit! + 7. Figure out how to handle workflows. Be explicit! -8. The interfaces to external parties will be + 8. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determined) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or @@ -105,7 +105,7 @@ that justifies adding these checks). it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. -9. Declare legal hss/dpv combinations, batches must use legal combos. + 9. Declare legal hss/dpv combinations, batches must use legal combos. -10. Declare prime instances (should make sense to have both prod and dev defined + 10. Declare prime instances (should make sense to have both prod and dev defined with different constraints on them). diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 565a41016..0abd8540f 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -27,13 +27,13 @@ type Client interface { /// /// Generic headers for invocations and responses /// -type es2PlusHeader struct { +type Es2PlusHeader struct { FunctionrequesterIDentifier string `json:"functionrequesterIDentifier"` FunctionCallIdentifier string `json:"functionCallIdentifier"` } -type es2PlusGetProfileStatusRequest struct { - Header es2PlusHeader `json:"header"` +type Es2PlusGetProfileStatusRequest struct { + Header Es2PlusHeader `json:"header"` IccidList []es2PlusIccid `json:"iccidList"` } @@ -83,8 +83,8 @@ type ProfileStatus struct { // Profile reset invocation // -type es2PlusRecoverProfileRequest struct { - Header es2PlusHeader `json:"header"` +type Es2PlusCancelOrderRequest struct { + Header Es2PlusHeader `json:"header"` Iccid string `json:"iccid"` ProfileStatus string `json:"profileStatus"` } @@ -98,7 +98,7 @@ type RecoverProfileResponse struct { // type es2PlusCancelOrderRequest struct { - Header es2PlusHeader `json:"header"` + Header Es2PlusHeader `json:"header"` Iccid string `json:"iccid"` FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` } @@ -111,8 +111,8 @@ type CancelOrderResponse struct { // Download order invocation // -type downloadOrderRequest struct { - Header es2PlusHeader `json:"header"` +type DownloadOrderRequest struct { + Header Es2PlusHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` Profiletype string `json:"profiletype,omitempty"` @@ -127,8 +127,8 @@ type DownloadOrderResponse struct { // ConfirmOrder invocation // -type confirmOrderRequest struct { - Header es2PlusHeader `json:"header"` +type ConfirmOrderRequest struct { + Header Es2PlusHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` MatchingID string `json:"matchingId,omitempty"` @@ -149,7 +149,7 @@ type ConfirmOrderResponse struct { // Generating new ES2Plus clients // -type clientState struct { +type ClientState struct { httpClient *http.Client hostport string requesterID string @@ -157,8 +157,8 @@ type clientState struct { logHeaders bool } -func NewClient(certFilePath string, keyFilePath string, hostport string, requesterID string) *clientState { - return &clientState{ +func NewClient(certFilePath string, keyFilePath string, hostport string, requesterID string) *ClientState { + return &ClientState{ httpClient: newHTTPClient(certFilePath, keyFilePath), hostport: hostport, requesterID: requesterID, @@ -230,20 +230,20 @@ func newUUID() (string, error) { return uuid.URN(), nil } -func newEs2plusHeader(client Client) (*es2PlusHeader, error) { +func newEs2PlusHeader(client Client) (*Es2PlusHeader, error) { functionCallIdentifier, err := newUUID() if err != nil { return nil, err } - return &es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionrequesterIDentifier: client.RequesterID()}, nil + return &Es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionrequesterIDentifier: client.RequesterID()}, nil } // execute is an internal function that will package a payload // by json serializing it, then execute an ES2+ command // and unmarshal the result into the result object. -func (client *clientState) execute( +func (client *ClientState) execute( es2plusCommand string, payload interface{}, result interface{}) error { @@ -289,14 +289,14 @@ func (client *clientState) execute( // GetStatus will return the status of a profile with a specific ICCID. -func (client *clientState) GetStatus(iccid string) (*ProfileStatus, error) { +func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(es2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - header, err := newEs2plusHeader(client) + header, err := newEs2PlusHeader(client) if err != nil { return nil, err } - payload := &es2PlusGetProfileStatusRequest{ + payload := &Es2PlusGetProfileStatusRequest{ Header: *header, IccidList: []es2PlusIccid{es2PlusIccid{Iccid: iccid}}, } @@ -315,14 +315,14 @@ func (client *clientState) GetStatus(iccid string) (*ProfileStatus, error) { // RecoverProfile will recover the state of the profile with a particular ICCID, // by setting it to the target state. -func (client *clientState) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) { +func (client *ClientState) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) { result := new(RecoverProfileResponse) es2plusCommand := "recoverProfile" - header, err := newEs2plusHeader(client) + header, err := newEs2PlusHeader(client) if err != nil { return nil, err } - payload := &es2PlusRecoverProfileRequest{ + payload := &Es2PlusCancelOrderRequest{ Header: *header, Iccid: iccid, ProfileStatus: targetState, @@ -332,10 +332,10 @@ func (client *clientState) RecoverProfile(iccid string, targetState string) (*Re // CancelOrder will cancel an order by setting the state of the profile with a particular ICCID, // the target state. -func (client *clientState) CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) { +func (client *ClientState) CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) { result := new(CancelOrderResponse) es2plusCommand := "cancelOrder" - header, err := newEs2plusHeader(client) + header, err := newEs2PlusHeader(client) if err != nil { return nil, err } @@ -349,14 +349,14 @@ func (client *clientState) CancelOrder(iccid string, targetState string) (*Cance // DownloadOrder will prepare the profile to be downloaded (first of two steps, the // ConfirmDownload is also necessary). -func (client *clientState) DownloadOrder(iccid string) (*DownloadOrderResponse, error) { +func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, error) { result := new(DownloadOrderResponse) es2plusCommand := "downloadOrder" - header, err := newEs2plusHeader(client) + header, err := newEs2PlusHeader(client) if err != nil { return nil, err } - payload := &downloadOrderRequest{ + payload := &DownloadOrderRequest{ Header: *header, Iccid: iccid, Eid: "", @@ -369,21 +369,20 @@ func (client *clientState) DownloadOrder(iccid string) (*DownloadOrderResponse, executionStatus := result.Header.functionExecutionStatus.functionExecutionStatusType if executionStatus == "Executed-Success" { return result, nil - } else { - return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) } + return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) } // ConfirmOrder will execute the second of the two steps that are necessary to prepare a profile for // to be downloaded. -func (client *clientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) { +func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) { result := new(ConfirmOrderResponse) es2plusCommand := "confirmOrder" - header, err := newEs2plusHeader(client) + header, err := newEs2PlusHeader(client) if err != nil { return nil, err } - payload := &confirmOrderRequest{ + payload := &ConfirmOrderRequest{ Header: *header, Iccid: iccid, Eid: "", @@ -410,7 +409,7 @@ func (client *clientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, er // This function will if poll the current status of the profile, and if // necessary advance the state by executing the DownloadOrder and // ConfirmOrder functions. -func (client *clientState) ActivateIccid(iccid string) (*ProfileStatus, error) { +func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { result, err := client.GetStatus(iccid) if err != nil { @@ -439,6 +438,6 @@ func (client *clientState) ActivateIccid(iccid string) (*ProfileStatus, error) { } // TODO: This shouldn't have to be public, but how can it be avoided? -func (clientState *clientState) RequesterID() string { - return clientState.requesterID +func (ClientState *ClientState) RequesterID() string { + return ClientState.requesterID } From 5abb11d192ba1577ffc1d43f8af4343b30fd5d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 13:43:27 +0100 Subject: [PATCH 291/309] Work a bit on visibility again. --- .../sim-batch-management/es2plus/es2plus.go | 18 +++++-- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 42 +++++++++++++++-- .../sim-batch-management/model/model.go | 12 +++++ .../outfileparser/outfileparser.go | 41 +++++++++------- .../sim-batch-management/store/store.go | 47 +++++++++++++------ 5 files changed, 121 insertions(+), 39 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 0abd8540f..8b62a7d71 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -41,13 +41,13 @@ type es2PlusIccid struct { Iccid string `json:"iccid"` } -type functionExecutionStatus struct { - functionExecutionStatusType string `json:"status"` +type FunctionExecutionStatus struct { + FunctionExecutionStatusType string `json:"status"` StatusCodeData es2PlusStatusCodeData `json:"statusCodeData"` } type es2PlusResponseHeader struct { - functionExecutionStatus functionExecutionStatus `json:"functionExecutionStatus"` + FunctionExecutionStatus FunctionExecutionStatus `json:"FunctionExecutionStatus"` } // @@ -127,6 +127,9 @@ type DownloadOrderResponse struct { // ConfirmOrder invocation // + +// ConfirmOrderRequest contains parameters for the es2+ confirmOrder +// command. type ConfirmOrderRequest struct { Header Es2PlusHeader `json:"header"` Iccid string `json:"iccid"` @@ -137,6 +140,9 @@ type ConfirmOrderRequest struct { ReleaseFlag bool `json:"releaseFlag"` } + +// ConfirmOrderResponse contains the response value for the es2+ confirmOrder +// command. type ConfirmOrderResponse struct { Header es2PlusResponseHeader `json:"header"` Iccid string `json:"iccid"` @@ -149,6 +155,8 @@ type ConfirmOrderResponse struct { // Generating new ES2Plus clients // + +// ClientState struct representing the state of a ES2+ client. type ClientState struct { httpClient *http.Client hostport string @@ -366,7 +374,7 @@ func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, return nil, err } - executionStatus := result.Header.functionExecutionStatus.functionExecutionStatusType + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType if executionStatus == "Executed-Success" { return result, nil } @@ -396,7 +404,7 @@ func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, er return nil, err } - executionStatus := result.Header.functionExecutionStatus.functionExecutionStatusType + executionStatus := result.Header.FunctionExecutionStatus.FunctionExecutionStatusType if executionStatus != "Executed-Success" { return result, fmt.Errorf("ExecutionStatus was: ''%s'", executionStatus) } diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go index ba3b25e8c..e408427d1 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -43,20 +43,27 @@ func calculateChecksum(luhnString string, double bool) int { return checksum } +// LuhnChecksum generates a Luhn checksum control digit for the parameter number. func LuhnChecksum(number int) int { return generateControlDigit(strconv.Itoa(number)) } -func AddLuhnChecksum(original string) string { - checksum := generateControlDigit(original) - return fmt.Sprintf("%s%1d", original, checksum) +// AddLuhnChecksum interprets the parameter as a number, then +// generates and appends an one-digit Luhn checksum. +func AddLuhnChecksum(parameterNumber string) string { + checksum := generateControlDigit(parameterNumber) + return fmt.Sprintf("%s%1d", parameterNumber, checksum) } +// IsICCID if the string is an 18 or 19 digit positive integer. +// Does not check luhn checksum. func IsICCID(s string) bool { match, _ := regexp.MatchString("^\\d{18}\\d?\\d?$", s) return match } +// IsICCID if the string is an 18 or 19 digit positive integer. +// Does check luhn checksum. func CheckICCIDSyntax(name string, potentialIccid string) { if !IsICCID(potentialIccid) { log.Fatalf("Not a valid '%s' ICCID: '%s'. Must be 18 or 19 (or 20) digits (_including_ luhn checksum).", name, potentialIccid) @@ -70,30 +77,49 @@ func CheckICCIDSyntax(name string, potentialIccid string) { } } +// IsIMSI is true iff the parameter string is a 15 digit number, +// indicating that it could be a valid IMSI. func IsIMSI(s string) bool { match, _ := regexp.MatchString("^\\d{15}$", s) return match } +// CheckIMSISyntax is a convenience function that checks +// if the potentialIMSI string is a syntactically correct +// IMSI. IF it isn't a log message is written and the program +// terminates. func CheckIMSISyntax(name string, potentialIMSI string) { if !IsIMSI(potentialIMSI) { + // TODO: Modify. Return error value instead. log.Fatalf("Not a valid %s IMSI: '%s'. Must be 15 digits.", name, potentialIMSI) } } +// IsMSISDN bis true if the parameter string is a positive number. func IsMSISDN(s string) bool { match, _ := regexp.MatchString("^\\d+$", s) return match } +// CheckMSISDNSyntax is a convenience function that checks if +// the potential msisdn is an actual msisdn or not. If it isn't +// then an error message is written and the program +// terminates. func CheckMSISDNSyntax(name string, potentialMSISDN string) { if !IsMSISDN(potentialMSISDN) { + // TODO: Fix this to return error value instead. log.Fatalf("Not a valid %s MSISDN: '%s'. Must be non-empty sequence of digits.", name, potentialMSISDN) } } + +// CheckURLSyntax is a convenience function that checks if +// the potential url is an actual url or not. If it isn't +// then an error message is written and the program +// terminates. func CheckURLSyntax(name string, theUrl string) { if _, err := url.ParseRequestURI(theUrl); err != nil { + // TODO: Fix this to return error value instead. log.Fatalf("Not a valid %s URL: '%s'.", name, theUrl) } } @@ -103,12 +129,22 @@ func IsProfileName(s string) bool { return match } + +// CheckProfileType is a convenience function that checks if +// the potential profile name is a syntacticaly correct +// profile name or not.. If it isn't +// then an error message is written and the program +// terminates. func CheckProfileType(name string, potentialProfileName string) { if !IsProfileName(potentialProfileName) { + // TODO: Fix this to return error value instead. log.Fatalf("Not a valid %s MSISDN: '%s'. Must be uppercase characters, numbers and underscores. ", name, potentialProfileName) } } + +// IccidWithoutLuhnChecksum takes an ICCID with a trailing Luhn +// checksum, and returns the value without the trailing checksum. func IccidWithoutLuhnChecksum(s string) string { return loltelutils.TrimSuffix(s, 1) } diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index dca4694b3..810931f18 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -2,6 +2,11 @@ package model // TODO: Delete all the ICCID entries that are not necessary, that would be at // about three of them. + + +// SimEntry represents individual sim profiles. Instances can be +// subject to JSON serialisation/deserialisation, and can be stored +// in persistent storage. type SimEntry struct { Id int64 `db:"id" json:"id"` BatchID int64 `db:"batchID" json:"batchID"` @@ -15,6 +20,9 @@ type SimEntry struct { ActivationCode string `db:"activationCode" json:"activationCode"` } +// Batch represents batches of sim profiles. Instances can be +// subject to JSON serialisation/deserialisation, and can be stored +// in persistent storage. type Batch struct { BatchID int64 `db:"id" json:"id"` // TODO: SHould this be called 'Id' Name string `db:"name" json:"name"` @@ -40,6 +48,10 @@ type Batch struct { ProfileVendor string `db:"profileVendor" json:"profileVendor"` } + +// Batch represents sim vendors. Instances can be +// subject to JSON serialisation/deserialisation, and can be stored +// in persistent storage. type ProfileVendor struct { Id int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index cb522b3d7..dee0b2e3c 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -69,24 +69,29 @@ func parseVarOutLine(varOutLine string, result *map[string]int) error { return nil } -// Implement a state machine that parses an output file. -func ParseOutputFile(filename string) (*OutputFileRecord, error) { - if _, err := os.Stat(filename); err != nil { +// Parse an output file, returning an OutputFileRecord, contained +// a parsed version of the inputfile. +func ParseOutputFile(filePath string) (*OutputFileRecord, error) { + + if _, err := os.Stat(filePath); err != nil { if os.IsNotExist(err) { - return nil, fmt.Errorf("couldn't find file '%s'", filename) - } else { - return nil, fmt.Errorf("couldn't stat file '%s'", filename) + return nil, fmt.Errorf("couldn't find file '%s'", filePath) } + return nil, fmt.Errorf("couldn't stat file '%s'", filePath) } - file, err := os.Open(filename) // For read access. + file, err := os.Open(filePath) // For read access. if err != nil { log.Fatal(err) } defer file.Close() + + // Implement a state machine that parses an output file. + + state := parserState{ currentState: INITIAL, inputVariables: make(map[string]string), @@ -183,7 +188,7 @@ func ParseOutputFile(filename string) (*OutputFileRecord, error) { declaredNoOfEntities, err := strconv.Atoi(state.headerDescription["Quantity"]) if err != nil { - return nil, fmt.Errorf("could not find 'Quantity' field while parsing file '%s'", filename) + return nil, fmt.Errorf("could not find 'Quantity' field while parsing file '%s'", filePath) } if countedNoOfEntries != declaredNoOfEntities { @@ -193,7 +198,7 @@ func ParseOutputFile(filename string) (*OutputFileRecord, error) { } result := OutputFileRecord{ - Filename: filename, + Filename: filePath, InputVariables: state.inputVariables, HeaderDescription: state.headerDescription, Entries: state.entries, @@ -268,19 +273,23 @@ func fileExists(filename string) bool { } // TODO: Move this into some other package. "hssoutput" or something. -func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) error { - if fileExists(filename) { - return fmt.Errorf("output file already exists. '%s'", filename) + +// WriteHssCsvFile will write all sim profile instances associated to a +// batch object to a file located at filepath. +func WriteHssCsvFile(filepath string, sdb *store.SimBatchDB, batch *model.Batch) error { + + if fileExists(filepath) { + return fmt.Errorf("output file already exists. '%s'", filepath) } - f, err := os.Create(filename) + f, err := os.Create(filepath) if err != nil { - return fmt.Errorf("couldn't create hss csv file '%s', %v", filename, err) + return fmt.Errorf("couldn't create hss csv file '%s', %v", filepath, err) } if _, err = f.WriteString("ICCID, IMSI, KI\n"); err != nil { - return fmt.Errorf("couldn't header to hss csv file '%s', %v", filename, err) + return fmt.Errorf("couldn't header to hss csv file '%s', %v", filepath, err) } entries, err := sdb.GetAllSimEntriesForBatch(batch.BatchID) @@ -292,7 +301,7 @@ func WriteHssCsvFile(filename string, sdb *store.SimBatchDB, batch *model.Batch) for i, entry := range entries { s := fmt.Sprintf("%s, %s, %s\n", entry.IccidWithChecksum, entry.Imsi, entry.Ki) if _, err = f.WriteString(s); err != nil { - return fmt.Errorf("couldn't write to hss csv file '%s', %v", filename, err) + return fmt.Errorf("couldn't write to hss csv file '%s', %v", filepath, err) } max = i + 1 } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 180ff71d7..5958adc5d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -5,7 +5,7 @@ import ( "flag" "fmt" "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" + _ "github.com/mattn/go-sqlite3" // We need this "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" @@ -45,6 +45,10 @@ type Store interface { Begin() } +// Begin starts a transaction, returns the +// transaction objects. If for some reason the transaction +// can't be started, the functioni will terminate the program +// via panic. func (sdb *SimBatchDB) Begin() *sql.Tx { tx, err := sdb.Db.Begin() if err != nil { @@ -53,7 +57,7 @@ func (sdb *SimBatchDB) Begin() *sql.Tx { return tx } -// Create a new in-memory instance of an SQLIte database +// NewInMemoryDatabase creates a new in-memory instance of an SQLIte database func NewInMemoryDatabase() (*SimBatchDB, error) { db, err := sqlx.Connect("sqlite3", ":memory:") if err != nil { @@ -68,6 +72,7 @@ func NewInMemoryDatabase() (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } +// OpenFileSqliteDatabaseFromPathInEnvironmentVariable // Create a new instance of an SQLIte database backed by a file whose path // is found in the named environment variable. If the // file doesn't exist, then it is created. If the environment variable is not @@ -100,14 +105,14 @@ func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } -// Get a slice containing all the batches in the database +// GetAllBatches gets a slice containing all the batches in the database func (sdb SimBatchDB) GetAllBatches() ([]model.Batch, error) { //noinspection GoPreferNilSlice result := []model.Batch{} return result, sdb.Db.Select(&result, "SELECT * from BATCH") } -// Get a batch identified by its datbase ID number. If nothing is found +// GetBatchByID gets a batch identified by its datbase ID number. If nothing is found // a nil value is returned. func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { //noinspection GoPreferNilSlice @@ -122,7 +127,7 @@ func (sdb SimBatchDB) GetBatchByID(id int64) (*model.Batch, error) { } } -// Get a batch identified by its name. If nothing is found +// GetBatchByName gets a batch identified by its name. If nothing is found // a nil value is returned. func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { //noinspection GoPreferNilSlice @@ -136,7 +141,7 @@ func (sdb SimBatchDB) GetBatchByName(name string) (*model.Batch, error) { } } -// Create an instance of a batch in the database. Populate it with +// CreateBatch Creates an instance of a batch in the database. Populate it with // fields from the parameter "theBatch". No error checking // (except uniqueness of name field) is performed on the the // "theBatch" parameter. @@ -217,7 +222,7 @@ func (sdb *SimBatchDB) GenerateTables() error { return err } -// +//CreateProfileVendor inject a new profile vendor instance into the database. func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { // TODO: This insert string can be made through reflection, and at some point should be. @@ -242,7 +247,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { return nil } -// Find a profile vendor in the database by looking it up by name. +// GetProfileVendorByID find a profile vendor in the database by looking it up by name. func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, error) { //noinspection GoPreferNilSlice result := []model.ProfileVendor{} @@ -257,7 +262,7 @@ func (sdb SimBatchDB) GetProfileVendorByID(id int64) (*model.ProfileVendor, erro } } -// Find a profile vendor in the database by looking it up by name. +// GetProfileVendorByName find a profile vendor in the database by looking it up by name. func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, error) { //noinspection GoPreferNilSlice result := []model.ProfileVendor{} @@ -272,6 +277,8 @@ func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, } } + +// CreateSimEntry persists a SimEntry instance in the database. func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { res := sdb.Db.MustExec("INSERT INTO SIM_PROFILE (batchID, activationCode, rawIccid, iccidWithChecksum, iccidWithoutChecksum, iccid, imsi, msisdn, ki) values (?,?,?,?,?,?,?,?,?)", @@ -294,6 +301,8 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { return err } +// GetSimEntryById retrieves a sim entryh instance stored in the database. If no +// matching instance can be found, nil is returned. func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { //noinspection GoPreferNilSlice result := []model.SimEntry{} @@ -308,6 +317,8 @@ func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { } } +// GetAllSimEntriesForBatch retrieves a sim entryh instance stored in the database. If no +// matching instance can be found, nil is returned. func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, error) { //noinspection GoPreferNilSlice result := []model.SimEntry{} @@ -316,14 +327,15 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, } if len(result) == 0 { - return nil, nil + return nil, nil // TODO: Evaluate if this is wrong. Returning an empty slice isn't necessarily wrong. } else { return result, nil } } -// TODO: Add unit test for this method. +// GetSimProfileByIccid gets a sim profile from the database, return nil of one +// can't be found. func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error) { //noinspection GoPreferNilSlice result := []model.SimEntry{} @@ -338,6 +350,8 @@ func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error } } +// GetSimProfileByIccid gets a sim profile from the database, return nil of one +// can't be found. func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) { //noinspection GoPreferNilSlice result := []model.SimEntry{} @@ -352,6 +366,7 @@ func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) } } +// UpdateSimEntryMsisdn Sets the MSISDN field of a persisted instance of a sim entry. func (sdb SimBatchDB) UpdateSimEntryMsisdn(simID int64, msisdn string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET msisdn=:msisdn WHERE id = :simID", map[string]interface{}{ @@ -361,6 +376,7 @@ func (sdb SimBatchDB) UpdateSimEntryMsisdn(simID int64, msisdn string) error { return err } +// UpdateSimEntryKi Sets the Ki field of a persisted instance of a sim entry. func (sdb SimBatchDB) UpdateSimEntryKi(simID int64, ki string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET ki=:ki WHERE id = :simID", map[string]interface{}{ @@ -370,6 +386,7 @@ func (sdb SimBatchDB) UpdateSimEntryKi(simID int64, ki string) error { return err } +// UpdateActivationCode Sets the activation code field of a persisted instance of a sim entry. func (sdb SimBatchDB) UpdateActivationCode(simID int64, activationCode string) error { _, err := sdb.Db.NamedExec("UPDATE SIM_PROFILE SET activationCode=:activationCode WHERE id = :simID", map[string]interface{}{ @@ -379,7 +396,7 @@ func (sdb SimBatchDB) UpdateActivationCode(simID int64, activationCode string) e return err } -// Drop all tables used by the store package. +// DropTables Drop all tables used by the store package. func (sdb *SimBatchDB) DropTables() error { foo := `DROP TABLE BATCH` _, err := sdb.Db.Exec(foo) @@ -391,9 +408,9 @@ func (sdb *SimBatchDB) DropTables() error { return err } -/** - * DeclareBatch a new batch, assuming that it doesn't exist. Do all kind of checking of fields etc. - */ + +// DeclareBatch generates a batch instance by first checking all of its +// parameters, and then storing it, and finally returning it from the function. func (sdb SimBatchDB) DeclareBatch( name string, addLuhn bool, From 6af6318d3f2ba1397c38a913b0de8bf81d0dfe5a Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 21 Nov 2019 13:43:47 +0100 Subject: [PATCH 292/309] Switch to using the 'flatMapLeft' function to change the Either left type --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 5260336dc..8682cc84c 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -494,24 +494,21 @@ object Neo4jStoreSingleton : GraphStore { in neo4j but will still be present in payment backend. In that case the profile must be removed from the payment backend manually. */ paymentProcessor.removePaymentProfile(customerId) - .fold( - { - if (it is org.ostelco.prime.paymentprocessor.core.NotFoundError) { - /* Ignore. Customer has not bought products yet. */ - Unit.right() - } else { - logger.error(NOTIFY_OPS_MARKER, - "Removing corresponding payment profile when removing customer $customerId " + - "failed with error ${it.message} : ${it.description}") - PartiallyNotDeletedError(type = customerEntity.name, - id = "Failed to remove corresponding payment profile when removing customer $customerId", - error = it).left() - } - }, - { - Unit.right() - } - ).bind() + .map { + Unit + }.flatMapLeft { + if (it is org.ostelco.prime.paymentprocessor.core.NotFoundError) { + /* Ignore. Customer has not bought products yet. */ + Unit.right() + } else { + logger.error(NOTIFY_OPS_MARKER, + "Removing corresponding payment profile when removing customer $customerId " + + "failed with error ${it.message} : ${it.description}") + PartiallyNotDeletedError(type = customerEntity.name, + id = "Failed to remove corresponding payment profile when removing customer $customerId", + error = it).left() + } + }.bind() }.fix() }.unsafeRunSync() .ifFailedThenRollback(transaction) @@ -2185,13 +2182,14 @@ object Neo4jStoreSingleton : GraphStore { } } - private inline fun EitherOf.flatMapLeft(f: (A) -> Either): Either = + private inline fun EitherOf.flatMapLeft(f: (LEFT) -> Either): Either = fix().let { when (it) { is Right -> it is Left -> f(it.a) } } + // // Balance (Customer - Subscription - Bundle) // From d36f4444a5a8b04a22424f1deefbdd3d0e04a97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 13:53:41 +0100 Subject: [PATCH 293/309] More refactoring courtesy of codacy --- .../sim-batch-management/store/store.go | 21 +++++++++---------- .../sim-batch-management/store/store_test.go | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 5958adc5d..16bb025da 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -15,12 +15,12 @@ import ( "strings" ) -/// Holding database abstraction for the sim batch management system. +// SimBatchDB Holding database abstraction for the sim batch management system. type SimBatchDB struct { Db *sqlx.DB } -// An interface used to abstract the CRUD operations on the +// Store is an interface used to abstract the CRUD operations on the // various entities in the sim batch management database. type Store interface { GenerateTables() error @@ -72,8 +72,8 @@ func NewInMemoryDatabase() (*SimBatchDB, error) { return &SimBatchDB{Db: db}, nil } -// OpenFileSqliteDatabaseFromPathInEnvironmentVariable -// Create a new instance of an SQLIte database backed by a file whose path +// OpenFileSqliteDatabaseFromPathInEnvironmentVariable creates +// a new instance of an SQLIte database backed by a file whose path // is found in the named environment variable. If the // file doesn't exist, then it is created. If the environment variable is not // defined or empty, an error is returned. @@ -86,7 +86,7 @@ func OpenFileSqliteDatabaseFromPathInEnvironmentVariable(variablename string) (* return db, err } -// Create a new instance of an SQLIte database backed by a file. If the +// OpenFileSqliteDatabase creates a new instance of an SQLIte database backed by a file. If the // file doesn't exist, then it is created. func OpenFileSqliteDatabase(path string) (*SimBatchDB, error) { @@ -168,7 +168,7 @@ func (sdb SimBatchDB) CreateBatch(theBatch *model.Batch) error { return err } -// If they don't already exist, then generate the tables used by the +// GenerateTables will, if they don't already exist, then generate the tables used by the // store package. func (sdb *SimBatchDB) GenerateTables() error { s := `CREATE TABLE IF NOT EXISTS BATCH ( @@ -301,9 +301,9 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { return err } -// GetSimEntryById retrieves a sim entryh instance stored in the database. If no +// GetSimEntryByID retrieves a sim entryh instance stored in the database. If no // matching instance can be found, nil is returned. -func (sdb SimBatchDB) GetSimEntryById(simID int64) (*model.SimEntry, error) { +func (sdb SimBatchDB) GetSimEntryByID(simID int64) (*model.SimEntry, error) { //noinspection GoPreferNilSlice result := []model.SimEntry{} if err := sdb.Db.Select(&result, "select * from SIM_PROFILE where id = ?", simID); err != nil { @@ -350,7 +350,7 @@ func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error } } -// GetSimProfileByIccid gets a sim profile from the database, return nil of one +// GetSimProfileByImsi gets a sim profile from the database, return nil of one // can't be found. func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) { //noinspection GoPreferNilSlice @@ -361,9 +361,8 @@ func (sdb SimBatchDB) GetSimProfileByImsi(imsi string) (*model.SimEntry, error) if len(result) == 0 { return nil, nil - } else { - return &result[0], nil } + return &result[0], nil } // UpdateSimEntryMsisdn Sets the MSISDN field of a persisted instance of a sim entry. diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index d4f9f93ba..16bc210ce 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -280,7 +280,7 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { } assert.Assert(t, entry.Id != 0) - retrivedEntry, err := sdb.GetSimEntryById(entry.Id) + retrivedEntry, err := sdb.GetSimEntryByID(entry.Id) if err != nil { t.Fatal(err) } @@ -322,7 +322,7 @@ func TestSimBatchDB_UpdateSimEntryKi(t *testing.T) { t.Fatal(err) } - retrivedEntry, err := sdb.GetSimEntryById(entry.Id) + retrivedEntry, err := sdb.GetSimEntryByID(entry.Id) if err != nil { t.Fatal(err) } From e327e820dc0cacf63c561895786705c7ff908713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 14:30:51 +0100 Subject: [PATCH 294/309] More codacity --- .../sim-batch-management/es2plus/es2plus.go | 4 +- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 3 +- .../sim-batch-management/model/model.go | 6 +-- .../outfileparser/outfileparser.go | 33 ++++++------ .../sim-batch-management/sbm.go | 50 +++++++++---------- .../sim-batch-management/store/store.go | 6 +-- .../sim-batch-management/store/store_test.go | 4 +- .../uploadtoprime/uploadtoprime.go | 4 ++ 8 files changed, 58 insertions(+), 52 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 8b62a7d71..4c159c7e2 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -446,6 +446,6 @@ func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { } // TODO: This shouldn't have to be public, but how can it be avoided? -func (ClientState *ClientState) RequesterID() string { - return ClientState.requesterID +func (state *ClientState) RequesterID() string { + return state.requesterID } diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go index e408427d1..d57bc86de 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -124,8 +124,9 @@ func CheckURLSyntax(name string, theUrl string) { } } +// IsProfileName checks if the parameter string is a syntactically valid profile name func IsProfileName(s string) bool { - match, _ := regexp.MatchString("^[A-Z][A-Z0-9_]*$", s) + match, _ := regexp.MatchString("^[A-Z][A-Z0-9_]+$", s) return match } diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index 810931f18..cbb78b9ce 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -42,7 +42,7 @@ type Batch struct { FirstImsi string `db:"firstImsi" json:"firstImsi"` URL string `db:"url" json:"url"` MsisdnIncrement int `db:"msisdnIncrement" json:"msisdnIncrement"` - IccidIncrement int `db:"iccidIncrement" json:"msisdnIncrement"` + IccidIncrement int `db:"iccidIncrement" json:"iccidIncrement"` ImsiIncrement int `db:"imsiIncrement" json:"imsiIncrement"` FirstMsisdn string `db:"firstMsisdn" json:"firstMsisdn"` ProfileVendor string `db:"profileVendor" json:"profileVendor"` @@ -53,11 +53,11 @@ type Batch struct { // subject to JSON serialisation/deserialisation, and can be stored // in persistent storage. type ProfileVendor struct { - Id int64 `db:"id" json:"id"` + ID int64 `db:"id" json:"id"` Name string `db:"name" json:"name"` Es2PlusCert string `db:"es2PlusCertPath" json:"es2plusCertPath"` Es2PlusKey string `db:"es2PlusKeyPath" json:"es2PlusKeyPath"` Es2PlusHost string `db:"es2PlusHostPath" json:"es2plusHostPath"` Es2PlusPort int `db:"es2PlusPort" json:"es2plusPort"` - Es2PlusRequesterId string `db:"es2PlusRequesterId" json:"es2PlusRequesterId"` + Es2PlusRequesterID string `db:"es2PlusRequesterId" json:"es2PlusRequesterId"` } diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index dee0b2e3c..5e970f155 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -16,13 +16,14 @@ import ( //noinspection GoSnakeCaseUsage const ( - INITIAL = "initial" - HEADER_DESCRIPTION = "header_description" - INPUT_VARIABLES = "input_variables" - OUTPUT_VARIABLES = "output_variables" - UNKNOWN_HEADER = "unknown" + initial = "initial" + headerDescription = "header_description" + inputVariables = "input_variables" + outputVariables = "output_variables" + unknownHeader = "unknown" ) +// OutputFileRecord is a struct used to represent a parsed outputfile. type OutputFileRecord struct { Filename string InputVariables map[string]string @@ -93,7 +94,7 @@ func ParseOutputFile(filePath string) (*OutputFileRecord, error) { state := parserState{ - currentState: INITIAL, + currentState: initial, inputVariables: make(map[string]string), headerDescription: make(map[string]string), csvFieldMap: make(map[string]int), @@ -115,7 +116,7 @@ func ParseOutputFile(filePath string) (*OutputFileRecord, error) { transitionMode(&state, nextMode) continue } else if line == "OUTPUT VARIABLES" { - transitionMode(&state, OUTPUT_VARIABLES) + transitionMode(&state, outputVariables) continue } @@ -123,15 +124,15 @@ func ParseOutputFile(filePath string) (*OutputFileRecord, error) { // looking for real content? switch state.currentState { - case HEADER_DESCRIPTION: + case headerDescription: parseLineIntoKeyValueMap(line, state.headerDescription) - case INPUT_VARIABLES: + case inputVariables: if line == "var_In:" || line == "Var_In_List:" || strings.TrimSpace(line) == "" { continue } parseLineIntoKeyValueMap(line, state.inputVariables) - case OUTPUT_VARIABLES: + case outputVariables: line = strings.TrimSpace(line) lowercaseLine := strings.ToLower(line) @@ -172,7 +173,7 @@ func ParseOutputFile(filePath string) (*OutputFileRecord, error) { Ki: ki} state.entries = append(state.entries, entry) - case UNKNOWN_HEADER: + case unknownHeader: continue default: @@ -240,15 +241,15 @@ func modeFromSectionHeader(s string) string { sectionName := strings.Trim(s, "* ") switch sectionName { case "HEADER DESCRIPTION": - return HEADER_DESCRIPTION + return headerDescription case "INPUT VARIABLES": - return INPUT_VARIABLES + return inputVariables case "INPUT VARIABLES DESCRIPTION": - return INPUT_VARIABLES + return inputVariables case "OUTPUT VARIABLES": - return OUTPUT_VARIABLES + return outputVariables default: - return UNKNOWN_HEADER + return unknownHeader } } diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index f08b08a6f..455775f8c 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -39,7 +39,7 @@ var ( dpvKeyFilePath = dpv.Flag("key", "Certificate key file.").Required().String() dpvHost = dpv.Flag("host", "Host of ES2+ endpoint.").Required().String() dpvPort = dpv.Flag("port", "Port of ES2+ endpoint").Required().Int() - dpvRequesterId = dpv.Flag("requester-id", "ES2+ requester ID.").Required().String() + dpvRequesterID = dpv.Flag("requester-id", "ES2+ requester ID.").Required().String() /// /// ICCID - centric commands @@ -115,8 +115,8 @@ var ( // spUploadOutputFilePrefix = spUpload.Flag("output-file-prefix", // "prefix to path to .csv file used as input file, filename will be autogenerated").Required().String() - generateActivationCodeSql = kingpin.Command("batch-generate-activation-code-updating-sql", "Generate SQL code to update access codes") - generateActivationCodeSqlBatch = generateActivationCodeSql.Arg("batch-name", "The batch to generate sql coce for").String() + generateActivationCodeSQL = kingpin.Command("batch-generate-activation-code-updating-sql", "Generate SQL code to update access codes") + generateActivationCodeSQLBatch = generateActivationCodeSQL.Arg("batch-name", "The batch to generate sql coce for").String() bd = kingpin.Command("batch-declare", "Declare a batch to be persisted, and used by other commands") dbName = bd.Flag("name", "Unique name of this batch").Required().String() @@ -212,7 +212,7 @@ func parseCommandLine() error { Es2PlusKey: absDpvKeyFilePath, Es2PlusHost: *dpvHost, Es2PlusPort: *dpvPort, - Es2PlusRequesterId: *dpvRequesterId, + Es2PlusRequesterID: *dpvRequesterID, } if err := db.CreateProfileVendor(v); err != nil { @@ -231,7 +231,7 @@ func parseCommandLine() error { return fmt.Errorf("unknown batch '%s'", batchName) } - client, err := ClientForVendor(db, batch.ProfileVendor) + client, err := clientForVendor(db, batch.ProfileVendor) if err != nil { return err } @@ -397,9 +397,9 @@ func parseCommandLine() error { } case "batch-generate-activation-code-updating-sql": - batch, err := db.GetBatchByName(*generateActivationCodeSqlBatch) + batch, err := db.GetBatchByName(*generateActivationCodeSQLBatch) if err != nil { - return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSqlBatch, err) + return fmt.Errorf("couldn't find batch named '%s' (%s) ", *generateActivationCodeSQLBatch, err) } simEntries, err := db.GetAllSimEntriesForBatch(batch.BatchID) @@ -436,7 +436,7 @@ func parseCommandLine() error { if batch == nil { return fmt.Errorf("no batch found with name '%s'", *generateInputFileBatchname) } else { - var result = GenerateInputFile(batch) + var result = generateInputFileString(batch) fmt.Println(result) } @@ -549,7 +549,7 @@ func parseCommandLine() error { tx.Rollback() return err } - noOfRecordsUpdated += 1 + noOfRecordsUpdated++ } } tx.Commit() @@ -585,7 +585,7 @@ func parseCommandLine() error { return nil case "iccid-get-status": - client, err := ClientForVendor(db, *getStatusProfileVendor) + client, err := clientForVendor(db, *getStatusProfileVendor) if err != nil { return err } @@ -597,7 +597,7 @@ func parseCommandLine() error { log.Printf("Iccid='%s', state='%s', acToken='%s'\n", *getStatusProfileIccid, (*result).State, (*result).ACToken) case "iccid-recover-profile": - client, err := ClientForVendor(db, *recoverProfileVendor) + client, err := clientForVendor(db, *recoverProfileVendor) if err != nil { return err } @@ -613,7 +613,7 @@ func parseCommandLine() error { log.Println("result -> ", result) case "iccid-download-order": - client, err := ClientForVendor(db, *downloadOrderVendor) + client, err := clientForVendor(db, *downloadOrderVendor) if err != nil { return err } @@ -624,7 +624,7 @@ func parseCommandLine() error { log.Println("result -> ", result) case "iccid-confirm-order": - client, err := ClientForVendor(db, *confirmOrderVendor) + client, err := clientForVendor(db, *confirmOrderVendor) if err != nil { return err } @@ -635,7 +635,7 @@ func parseCommandLine() error { fmt.Println("result -> ", result) case "iccid-activate": - client, err := ClientForVendor(db, *activateIccidVendor) + client, err := clientForVendor(db, *activateIccidVendor) if err != nil { return err } @@ -648,7 +648,7 @@ func parseCommandLine() error { fmt.Printf("%s, %s\n", *activateIccidIccid, result.ACToken) case "iccid-cancel": - client, err := ClientForVendor(db, *cancelIccidVendor) + client, err := clientForVendor(db, *cancelIccidVendor) if err != nil { return err } @@ -662,7 +662,7 @@ func parseCommandLine() error { } case "iccids-activate-from-file": - client, err := ClientForVendor(db, *activateIccidFileVendor) + client, err := clientForVendor(db, *activateIccidFileVendor) if err != nil { return err } @@ -772,7 +772,7 @@ func parseCommandLine() error { case "batch-activate-all-profiles": - client, batch, err := ClientForBatch(db, *setBatchActivationCodesBatch) + client, batch, err := clientForBatch(db, *setBatchActivationCodesBatch) if err != nil { return err } @@ -835,7 +835,7 @@ func parseCommandLine() error { tx.Commit() case "iccids-bulk-activate": - client, err := ClientForVendor(db, *bulkActivateIccidsVendor) + client, err := clientForVendor(db, *bulkActivateIccidsVendor) if err != nil { return err } @@ -881,16 +881,16 @@ func parseCommandLine() error { func checkEs2TargetState(target string) error { if target != "AVAILABLE" { return fmt.Errorf("target ES2+ state unexpected, legal value(s) is(are): 'AVAILABLE'") - } else { - return nil } + + return nil } /// /// Input batch management /// -func GenerateInputFile(batch *model.Batch) string { +func generateInputFileString(batch *model.Batch) string { result := "*HEADER DESCRIPTION\n" + "***************************************\n" + fmt.Sprintf("Customer : %s\n", batch.Customer) + @@ -911,7 +911,7 @@ func GenerateInputFile(batch *model.Batch) string { return result } -func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Client, error) { +func clientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Client, error) { vendor, err := db.GetProfileVendorByName(vendorName) if err != nil { return nil, err @@ -922,10 +922,10 @@ func ClientForVendor(db *store.SimBatchDB, vendorName string) (es2plus.Client, e } hostport := fmt.Sprintf("%s:%d", vendor.Es2PlusHost, vendor.Es2PlusPort) - return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterId), nil + return es2plus.NewClient(vendor.Es2PlusCert, vendor.Es2PlusKey, hostport, vendor.Es2PlusRequesterID), nil } -func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Client, *model.Batch, error) { +func clientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Client, *model.Batch, error) { batch, err := db.GetBatchByName(batchName) if err != nil { @@ -935,7 +935,7 @@ func ClientForBatch(db *store.SimBatchDB, batchName string) (es2plus.Client, *mo return nil, nil, fmt.Errorf("unknown batch '%s'", batchName) } - client, err := ClientForVendor(db, batch.ProfileVendor) + client, err := clientForVendor(db, batch.ProfileVendor) if err != nil { return nil, nil, err } diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 16bb025da..26c513c1a 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -243,7 +243,7 @@ func (sdb SimBatchDB) CreateProfileVendor(theEntry *model.ProfileVendor) error { if err != nil { return fmt.Errorf("getting last inserted id failed '%s'", err) } - theEntry.Id = id + theEntry.ID = id return nil } @@ -345,9 +345,9 @@ func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error if len(result) == 0 { return nil, nil - } else { - return &result[0], nil } + return &result[0], nil + } // GetSimProfileByImsi gets a sim profile from the database, return nil of one diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 16bc210ce..9a72ec3a0 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -222,7 +222,7 @@ func injectTestprofileVendor(t *testing.T) *model.ProfileVendor { Es2PlusKey: "key", Es2PlusHost: "host", Es2PlusPort: 4711, - Es2PlusRequesterId: "1.2.3", + Es2PlusRequesterID: "1.2.3", } if err := sdb.CreateProfileVendor(v); err != nil { @@ -246,7 +246,7 @@ func TestDeclareAndRetrieveProfileVendorEntry(t *testing.T) { t.Fatalf("name retrieved and stored profile vendor entries are different, %v v.s. %v", nameRetrievedVendor, v) } - idRetrievedVendor, err := sdb.GetProfileVendorByID(v.Id) + idRetrievedVendor, err := sdb.GetProfileVendorByID(v.ID) if err != nil { t.Fatal(err) } diff --git a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go index b62303ece..b3970ebc7 100755 --- a/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go +++ b/sim-administration/sim-batch-management/uploadtoprime/uploadtoprime.go @@ -7,6 +7,9 @@ import ( "strings" ) + +// GeneratePostingCurlscript print on standard output a bash script +// that can be used to upload the payload to an url. func GeneratePostingCurlscript(url string, payload string) { fmt.Printf("#!/bin/bash\n") @@ -15,6 +18,7 @@ func GeneratePostingCurlscript(url string, payload string) { fmt.Print("EOF\n") } +// GenerateCsvPayload generate the csv payload to be sent to prime func GenerateCsvPayload(db *store.SimBatchDB, batch model.Batch) string { var sb strings.Builder sb.WriteString("ICCID, IMSI, MSISDN, PIN1, PIN2, PUK1, PUK2, PROFILE\n") From 8c25c16acc008d2291a5a229d7200d67e1d83377 Mon Sep 17 00:00:00 2001 From: "Kjell M. Myksvoll" Date: Thu, 21 Nov 2019 14:40:32 +0100 Subject: [PATCH 295/309] Added fixes to 'delete customer in Stripe' from code review --- .../kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 5 ++--- .../main/kotlin/org/ostelco/prime/apierror/ApiError.kt | 1 - .../main/kotlin/org/ostelco/prime/storage/StoreError.kt | 8 -------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 8682cc84c..311d04e37 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -99,7 +99,6 @@ import org.ostelco.prime.storage.NotCreatedError import org.ostelco.prime.storage.NotDeletedError import org.ostelco.prime.storage.NotFoundError import org.ostelco.prime.storage.NotUpdatedError -import org.ostelco.prime.storage.PartiallyNotDeletedError import org.ostelco.prime.storage.ScanInformationStore import org.ostelco.prime.storage.StoreError import org.ostelco.prime.storage.SystemError @@ -504,8 +503,8 @@ object Neo4jStoreSingleton : GraphStore { logger.error(NOTIFY_OPS_MARKER, "Removing corresponding payment profile when removing customer $customerId " + "failed with error ${it.message} : ${it.description}") - PartiallyNotDeletedError(type = customerEntity.name, - id = "Failed to remove corresponding payment profile when removing customer $customerId", + NotDeletedError(type = "Payment profile for customer", + id = customerId, error = it).left() } }.bind() diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt index ef4091475..eebe0ac44 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/apierror/ApiError.kt @@ -52,7 +52,6 @@ object ApiErrorMapper { is org.ostelco.prime.storage.NotCreatedError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.NotUpdatedError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.NotDeletedError -> InternalServerError(description, errorCode) - is org.ostelco.prime.storage.PartiallyNotDeletedError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.ValidationError -> ForbiddenError(description, errorCode, storeError) is org.ostelco.prime.storage.FileDownloadError -> InternalServerError(description, errorCode) is org.ostelco.prime.storage.FileDeleteError -> InternalServerError(description, errorCode) diff --git a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt index 21051e2f8..3c8a6c926 100644 --- a/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt +++ b/prime-modules/src/main/kotlin/org/ostelco/prime/storage/StoreError.kt @@ -51,14 +51,6 @@ class NotDeletedError(type: String, message = "$type - $id not deleted.", error = error) -class PartiallyNotDeletedError(type: String, - id: String, - error: InternalError? = null) : - StoreError(type = type, - id = id, - message = "$type - $id not deleted.", - error = error) - class ValidationError(type: String, id: String, message: String, From f54fefeb6aa35c801db4d9bc18eda0d449f93134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 14:54:29 +0100 Subject: [PATCH 296/309] Even more fixes --- .../sim-batch-management/es2plus/es2plus.go | 91 +++++++++++-------- .../fieldsyntaxchecks/fieldsyntaxchecks.go | 8 +- .../sim-batch-management/sbm.go | 5 +- .../sim-batch-management/store/store.go | 5 +- 4 files changed, 60 insertions(+), 49 deletions(-) diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index 4c159c7e2..a80f83290 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -27,26 +27,33 @@ type Client interface { /// /// Generic headers for invocations and responses /// -type Es2PlusHeader struct { + + +// Header is a generic header for all ES2+ invocations. +type Header struct { FunctionrequesterIDentifier string `json:"functionrequesterIDentifier"` FunctionCallIdentifier string `json:"functionCallIdentifier"` } -type Es2PlusGetProfileStatusRequest struct { - Header Es2PlusHeader `json:"header"` - IccidList []es2PlusIccid `json:"iccidList"` + + +// GetProfileStatusRequest holds a request object for the profileStatus es2+ command. +type GetProfileStatusRequest struct { + Header Header `json:"header"` + IccidList []ICCID `json:"iccidList"` } -type es2PlusIccid struct { +// ICCID holder of ICCID values in the Es2+ protocol. +type ICCID struct { Iccid string `json:"iccid"` } type FunctionExecutionStatus struct { FunctionExecutionStatusType string `json:"status"` - StatusCodeData es2PlusStatusCodeData `json:"statusCodeData"` + StatusCodeData StatusCodeData `json:"statusCodeData"` } -type es2PlusResponseHeader struct { +type ResponseHeader struct { FunctionExecutionStatus FunctionExecutionStatus `json:"FunctionExecutionStatus"` } @@ -54,7 +61,10 @@ type es2PlusResponseHeader struct { // Status code invocation. // -type es2PlusStatusCodeData struct { + +// StatusCodeData payload from the function execution status field of the +// ES2+ protocol header. +type StatusCodeData struct { SubjectCode string `json:"subjectCode"` ReasonCode string `json:"reasonCode"` SubjectIdentifier string `json:"subjectIdentifier"` @@ -62,7 +72,7 @@ type es2PlusStatusCodeData struct { } type es2ProfileStatusResponse struct { - Header es2PlusResponseHeader `json:"header"` + Header ResponseHeader `json:"header"` ProfileStatusList []ProfileStatus `json:"profileStatusList"` CompletionTimestamp string `json:"completionTimestamp"` } @@ -83,43 +93,44 @@ type ProfileStatus struct { // Profile reset invocation // -type Es2PlusCancelOrderRequest struct { - Header Es2PlusHeader `json:"header"` +// RecvoerProfileRequest is the payload of the recoverProfile request. +type RecoverProfileRequest struct { + Header Header `json:"header"` Iccid string `json:"iccid"` ProfileStatus string `json:"profileStatus"` } +// RecoverProfileResponse is the payload of the recoverProfile response. type RecoverProfileResponse struct { - Header es2PlusResponseHeader `json:"header"` + Header ResponseHeader `json:"header"` } -// -// Cancel order invocation -// -type es2PlusCancelOrderRequest struct { - Header Es2PlusHeader `json:"header"` +// CancelOrderRequest the payload of the cancelOrder request response. +type CancelOrderRequest struct { + Header Header `json:"header"` Iccid string `json:"iccid"` FinalProfileStatusIndicator string `json:"finalProfileStatusIndicator"` } +// CancelOrderResponse the payload of the cancelOrder request response. type CancelOrderResponse struct { - Header es2PlusResponseHeader `json:"header"` + Header ResponseHeader `json:"header"` } -// -// Download order invocation -// +// DownloadOrderRequest The payload of a downloadOrder request. type DownloadOrderRequest struct { - Header Es2PlusHeader `json:"header"` + Header Header `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` Profiletype string `json:"profiletype,omitempty"` } + +// DownloadOrderResponse the response of a DownloadOrder command type DownloadOrderResponse struct { - Header es2PlusResponseHeader `json:"header"` + Header ResponseHeader `json:"header"` Iccid string `json:"iccid"` } @@ -131,7 +142,7 @@ type DownloadOrderResponse struct { // ConfirmOrderRequest contains parameters for the es2+ confirmOrder // command. type ConfirmOrderRequest struct { - Header Es2PlusHeader `json:"header"` + Header Header `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` MatchingID string `json:"matchingId,omitempty"` @@ -144,7 +155,7 @@ type ConfirmOrderRequest struct { // ConfirmOrderResponse contains the response value for the es2+ confirmOrder // command. type ConfirmOrderResponse struct { - Header es2PlusResponseHeader `json:"header"` + Header ResponseHeader `json:"header"` Iccid string `json:"iccid"` Eid string `json:"eid,omitempty"` MatchingID string `json:"matchingId,omitempty"` @@ -165,6 +176,8 @@ type ClientState struct { logHeaders bool } + +// NewClient create a new es2+ client instance func NewClient(certFilePath string, keyFilePath string, hostport string, requesterID string) *ClientState { return &ClientState{ httpClient: newHTTPClient(certFilePath, keyFilePath), @@ -238,14 +251,14 @@ func newUUID() (string, error) { return uuid.URN(), nil } -func newEs2PlusHeader(client Client) (*Es2PlusHeader, error) { +func newHeader(client Client) (*Header, error) { functionCallIdentifier, err := newUUID() if err != nil { return nil, err } - return &Es2PlusHeader{FunctionCallIdentifier: functionCallIdentifier, FunctionrequesterIDentifier: client.RequesterID()}, nil + return &Header{FunctionCallIdentifier: functionCallIdentifier, FunctionrequesterIDentifier: client.RequesterID()}, nil } // execute is an internal function that will package a payload @@ -300,13 +313,13 @@ func (client *ClientState) execute( func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { result := new(es2ProfileStatusResponse) es2plusCommand := "getProfileStatus" - header, err := newEs2PlusHeader(client) + header, err := newHeader(client) if err != nil { return nil, err } - payload := &Es2PlusGetProfileStatusRequest{ + payload := &GetProfileStatusRequest{ Header: *header, - IccidList: []es2PlusIccid{es2PlusIccid{Iccid: iccid}}, + IccidList: []ICCID{ICCID{Iccid: iccid}}, } if err = client.execute(es2plusCommand, payload, result); err != nil { return nil, err @@ -326,11 +339,11 @@ func (client *ClientState) GetStatus(iccid string) (*ProfileStatus, error) { func (client *ClientState) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) { result := new(RecoverProfileResponse) es2plusCommand := "recoverProfile" - header, err := newEs2PlusHeader(client) + header, err := newHeader(client) if err != nil { return nil, err } - payload := &Es2PlusCancelOrderRequest{ + payload := &RecoverProfileRequest{ Header: *header, Iccid: iccid, ProfileStatus: targetState, @@ -343,11 +356,11 @@ func (client *ClientState) RecoverProfile(iccid string, targetState string) (*Re func (client *ClientState) CancelOrder(iccid string, targetState string) (*CancelOrderResponse, error) { result := new(CancelOrderResponse) es2plusCommand := "cancelOrder" - header, err := newEs2PlusHeader(client) + header, err := newHeader(client) if err != nil { return nil, err } - payload := &es2PlusCancelOrderRequest{ + payload := &CancelOrderRequest{ Header: *header, Iccid: iccid, FinalProfileStatusIndicator: targetState, @@ -360,7 +373,7 @@ func (client *ClientState) CancelOrder(iccid string, targetState string) (*Cance func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, error) { result := new(DownloadOrderResponse) es2plusCommand := "downloadOrder" - header, err := newEs2PlusHeader(client) + header, err := newHeader(client) if err != nil { return nil, err } @@ -386,7 +399,7 @@ func (client *ClientState) DownloadOrder(iccid string) (*DownloadOrderResponse, func (client *ClientState) ConfirmOrder(iccid string) (*ConfirmOrderResponse, error) { result := new(ConfirmOrderResponse) es2plusCommand := "confirmOrder" - header, err := newEs2PlusHeader(client) + header, err := newHeader(client) if err != nil { return nil, err } @@ -445,7 +458,7 @@ func (client *ClientState) ActivateIccid(iccid string) (*ProfileStatus, error) { return result, err } -// TODO: This shouldn't have to be public, but how can it be avoided? -func (state *ClientState) RequesterID() string { - return state.requesterID +// RequesterID TODO: This shouldn't have to be public, but how can it be avoided? +func (client *ClientState) RequesterID() string { + return client.requesterID } diff --git a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go index d57bc86de..c11d69995 100644 --- a/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go +++ b/sim-administration/sim-batch-management/fieldsyntaxchecks/fieldsyntaxchecks.go @@ -62,7 +62,7 @@ func IsICCID(s string) bool { return match } -// IsICCID if the string is an 18 or 19 digit positive integer. +// CheckICCIDSyntax if the string is an 18 or 19 digit positive integer. // Does check luhn checksum. func CheckICCIDSyntax(name string, potentialIccid string) { if !IsICCID(potentialIccid) { @@ -117,10 +117,10 @@ func CheckMSISDNSyntax(name string, potentialMSISDN string) { // the potential url is an actual url or not. If it isn't // then an error message is written and the program // terminates. -func CheckURLSyntax(name string, theUrl string) { - if _, err := url.ParseRequestURI(theUrl); err != nil { +func CheckURLSyntax(name string, theURL string) { + if _, err := url.ParseRequestURI(theURL); err != nil { // TODO: Fix this to return error value instead. - log.Fatalf("Not a valid %s URL: '%s'.", name, theUrl) + log.Fatalf("Not a valid %s URL: '%s'.", name, theURL) } } diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index 455775f8c..de26bc608 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -435,10 +435,9 @@ func parseCommandLine() error { if batch == nil { return fmt.Errorf("no batch found with name '%s'", *generateInputFileBatchname) - } else { - var result = generateInputFileString(batch) - fmt.Println(result) } + var result = generateInputFileString(batch) + fmt.Println(result) case "batch-add-msisdn-from-file": batchName := *addMsisdnFromFileBatch diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 26c513c1a..7aef6b0cf 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -327,10 +327,9 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, } if len(result) == 0 { - return nil, nil // TODO: Evaluate if this is wrong. Returning an empty slice isn't necessarily wrong. - } else { - return result, nil + return nil, nil // TODO: Evaluate if this is wrong. Returning an empty slice isn't necessarily wrong. } + return result, nil } From fa56c338afd9d159dfd90574eed6c8c03cbc9bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 15:04:19 +0100 Subject: [PATCH 297/309] More codacy suggestions --- sim-administration/sim-batch-management/build-all.sh | 2 +- .../sim-batch-management/es2plus/es2plus.go | 10 +++++----- .../sim-batch-management/model/model.go | 4 ++-- sim-administration/sim-batch-management/sbm.go | 12 ++++++------ .../sim-batch-management/store/store.go | 11 ++++------- .../sim-batch-management/store/store_test.go | 10 +++++----- 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/sim-administration/sim-batch-management/build-all.sh b/sim-administration/sim-batch-management/build-all.sh index de2d3f979..054e77781 100755 --- a/sim-administration/sim-batch-management/build-all.sh +++ b/sim-administration/sim-batch-management/build-all.sh @@ -28,7 +28,7 @@ fi ~/go/bin/staticcheck ./... -mv sim-batch-management sbm + # If sourcing this script, then the line below # will modify command line compesion in bash diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index a80f83290..fdaec5663 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -35,8 +35,6 @@ type Header struct { FunctionCallIdentifier string `json:"functionCallIdentifier"` } - - // GetProfileStatusRequest holds a request object for the profileStatus es2+ command. type GetProfileStatusRequest struct { Header Header `json:"header"` @@ -48,11 +46,13 @@ type ICCID struct { Iccid string `json:"iccid"` } +// FunctionExecutionStatus is part of the generic es2+ response. type FunctionExecutionStatus struct { FunctionExecutionStatusType string `json:"status"` StatusCodeData StatusCodeData `json:"statusCodeData"` } +// ResponseHeader is part of the generic response header in es2+ reponses. type ResponseHeader struct { FunctionExecutionStatus FunctionExecutionStatus `json:"FunctionExecutionStatus"` } @@ -77,9 +77,9 @@ type es2ProfileStatusResponse struct { CompletionTimestamp string `json:"completionTimestamp"` } -/// -/// The status of a profile as retrieved from the SM-DP+ -/// + +// ProfileStatus holds the "profile status" part of a ProfileStatusResponse +// - response returned from an es2+ request. type ProfileStatus struct { StatusLastUpdateTimestamp string `json:"status_last_update_timestamp"` ACToken string `json:"acToken"` diff --git a/sim-administration/sim-batch-management/model/model.go b/sim-administration/sim-batch-management/model/model.go index cbb78b9ce..a4eff8126 100644 --- a/sim-administration/sim-batch-management/model/model.go +++ b/sim-administration/sim-batch-management/model/model.go @@ -8,7 +8,7 @@ package model // subject to JSON serialisation/deserialisation, and can be stored // in persistent storage. type SimEntry struct { - Id int64 `db:"id" json:"id"` + ID int64 `db:"id" json:"id"` BatchID int64 `db:"batchID" json:"batchID"` RawIccid string `db:"rawIccid" json:"rawIccid"` IccidWithChecksum string `db:"iccidWithChecksum" json:"iccidWithChecksum"` @@ -49,7 +49,7 @@ type Batch struct { } -// Batch represents sim vendors. Instances can be +// ProfileVendor represents sim profile vendors. Instances can be // subject to JSON serialisation/deserialisation, and can be stored // in persistent storage. type ProfileVendor struct { diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index de26bc608..7e2083718 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -341,7 +341,7 @@ func parseCommandLine() error { if simProfile.Imsi != e.Imsi { return fmt.Errorf("profile enty for ICCID=%s has IMSI (%s), but we expected (%s)", e.Iccid, e.Imsi, simProfile.Imsi) } - db.UpdateSimEntryKi(simProfile.Id, e.Ki) + db.UpdateSimEntryKi(simProfile.ID, e.Ki) } // Signal to defered transaction cleanup that we're cool @@ -422,11 +422,11 @@ func parseCommandLine() error { if batch == nil { return fmt.Errorf("no batch found with name '%s'", *describeBatchBatch) - } else { - var csvPayload = uploadtoprime.GenerateCsvPayload(db, *batch) - uploadtoprime.GeneratePostingCurlscript(batch.URL, csvPayload) } + var csvPayload = uploadtoprime.GenerateCsvPayload(db, *batch) + uploadtoprime.GeneratePostingCurlscript(batch.URL, csvPayload) + case "batch-generate-input-file": batch, err := db.GetBatchByName(*generateInputFileBatchname) if err != nil { @@ -543,7 +543,7 @@ func parseCommandLine() error { } if entry.Msisdn == "" && record.msisdn != "" { - err = db.UpdateSimEntryMsisdn(entry.Id, record.msisdn) + err = db.UpdateSimEntryMsisdn(entry.ID, record.msisdn) if err != nil { tx.Rollback() return err @@ -820,7 +820,7 @@ func parseCommandLine() error { mutex.Lock() fmt.Printf("%s, %s\n", entry.Iccid, result.ACToken) - db.UpdateActivationCode(entry.Id, result.ACToken) + db.UpdateActivationCode(entry.ID, result.ACToken) mutex.Unlock() waitgroup.Done() }(entry) diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 7aef6b0cf..42893b11d 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -5,7 +5,7 @@ import ( "flag" "fmt" "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" // We need this + _ "github.com/mattn/go-sqlite3" // We need this "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/fieldsyntaxchecks" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/loltelutils" "github.com/ostelco/ostelco-core/sim-administration/sim-batch-management/model" @@ -277,7 +277,6 @@ func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, } } - // CreateSimEntry persists a SimEntry instance in the database. func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { @@ -297,7 +296,7 @@ func (sdb SimBatchDB) CreateSimEntry(theEntry *model.SimEntry) error { if err != nil { return fmt.Errorf("getting last inserted id failed '%s'", err) } - theEntry.Id = id + theEntry.ID = id return err } @@ -312,9 +311,9 @@ func (sdb SimBatchDB) GetSimEntryByID(simID int64) (*model.SimEntry, error) { if len(result) == 0 { return nil, nil - } else { - return &result[0], nil } + + return &result[0], nil } // GetAllSimEntriesForBatch retrieves a sim entryh instance stored in the database. If no @@ -332,7 +331,6 @@ func (sdb SimBatchDB) GetAllSimEntriesForBatch(batchID int64) ([]model.SimEntry, return result, nil } - // GetSimProfileByIccid gets a sim profile from the database, return nil of one // can't be found. func (sdb SimBatchDB) GetSimProfileByIccid(iccid string) (*model.SimEntry, error) { @@ -406,7 +404,6 @@ func (sdb *SimBatchDB) DropTables() error { return err } - // DeclareBatch generates a batch instance by first checking all of its // parameters, and then storing it, and finally returning it from the function. func (sdb SimBatchDB) DeclareBatch( diff --git a/sim-administration/sim-batch-management/store/store_test.go b/sim-administration/sim-batch-management/store/store_test.go index 9a72ec3a0..e165f7c1d 100644 --- a/sim-administration/sim-batch-management/store/store_test.go +++ b/sim-administration/sim-batch-management/store/store_test.go @@ -278,9 +278,9 @@ func TestDeclareAndRetrieveSimEntries(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Assert(t, entry.Id != 0) + assert.Assert(t, entry.ID != 0) - retrivedEntry, err := sdb.GetSimEntryByID(entry.Id) + retrivedEntry, err := sdb.GetSimEntryByID(entry.ID) if err != nil { t.Fatal(err) } @@ -313,16 +313,16 @@ func TestSimBatchDB_UpdateSimEntryKi(t *testing.T) { t.Fatal(err) } - assert.Assert(t, entry.Id != 0) + assert.Assert(t, entry.ID != 0) newKi := "12" - err = sdb.UpdateSimEntryKi(entry.Id, newKi) + err = sdb.UpdateSimEntryKi(entry.ID, newKi) if err != nil { t.Fatal(err) } - retrivedEntry, err := sdb.GetSimEntryByID(entry.Id) + retrivedEntry, err := sdb.GetSimEntryByID(entry.ID) if err != nil { t.Fatal(err) } From 921677d133d88a80c84168c4561114acf164c2d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Remseth?= Date: Thu, 21 Nov 2019 15:12:44 +0100 Subject: [PATCH 298/309] Building more --- .../sim-batch-management/README.md | 26 ++++++++++--------- .../sim-batch-management/es2plus/es2plus.go | 7 +++-- .../outfileparser/outfileparser.go | 2 +- .../sim-batch-management/sbm.go | 13 +++++----- .../sim-batch-management/store/store.go | 4 +-- 5 files changed, 27 insertions(+), 25 deletions(-) diff --git a/sim-administration/sim-batch-management/README.md b/sim-administration/sim-batch-management/README.md index f67700f2d..0335ff988 100644 --- a/sim-administration/sim-batch-management/README.md +++ b/sim-administration/sim-batch-management/README.md @@ -23,11 +23,13 @@ comments near the top of the files for instructions on how to use them. ##To build everything +Before we build, some things neds to be in order. + ### Prerequisites - * Go has to be installed on the system being run. + * Go has to be installed on the system being run. - * Prime needs to be accessible via ssh tunnel or otherwise from the host + * Prime needs to be accessible via ssh tunnel or otherwise from the host where the script is being run. ### Building @@ -81,23 +83,23 @@ that justifies adding these checks). ##TODO - 1. Create a very clean PR for future code review. +1. Create a very clean PR for future code review. - 2. Write up a nice markdown documentation describing common usecases. +2. Write up a nice markdown documentation describing common usecases. - 3. Add crypto resources so that the program can talk to external parties. +3. Add crypto resources so that the program can talk to external parties. - 4. Add code to activate profiles in HSS (if API is known) +4. Add code to activate profiles in HSS (if API is known) - 5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) +5. Add config for crypto parameters for HSSes, profile-vendors and operators (sftp in particular) - 6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that +6. Add misc. parameters about sim vendors, HSSes, Prime instances etc., so that batches can be properly constrained, defaults set the right way and external components accessed from gocode. - 7. Figure out how to handle workflows. Be explicit! +7. Figure out how to handle workflows. Be explicit! - 8. The interfaces to external parties will be +8. The interfaces to external parties will be - input/output files for profile generation. - some kind of file (not yet determined) for msisdn lists. - HTTP upload commands, either indirectly via curl (as now), or @@ -105,7 +107,7 @@ that justifies adding these checks). it will be assumed that tunnels are set up out of band, and tunnel setup is not part of this program. - 9. Declare legal hss/dpv combinations, batches must use legal combos. +9. Declare legal hss/dpv combinations, batches must use legal combos. - 10. Declare prime instances (should make sense to have both prod and dev defined +10. Declare prime instances (should make sense to have both prod and dev defined with different constraints on them). diff --git a/sim-administration/sim-batch-management/es2plus/es2plus.go b/sim-administration/sim-batch-management/es2plus/es2plus.go index fdaec5663..263cfd869 100644 --- a/sim-administration/sim-batch-management/es2plus/es2plus.go +++ b/sim-administration/sim-batch-management/es2plus/es2plus.go @@ -11,9 +11,8 @@ import ( "strings" ) -/// -/// External interface for ES2+ client -/// + +// Client is an external interface for ES2+ client type Client interface { GetStatus(iccid string) (*ProfileStatus, error) RecoverProfile(iccid string, targetState string) (*RecoverProfileResponse, error) @@ -93,7 +92,7 @@ type ProfileStatus struct { // Profile reset invocation // -// RecvoerProfileRequest is the payload of the recoverProfile request. +// RecoverProfileRequest is the payload of the recoverProfile request. type RecoverProfileRequest struct { Header Header `json:"header"` Iccid string `json:"iccid"` diff --git a/sim-administration/sim-batch-management/outfileparser/outfileparser.go b/sim-administration/sim-batch-management/outfileparser/outfileparser.go index 5e970f155..2a93a1ef1 100644 --- a/sim-administration/sim-batch-management/outfileparser/outfileparser.go +++ b/sim-administration/sim-batch-management/outfileparser/outfileparser.go @@ -71,7 +71,7 @@ func parseVarOutLine(varOutLine string, result *map[string]int) error { } -// Parse an output file, returning an OutputFileRecord, contained +// ParseOutputFile parses an output file, returning an OutputFileRecord, contained // a parsed version of the inputfile. func ParseOutputFile(filePath string) (*OutputFileRecord, error) { diff --git a/sim-administration/sim-batch-management/sbm.go b/sim-administration/sim-batch-management/sbm.go index 7e2083718..cff9f89c6 100755 --- a/sim-administration/sim-batch-management/sbm.go +++ b/sim-administration/sim-batch-management/sbm.go @@ -387,15 +387,16 @@ func parseCommandLine() error { if batch == nil { return fmt.Errorf("no batch found with name '%s'", *describeBatchBatch) - } else { - bytes, err := json.MarshalIndent(batch, " ", " ") - if err != nil { - return fmt.Errorf("can't serialize batch '%v'", batch) - } + } - fmt.Printf("%v\n", string(bytes)) + bytes, err := json.MarshalIndent(batch, " ", " ") + if err != nil { + return fmt.Errorf("can't serialize batch '%v'", batch) } + fmt.Printf("%v\n", string(bytes)) + + case "batch-generate-activation-code-updating-sql": batch, err := db.GetBatchByName(*generateActivationCodeSQLBatch) if err != nil { diff --git a/sim-administration/sim-batch-management/store/store.go b/sim-administration/sim-batch-management/store/store.go index 42893b11d..1f9981269 100644 --- a/sim-administration/sim-batch-management/store/store.go +++ b/sim-administration/sim-batch-management/store/store.go @@ -272,9 +272,9 @@ func (sdb SimBatchDB) GetProfileVendorByName(name string) (*model.ProfileVendor, if len(result) == 0 { return nil, nil - } else { - return &result[0], nil } + + return &result[0], nil } // CreateSimEntry persists a SimEntry instance in the database. From a0fda3ce719ef5b0c7a4be86f70fdb31d69216f2 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:42:51 +0100 Subject: [PATCH 299/309] Updated Customer DSL --- neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt index 76dc11393..98c40d27b 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/dsl/Syntax.kt @@ -231,6 +231,11 @@ infix fun Customer.Companion.withSubscription(subscription: SubscriptionContext) toId = subscription.id ) +infix fun Customer.Companion.withSimProfile(simProfile: SimProfileContext) = + RelatedToClause( + relationType = customerToSimProfileRelation, + toId = simProfile.id + ) // // ExCustomer // From 3739168e3ea2489e053a9a35175d3d2629a4bddb Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:44:05 +0100 Subject: [PATCH 300/309] Helper function fro UTC Timestamp String --- .../kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 311d04e37..4523b7e8f 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -132,6 +132,8 @@ import org.ostelco.prime.storage.graph.model.SubscriptionToBundle import org.ostelco.prime.tracing.Trace import java.time.Instant import java.time.LocalDate +import java.time.ZoneOffset +import java.time.ZonedDateTime import java.util.* import java.util.stream.Collectors import javax.ws.rs.core.MultivaluedMap @@ -2839,4 +2841,6 @@ fun Map.copy(key: K, value: V): Map { val mutableMap = this.toMutableMap() mutableMap[key] = value return mutableMap.toMap() -} \ No newline at end of file +} + +fun utcTimeNow() = ZonedDateTime.now(ZoneOffset.UTC).toString() \ No newline at end of file From f55f43db0093010d145e6db2191ef975edac2409 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:45:21 +0100 Subject: [PATCH 301/309] Customer createdOn timestamp --- model/src/main/kotlin/org/ostelco/prime/model/Entities.kt | 1 + .../main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index 6fb4d6b67..627a57a6a 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -43,6 +43,7 @@ data class Customer( override val id: String = UUID.randomUUID().toString(), val nickname: String, val contactEmail: String, + val createdOn: String? = null, val analyticsId: String = UUID.randomUUID().toString(), val referralId: String = UUID.randomUUID().toString()) : HasId { diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 4523b7e8f..428429b04 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -412,7 +412,7 @@ object Neo4jStoreSingleton : GraphStore { validateCreateCustomerParams(customer, referredBy).bind() val bundleId = UUID.randomUUID().toString() create { Identity(id = identity.id, type = identity.type) }.bind() - create { customer }.bind() + create { customer.copy(createdOn = utcTimeNow()) }.bind() fact { (Identity withId identity.id) identifies (Customer withId customer.id) using Identifies(provider = identity.provider) }.bind() create { Bundle(id = bundleId, balance = 0L) }.bind() fact { (Customer withId customer.id) hasBundle (Bundle withId bundleId) }.bind() From 3ede50b6979ac117297a8318a2467dea0649883a Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:48:40 +0100 Subject: [PATCH 302/309] ExCustomer createdOn --- .../kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 5 +++-- .../kotlin/org/ostelco/prime/storage/graph/model/Model.kt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 428429b04..2f80cf272 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -448,9 +448,10 @@ object Neo4jStoreSingleton : GraphStore { IO { Either.monad().binding { // get customer id - val customerId = getCustomerId(identity).bind() + val customer = getCustomer(identity).bind() + val customerId = customer.id // create ex-customer with same id - create { ExCustomer(id = customerId, terminationDate = LocalDate.now().toString()) }.bind() + create { ExCustomer(id = customerId, terminationDate = LocalDate.now().toString(), createdOn = customer.createdOn) }.bind() // get all subscriptions and link them to ex-customer val subscriptions = get(Subscription subscribedBy (Customer withId customerId)).bind() for (subscription in subscriptions) { diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt index 2f5f9cc87..4f44bbf64 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt @@ -42,6 +42,7 @@ data class Offer(override val id: String) : HasId data class ExCustomer( override val id:String, + val createdOn: String? = null, val terminationDate: String) : HasId { companion object From ee957c85bc63f5e2fd700dac7db115d5b0edb73d Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 20:42:28 +0100 Subject: [PATCH 303/309] Unit tests --- .../org/ostelco/prime/storage/graph/Neo4jStoreTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt index be5112319..37f352b5f 100644 --- a/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt +++ b/neo4j-store/src/test/kotlin/org/ostelco/prime/storage/graph/Neo4jStoreTest.kt @@ -146,7 +146,7 @@ class Neo4jStoreTest { Neo4jStoreSingleton.getCustomer(IDENTITY).bimap( { fail(it.message) }, - { assertEquals(CUSTOMER, it) }) + { assertEquals(CUSTOMER.copy(createdOn = it.createdOn), it) }) // TODO vihang: fix argument captor for neo4j-store tests // val bundleArgCaptor: ArgumentCaptor = ArgumentCaptor.forClass(Bundle::class.java) @@ -169,7 +169,7 @@ class Neo4jStoreTest { val identity: Identity = list.first() Neo4jStoreSingleton.getCustomer(identity).bimap( { fail(it.message) }, - { assertEquals(CUSTOMER, it) })}) + { assertEquals(CUSTOMER.copy(createdOn = it.createdOn), it) })}) } @Test @@ -1209,7 +1209,7 @@ class Neo4jStoreTest { val exCustomer = get(ExCustomer withId CUSTOMER.id).bind() assertEquals( - expected = ExCustomer(id = CUSTOMER.id, terminationDate = "%d-%02d-%02d".format(LocalDate.now().year, LocalDate.now().monthValue, LocalDate.now().dayOfMonth)), + expected = ExCustomer(id = CUSTOMER.id, createdOn = exCustomer.createdOn, terminationDate = "%d-%02d-%02d".format(LocalDate.now().year, LocalDate.now().monthValue, LocalDate.now().dayOfMonth)), actual = exCustomer, message = "ExCustomer does not match") From 9c90a19c4eaa2d6ada40e565a174eef7e59de645 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:49:54 +0100 Subject: [PATCH 304/309] SimProfile timestamps --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 18 ++++++++++++++++-- .../ostelco/prime/storage/graph/model/Model.kt | 6 +++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 2f80cf272..419cc6f15 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -35,6 +35,7 @@ import org.ostelco.prime.dsl.withCode import org.ostelco.prime.dsl.withId import org.ostelco.prime.dsl.withKyc import org.ostelco.prime.dsl.withMsisdn +import org.ostelco.prime.dsl.withSimProfile import org.ostelco.prime.dsl.withSku import org.ostelco.prime.dsl.withSubscription import org.ostelco.prime.dsl.writeTransaction @@ -73,6 +74,8 @@ import org.ostelco.prime.model.ScanStatus import org.ostelco.prime.model.SimEntry import org.ostelco.prime.model.SimProfileStatus import org.ostelco.prime.model.SimProfileStatus.AVAILABLE_FOR_DOWNLOAD +import org.ostelco.prime.model.SimProfileStatus.DELETED +import org.ostelco.prime.model.SimProfileStatus.DOWNLOADED import org.ostelco.prime.model.SimProfileStatus.INSTALLED import org.ostelco.prime.model.SimProfileStatus.NOT_READY import org.ostelco.prime.model.Subscription @@ -683,7 +686,7 @@ object Neo4jStoreSingleton : GraphStore { fun subscribeToSimProfileStatusUpdates() { simManager.addSimProfileStatusUpdateListener { iccId, status -> - readTransaction { + writeTransaction { IO { Either.monad().binding { logger.info("Received status {} for iccId {}", status, iccId) @@ -692,6 +695,16 @@ object Neo4jStoreSingleton : GraphStore { logger.warn("Found {} SIM Profiles with iccId {}", simProfiles.size, iccId) } simProfiles.forEach { simProfile -> + val customers = get(Customer withSimProfile (SimProfile withId simProfile.id)).bind() + customers.forEach { customer -> + AuditLog.info(customerId = customer.id, message = "Sim Profile (iccId = $iccId) is $status") + } + when(status) { + DOWNLOADED -> update { simProfile.copy(downloadedOn = utcTimeNow()) }.bind() + INSTALLED -> update { simProfile.copy(installedOn = utcTimeNow()) }.bind() + DELETED -> update { simProfile.copy(deletedOn = utcTimeNow()) }.bind() + else -> logger.warn("Not storing timestamp for simProfile: {} for status: {}", iccId, status) + } val subscriptions = get(Subscription under (SimProfile withId simProfile.id)).bind() subscriptions.forEach { subscription -> logger.info("Notify status {} for subscription.analyticsId {}", status, subscription.analyticsId) @@ -700,6 +713,7 @@ object Neo4jStoreSingleton : GraphStore { } }.fix() }.unsafeRunSync() + // Skipping transaction rollback since it is just updating timestamps } } } @@ -743,7 +757,7 @@ object Neo4jStoreSingleton : GraphStore { val simEntry = simManager.allocateNextEsimProfile(hlr = hssNameLookup.getHssName(region.id.toLowerCase()), phoneType = profileType) .mapLeft { NotFoundError("eSIM profile", id = "Loltel") } .bind() - val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId) + val simProfile = SimProfile(id = UUID.randomUUID().toString(), iccId = simEntry.iccId, requestedOn = utcTimeNow()) create { simProfile }.bind() fact { (Customer withId customerId) has (SimProfile withId simProfile.id) }.bind() fact { (SimProfile withId simProfile.id) isFor (Region withCode regionCode.toLowerCase()) }.bind() diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt index 4f44bbf64..6f2abd46c 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt @@ -29,7 +29,11 @@ data class CustomerRegion( data class SimProfile( override val id: String, val iccId: String, - val alias: String = "") : HasId { + val alias: String = "", + val requestedOn: String? = null, + val downloadedOn: String? = null, + val installedOn: String? = null, + val deletedOn: String? = null) : HasId { companion object } From 75621ba265a0249d5e0084586ed04c6e5b34a5d6 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 19:00:09 +0100 Subject: [PATCH 305/309] Timestamps for Region approval init and complete. --- .../ostelco/prime/storage/graph/Neo4jStore.kt | 45 +++++++++++++++---- .../prime/storage/graph/model/Model.kt | 4 +- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 419cc6f15..6c94b3d8e 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -6,7 +6,6 @@ import arrow.core.Either.Right import arrow.core.EitherOf import arrow.core.fix import arrow.core.flatMap -import arrow.core.getOrElse import arrow.core.left import arrow.core.leftIfNull import arrow.core.right @@ -2054,14 +2053,28 @@ object Neo4jStoreSingleton : GraphStore { return IO { Either.monad().binding { + // get combinations of KYC needed for this region to be Approved val approvedKycTypeSetList = getApprovedKycTypeSetList(regionCode) + // fetch existing values from DB val existingCustomerRegion = customerRegionRelationStore.get( fromId = customer.id, toId = regionCode, transaction = transaction) - .getOrElse { CustomerRegion(status = PENDING, kycStatusMap = getKycStatusMapForRegion(regionCode)) } + .flatMapLeft { storeError -> + if(storeError is NotFoundError && storeError.type == customerRegionRelation.name) { + // default value if absent in DB + CustomerRegion( + status = PENDING, + kycStatusMap = getKycStatusMapForRegion(regionCode), + initiatedOn = utcTimeNow() + ).right() + } else { + storeError.left() + } + }.bind() + // using existing and received KYC status, compute new KYC status val existingKycStatusMap = existingCustomerRegion.kycStatusMap val existingKycStatus = existingKycStatusMap[kycType] val newKycStatus = when (existingKycStatus) { @@ -2071,6 +2084,7 @@ object Neo4jStoreSingleton : GraphStore { else -> kycStatus } + // if new status is different from existing status if (existingKycStatus != newKycStatus) { if (kycStatus == newKycStatus) { AuditLog.info(customerId = customer.id, message = "Setting $kycType status from $existingKycStatus to $newKycStatus") @@ -2092,29 +2106,42 @@ object Neo4jStoreSingleton : GraphStore { AuditLog.info(customerId = customer.id, message = "Ignoring setting $kycType status to $kycStatus since it is already $existingKycStatus") } + // update KYC status map with new value. This map will then be stored in DB. val newKycStatusMap = existingKycStatusMap.copy(key = kycType, value = newKycStatus) - val approved = approvedKycTypeSetList.any { kycTypeSet -> + // check if Region is Approved. + val isRegionApproved = approvedKycTypeSetList.any { kycTypeSet -> + // Region is approved if the set of Approved KYCs is a superset of any one of the set configured in the list - approvedKycTypeSetList. newKycStatusMap.filter { it.value == KycStatus.APPROVED }.keys.containsAll(kycTypeSet) } - val approvedNow = existingCustomerRegion.status == PENDING && approved + // if the Region status is Approved, but the existing status was not Approved, then it has been approved now. + val isRegionApprovedNow = existingCustomerRegion.status != APPROVED && isRegionApproved - val newStatus = if (approved) { + // Save Region status as APPROVED, if it is approved. Do not change Region status otherwise. + val newRegionStatus = if (isRegionApproved) { APPROVED } else { existingCustomerRegion.status } - if (approvedNow) { + // timestamp for region approval + val regionApprovedOn = if (isRegionApprovedNow) { + AuditLog.info(customerId = customer.id, message = "Approved for region - $regionCode") + onRegionApprovedAction.apply( customer = customer, regionCode = regionCode, transaction = PrimeTransaction(transaction) ).bind() + + utcTimeNow() + } else { + existingCustomerRegion.approvedOn } + // Save KYC expiry date if it is not null. val newKycExpiryDateMap = kycExpiryDate ?.let { existingCustomerRegion.kycExpiryDateMap.copy(key = kycType, value = it) } ?: existingCustomerRegion.kycExpiryDateMap @@ -2123,9 +2150,11 @@ object Neo4jStoreSingleton : GraphStore { .createOrUpdate( fromId = customer.id, relation = CustomerRegion( - status = newStatus, + status = newRegionStatus, kycStatusMap = newKycStatusMap, - kycExpiryDateMap = newKycExpiryDateMap + kycExpiryDateMap = newKycExpiryDateMap, + initiatedOn = existingCustomerRegion.initiatedOn, + approvedOn = regionApprovedOn ), toId = regionCode, transaction = transaction) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt index 6f2abd46c..3454353e3 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt @@ -24,7 +24,9 @@ data class PlanSubscription( data class CustomerRegion( val status: CustomerRegionStatus, val kycStatusMap: Map = emptyMap(), - val kycExpiryDateMap: Map = emptyMap()) + val kycExpiryDateMap: Map = emptyMap(), + val initiatedOn: String? = null, + val approvedOn: String? = null) data class SimProfile( override val id: String, From 17fc41a57db398e1af76e6b9c8afa9df7dc0cf69 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 20:09:32 +0100 Subject: [PATCH 306/309] OCS timestamps --- model/src/main/kotlin/org/ostelco/prime/model/Entities.kt | 6 ++++-- .../kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 4 ++-- .../kotlin/org/ostelco/prime/storage/graph/model/Model.kt | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt index 627a57a6a..801da16c5 100644 --- a/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt +++ b/model/src/main/kotlin/org/ostelco/prime/model/Entities.kt @@ -228,7 +228,8 @@ data class ApplicationToken( data class Subscription( val msisdn: String, - val analyticsId: String = UUID.randomUUID().toString()) : HasId { + val analyticsId: String = UUID.randomUUID().toString(), + val lastActiveOn: String? = null) : HasId { override val id: String @JsonIgnore @@ -239,7 +240,8 @@ data class Subscription( data class Bundle( override val id: String, - val balance: Long) : HasId { + val balance: Long, + val lastConsumedOn: String? = null) : HasId { companion object } diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 6c94b3d8e..268adef05 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1139,10 +1139,10 @@ object Neo4jStoreSingleton : GraphStore { writeSuspended(""" MATCH (sn:${subscriptionEntity.name} {id: '$msisdn'})-[r:${subscriptionToBundleRelation.name}]->(bundle:${bundleEntity.name}) - SET bundle._LOCK_ = true, r._LOCK_ = true + SET bundle._LOCK_ = true, r._LOCK_ = true, sn.lastActiveOn="${utcTimeNow()}" WITH r, bundle, sn.analyticsId AS msisdnAnalyticsId, (CASE WHEN ((toInteger(bundle.balance) + toInteger(r.reservedBytes) - $usedBytes) > 0) THEN (toInteger(bundle.balance) + toInteger(r.reservedBytes) - $usedBytes) ELSE 0 END) AS tmpBalance WITH r, bundle, msisdnAnalyticsId, tmpBalance, (CASE WHEN (tmpBalance < $requestedBytes) THEN tmpBalance ELSE $requestedBytes END) as tmpGranted - SET r.reservedBytes = toString(tmpGranted), bundle.balance = toString(tmpBalance - tmpGranted) + SET r.reservedBytes = toString(tmpGranted), r.reservedOn = "${utcTimeNow()}", bundle.balance = toString(tmpBalance - tmpGranted), bundle.lastConsumedOn="${utcTimeNow()}" REMOVE r._LOCK_, bundle._LOCK_ RETURN msisdnAnalyticsId, r.reservedBytes AS granted, bundle.balance AS balance """.trimIndent(), diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt index 3454353e3..2d352bb6e 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/model/Model.kt @@ -14,7 +14,10 @@ data class Identity( data class Identifies(val provider: String) -data class SubscriptionToBundle(val reservedBytes: Long = 0) +data class SubscriptionToBundle( + val reservedBytes: Long = 0, + val reservedOn: String? = null +) data class PlanSubscription( val subscriptionId: String, From c68cfeddf5d9d27dc945cc375b604d3ed4684709 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:50:36 +0100 Subject: [PATCH 307/309] Notify slack for failed Jumio cases only --- .../kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt index 268adef05..ee95e731d 100644 --- a/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt +++ b/neo4j-store/src/main/kotlin/org/ostelco/prime/storage/graph/Neo4jStore.kt @@ -1810,7 +1810,6 @@ object Neo4jStoreSingleton : GraphStore { customerId = customer.id, data = extendedStatus ) - logger.info(NOTIFY_OPS_MARKER, "Jumio verification succeeded for ${customer.contactEmail} Info: $extendedStatus") setKycStatus( customer = customer, regionCode = updatedScanInformation.countryCode.toLowerCase(), @@ -1829,7 +1828,9 @@ object Neo4jStoreSingleton : GraphStore { data = extendedStatus ) } - logger.info(NOTIFY_OPS_MARKER, "Jumio verification failed for ${customer.contactEmail} Info: $extendedStatus") + if(updatedScanInformation.scanResult?.verificationStatus != "NO_ID_UPLOADED") { + logger.info(NOTIFY_OPS_MARKER, "Jumio verification failed for ${customer.contactEmail} Info: $extendedStatus") + } setKycStatus( customer = customer, regionCode = updatedScanInformation.countryCode.toLowerCase(), From d55f8d94017ed11a0a6ee76578788262ce1d67e1 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 20:51:26 +0100 Subject: [PATCH 308/309] Removed MyInfo PersonData logging --- .../kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt index ed810bdbd..77a73b224 100644 --- a/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt +++ b/ekyc/src/main/kotlin/org/ostelco/prime/ekyc/myinfo/v3/MyInfoClient.kt @@ -228,11 +228,9 @@ object MyInfoClientSingleton : MyInfoKycService { } if (config.myInfoApiEnableSecurity && httpMethod == GET) { - // TODO vihang: Remove after initial testing is done. - logger.info("jwe PersonData: {}", content) + // logger.info("jwe PersonData: {}", content) val jws = decodeJweCompact(content) - // TODO vihang: Remove after initial testing is done. - logger.info("jws PersonData: {}", jws) + // logger.info("jws PersonData: {}", jws) return getPersonDataFromJwsClaims(jws) } From 1a2960aa175703ad9507790bdd6f7e1616a5b0c0 Mon Sep 17 00:00:00 2001 From: Vihang Patil Date: Thu, 21 Nov 2019 15:42:00 +0100 Subject: [PATCH 309/309] Updated prime version --- prime/build.gradle.kts | 2 +- prime/script/start.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prime/build.gradle.kts b/prime/build.gradle.kts index b70787748..60f722b1b 100644 --- a/prime/build.gradle.kts +++ b/prime/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } // Update version in [script/start.sh] too. -version = "1.73.2" +version = "1.74.0" dependencies { // interface module between prime and prime-modules diff --git a/prime/script/start.sh b/prime/script/start.sh index 23410f1a1..aeceda85c 100755 --- a/prime/script/start.sh +++ b/prime/script/start.sh @@ -5,5 +5,5 @@ exec java \ -Dfile.encoding=UTF-8 \ --add-opens java.base/java.lang=ALL-UNNAMED \ --add-opens java.base/java.io=ALL-UNNAMED \ - -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.73.2,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ + -agentpath:/opt/cprof/profiler_java_agent.so=-cprof_service=prime,-cprof_service_version=1.74.0,-logtostderr,-minloglevel=2,-cprof_enable_heap_sampling \ -jar /prime.jar server /config/config.yaml