Merge pull request #186 from shikharish/postgres

Migrate to Postgres in Development

.env.template

@@ -1,11 +1,9 @@
-DEV=false
-DEV_DB_PATH=devDB.db
 BACKEND_PORT=8080
 DATABASE_HOST=
 DATABASE_PORT=
 DATABASE_NAME=
 DATABASE_USERNAME=
-DATABASE_PORT=
+DATABASE_PASSWORD=
 JWT_SECRET_KEY=
 JWT_VALIDITY_TIME=720
 GH_OAUTH_CLIENT_ID=

.github/workflows/test_and_lint.yml

@@ -8,6 +8,14 @@ on:
 
 jobs:
   test_and_lint:
+    env:
+      BACKEND_PORT: 8080
+      DATABASE_HOST: localhost
+      DATABASE_PORT: 5432
+      DATABASE_NAME: testDb
+      DATABASE_USERNAME: user
+      DATABASE_PASSWORD: pass
+
     runs-on: ubuntu-latest
     strategy:
         matrix:
@@ -22,7 +30,10 @@ jobs:
           go-version: ${{ matrix.go-version }}
 
       - name: Test
-        run: go test -v ./...
+        run: |
+          docker compose -f docker-compose.yaml up --build -d
+          go test ./... -p 1
+          docker compose -f docker-compose.yaml down
 
       - name: Lint
         uses: golangci/golangci-lint-action@v3

.gitignore

@@ -1,3 +1,4 @@
 ./backend
-.env
-*.db
+/.env
+*.db
+vendor/

README.md

@@ -64,13 +64,14 @@
 ### Setting Up Locally
 
 - Install [Go](https://go.dev/).
+- Install [Docker](https://docs.docker.com/engine/install/).
 - Clone this repository.
 - Run `go mod download` in the cloned repository to download all the dependencies.
 - Create a `.env` file to store all the [environment variables](#environment-variables). You can use the `.env.template` file for this.
-- Set the `DEV` environment variable to `true`.
+- Run `docker compose up -d` to start the PostgreSQL database container. (Make sure all the required [environment variables](#environment-variables) are set)
 - Optionally set up [Github OAuth](#github-oauth) to test the endpoints which require login. (See also: [Endpoints](#endpoints))
 - Run `go run cmd/backend.go` to start the server.
-- Optionally install [SQLiteStudio](https://sqlitestudio.pl/) or a similar tool to help manage the local database `devDB.db` (sqlite3).
+- Optionally install [pgAdmin](https://www.pgadmin.org/) or [DBeaver](https://dbeaver.io/) or a similar tool to help manage the local database (PostgreSQL).
 - Optionally install [Postman](https://www.postman.com/) or a similar tool to test the API endpoints.
 - Optionally (but **recommended**) [set up pre-commit hooks](#setting-up-pre-commit-hooks).
 
@@ -88,7 +89,7 @@
 
 See also [File Structure](#file-structure).
 
-1. Test Files: Tests for a particular `file.go` are placed in a file named `file_test.go` and in the same directory.
+1. Test Files: Tests for a particular `file.go` are placed in the `tests` directory in a file named `file_test.go`.
 2. Model Files: Database models are placed in the `models` directory. Each file corresponds to one table in the database, and the name of the file corresponds with the name of the database table.
 
 ### Testing
@@ -97,7 +98,7 @@ See also [File Naming Convention](#file-naming-convention).
 
 All controllers except `/oauth/` are tested. New controllers should include tests. For controllers, not only successes but failures should also be tested.
 
-The file `controllers/common_test.go` exports functions commonly used in controller tests.
+The file `tests/common_test.go` exports functions commonly used in controller tests.
 
 ## Project Structure
 
@@ -323,14 +324,12 @@ The following command-line arguments are accepted by `cmd/backend.go`. `--argume
 
 Environment variables can be set using a `.env` (See [Command-Line Arguments](#command-line-arguments) to use a different file) file. The following variables are used. (See the `.env.template` file for an example)
 
-- `DEV`: When set to `true`, uses a local sqlite3 database from a file `devDB.db` (or `DEV_DB_PATH` if set).
-- `DEV_DB_PATH`: The path to a local sqlite3 database file to be used in development. (Valid when `DEV` is set to `true`) (Default: `devDB.db`) (NOTE: `testDB.db` is used for testing)
 - `BACKEND_PORT`: The port on which the backend server runs. (Default: `8080`)
-- `DATABASE_USERNAME`: The username used to log into the database. (Valid when `DEV` is not set to `true`)
-- `DATABASE_PASSWORD`: The password used to log into the database. (valid when `DEV` is not set to `true`)
-- `DATABASE_NAME`: The name of the database to log into. (Valid when `DEV` is not set to `true`)
-- `DATABASE_HOST`: The host/url used to log into the database. (Valid when `DEV` is not set to `true`)
-- `DATABASE_PORT`: The port used to log into the database. (Valid when `DEV` is not set to `true`)
+- `DATABASE_USERNAME`: The username used to log into the database.
+- `DATABASE_PASSWORD`: The password used to log into the database.
+- `DATABASE_NAME`: The name of the database to log into.
+- `DATABASE_HOST`: The host/url used to log into the database.
+- `DATABASE_PORT`: The port used to log into the database.
 - `GH_OAUTH_CLIENT_ID`: The client id used for Github OAuth. (See [Github OAuth](#github-oauth))
 - `GITHUB_OAUTH_CLIENT_SECRET`: The client secret used for Github OAuth. (See [Github OAuth](#github-oauth))
 - `JWT_SECRET_KEY`: The secret key used to create a JWT token. (It can be a randomly generated string)

docker-compose.yaml

@@ -0,0 +1,17 @@
+services:
+  database:
+    image: postgres
+    restart: always
+    environment:
+      - POSTGRES_USER=${DATABASE_USERNAME}
+      - POSTGRES_PASSWORD=${DATABASE_PASSWORD}
+      - POSTGRES_DB=${DATABASE_NAME}
+      - PGPORT=${DATABASE_PORT}
+      - PGHOST=${DATABASE_HOST}
+    ports:
+      - ${DATABASE_PORT}:${DATABASE_PORT}
+    volumes:
+      - db:/var/lib/postgresql/data
+
+volumes:
+  db:

go.mod

@@ -9,7 +9,6 @@ require (
 	github.com/rs/cors v1.10.1
 	github.com/rs/zerolog v1.28.0
 	gorm.io/driver/postgres v1.5.2
-	gorm.io/driver/sqlite v1.5.2
 	gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55
 )
 
@@ -21,7 +20,6 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-sqlite3 v1.14.17 // indirect
 	golang.org/x/crypto v0.8.0 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/text v0.9.0 // indirect

go.sum

@@ -22,8 +22,6 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -49,7 +47,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gorm.io/driver/postgres v1.5.2 h1:ytTDxxEv+MplXOfFe3Lzm7SjG09fcdb3Z/c056DTBx0=
 gorm.io/driver/postgres v1.5.2/go.mod h1:fmpX0m2I1PKuR7mKZiEluwrP3hbs+ps7JIGMUBpCgl8=
-gorm.io/driver/sqlite v1.5.2 h1:TpQ+/dqCY4uCigCFyrfnrJnrW9zjpelWVoEVNy5qJkc=
-gorm.io/driver/sqlite v1.5.2/go.mod h1:qxAuCol+2r6PannQDpOP1FP6ag3mKi4esLnB/jHed+4=
 gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55 h1:sC1Xj4TYrLqg1n3AN10w871An7wJM0gzgcm8jkIkECQ=
 gorm.io/gorm v1.25.2-0.20230530020048-26663ab9bf55/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=

controllers/common_test.go → tests/common_test.go

@@ -10,6 +10,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/joho/godotenv"
+	"github.com/kossiitkgp/kwoc-backend/v2/models"
 	"github.com/kossiitkgp/kwoc-backend/v2/server"
 	"github.com/kossiitkgp/kwoc-backend/v2/utils"
 
@@ -53,18 +55,18 @@ func expectResponseJSONBodyToBe[T comparable](t *testing.T, res *httptest.Respon
 }
 
 func setTestDB() *gorm.DB {
-	os.Setenv("DEV", "true")
-	os.Setenv("DEV_DB_PATH", "testDB.db")
+	_ = godotenv.Load("../.env")
 	db, _ := utils.GetDB()
 	_ = utils.MigrateModels(db)
 
 	return db
 }
 
-func unsetTestDB() {
-	os.Unsetenv("DEV_DB_PATH")
-	os.Unsetenv("DEV")
-	os.Remove("testDB.db")
+func unsetTestDB(db *gorm.DB) {
+	db.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(&models.Project{})
+	db.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(&models.Student{})
+	db.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(&models.Stats{})
+	db.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(&models.Mentor{})
 }
 
 func setTestJwtSecretKey() {

controllers/healthcheck_test.go → tests/healthcheck_test.go

@@ -15,7 +15,7 @@ func TestPing(t *testing.T) {
 
 func TestHealthCheck(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	req, _ := http.NewRequest("GET", "/healthcheck/", nil)
 	res := executeRequest(req, db)

controllers/mentor_test.go → tests/mentor_test.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"gorm.io/gorm"
 	"net/http"
 	"reflect"
 	"strings"
 	"testing"
 
+	"gorm.io/gorm"
+
 	"github.com/kossiitkgp/kwoc-backend/v2/controllers"
 	"github.com/kossiitkgp/kwoc-backend/v2/models"
 	"github.com/kossiitkgp/kwoc-backend/v2/utils"
@@ -133,7 +134,7 @@ func tMentorRegAsStudent(db *gorm.DB, t *testing.T) {
 func TestMentorRegOK(t *testing.T) {
 	// Set up a local test database path
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()
@@ -178,7 +179,7 @@ func TestMentorDashboardInvalidAuth(t *testing.T) {
 func TestMentorDashboardNoReg(t *testing.T) {
 	// Set up a local test database path
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()
@@ -207,7 +208,7 @@ func TestMentorDashboardNoReg(t *testing.T) {
 func TestMentorDashboardOK(t *testing.T) {
 	// Set up a local test database path
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()
@@ -229,9 +230,10 @@ func TestMentorDashboardOK(t *testing.T) {
 
 	mentorID := int32(modelMentor.ID)
 	testProjects := generateTestProjects(5, false, true)
-	testProjects[1].MentorId = int32(modelMentor.ID)
-	testProjects[3].SecondaryMentorId = &mentorID
-
+	for i := range testProjects {
+		testProjects[i].MentorId = mentorID
+		testProjects[i].SecondaryMentorId = &mentorID
+	}
 	var projects []controllers.ProjectInfo
 	var students []controllers.StudentInfo
 

controllers/project_fetch_test.go → tests/project_fetch_test.go

@@ -83,7 +83,7 @@ func areProjectsEquivalent(proj1 *controllers.Project, proj2 *models.Project) bo
 
 func TestFetchAllProjects(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	testProjects := generateTestProjects(10, true, true)
 
@@ -151,7 +151,7 @@ func TestFetchProjDetailsInvalidID(t *testing.T) {
 // Try fetching a project that does not exist
 func TestFetchProjDetailsDNE(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()
@@ -176,7 +176,7 @@ func TestFetchProjDetailsDNE(t *testing.T) {
 // Try to fetch a valid project
 func TestFetchProjDetailsOK(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()
@@ -189,9 +189,14 @@ func TestFetchProjDetailsOK(t *testing.T) {
 	testJwt, _ := utils.GenerateLoginJwtString(testLoginFields)
 
 	testProjects := generateTestProjects(5, false, true)
-	for i, proj := range testProjects {
-		proj.Mentor = models.Mentor{Username: testUsername}
-		testProjects[i] = proj
+	modelMentor := models.Mentor{Username: testUsername}
+	db.Table("mentors").Create(&modelMentor)
+	mentorId := int32(modelMentor.ID)
+
+	for i := range testProjects {
+		testProjects[i].Mentor = modelMentor
+		testProjects[i].MentorId = mentorId
+		testProjects[i].SecondaryMentorId = &mentorId
 	}
 
 	_ = db.Table("projects").Create(testProjects)

controllers/project_reg_test.go → tests/project_reg_test.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"gorm.io/gorm"
 	"math/rand"
 	"net/http"
 	"strings"
 	"testing"
 
+	"gorm.io/gorm"
+
 	"github.com/kossiitkgp/kwoc-backend/v2/controllers"
 	"github.com/kossiitkgp/kwoc-backend/v2/utils"
 )
@@ -74,7 +75,7 @@ func TestProjectRegSessionHijacking(t *testing.T) {
 func TestProjectRegInvalidMentor(t *testing.T) {
 	// Set up a local test database path
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()
@@ -135,7 +136,7 @@ func tProjectRegExisting(db *gorm.DB, testUsername string, testJwt string, t *te
 func TestProjectRegOK(t *testing.T) {
 	// Set up a local test database path
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()

controllers/project_update_test.go → tests/project_update_test.go

@@ -80,8 +80,7 @@ func tProjectUpdateNonExistent(db *gorm.DB, testUsername string, testJwt string,
 func tProjectUpdateExistent(db *gorm.DB, testUsername string, testJwt string, t *testing.T) {
 	// Register a test project
 	projRegFields := createTestProjectRegFields(testUsername, "")
-
-	db.Create(&models.Project{
+	proj := models.Project{
 		Name:          projRegFields.Name,
 		Description:   projRegFields.Description,
 		Tags:          strings.Join(projRegFields.Tags, ","),
@@ -96,11 +95,12 @@ func tProjectUpdateExistent(db *gorm.DB, testUsername string, testJwt string, t
 		SecondaryMentor: models.Mentor{
 			Username: projRegFields.SecondaryMentorUsername,
 		},
-	})
+	}
+	db.Table("projects").Create(&proj)
 
 	// Create updated fields
 	projUpdateFields := &controllers.UpdateProjectReqFields{
-		Id:             1,
+		Id:             proj.ID,
 		Name:           fmt.Sprintf("Nename %d", rand.Int()),
 		Description:    "New description.",
 		Tags:           strings.Split("New,Tags,Test", ","),
@@ -163,7 +163,7 @@ func tProjectUpdateExistent(db *gorm.DB, testUsername string, testJwt string, t
 func TestProjectUpdateOK(t *testing.T) {
 	// Set up a local test database path
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	// Generate a jwt secret key for testing
 	setTestJwtSecretKey()

controllers/stats_overall_test.go → tests/stats_overall_test.go

@@ -64,7 +64,7 @@ func areOverallStatsEquivalent(stats *controllers.OverallStats, dbStats *models.
 
 func TestFetchOverallStats(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	testStats := generateTestStats(10)
 

controllers/stats_project_test.go → tests/stats_project_test.go

@@ -40,7 +40,7 @@ func findProjIndex(repo_link string, list []models.Project) int {
 
 func TestFetchAllProjectStats(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	test_projects := generateTestProjects(10, true, true)
 

controllers/stats_student_test.go → tests/stats_student_test.go

@@ -55,7 +55,7 @@ func areStudentStatsEquivalent(stats *controllers.StudentBriefStats, student *mo
 
 func TestFetchAllStudentsStats(t *testing.T) {
 	db := setTestDB()
-	defer unsetTestDB()
+	defer unsetTestDB(db)
 
 	testStudents := generateTestStudents(10)