Fixed file similarity search and changed to gpt-4o

db-connector.go

@@ -2357,7 +2357,7 @@ func FindSimilarFile(ctx context.Context, md5, orgId string) ([]File, error) {
 			}
 		}
 	} else {
-		query := datastore.NewQuery(nameKey).Filter("md5_sum =", md5).Limit(25)
+		query := datastore.NewQuery(nameKey).Filter("md5_sum =", md5).Limit(250)
 		_, err := project.Dbclient.GetAll(ctx, query, &files)
 		if err != nil {
 			log.Printf("[WARNING] Failed getting deals for org: %s", orgId)
@@ -8521,9 +8521,11 @@ func SetFile(ctx context.Context, file File) error {
 		file.CreatedAt = timeNow
 	}
 
+	/*
 	if !strings.HasPrefix(file.Id, "file_") {
 		return errors.New("Invalid file ID. Must start with file_")
 	}
+	*/
 
 	cacheKey := fmt.Sprintf("%s_%s", nameKey, file.Id)
 

files.go

@@ -391,14 +391,14 @@ func HandleDeleteFile(resp http.ResponseWriter, request *http.Request) {
 		file.Status = "deleted"
 		err = SetFile(ctx, *file)
 		if err != nil {
-			log.Printf("[ERROR] Failed setting file to deleted")
+			log.Printf("[ERROR] Failed setting file to deleted: %s", err)
 			resp.WriteHeader(500)
 			resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`))
 			return
 		}
 
 		outputFiles, err := FindSimilarFile(ctx, file.Md5sum, file.OrgId)
-		log.Printf("[INFO] Found %d similar files", len(outputFiles))
+		log.Printf("[INFO] Found %d similar files for Md5 '%s'", len(outputFiles), file.Md5sum)
 		if len(outputFiles) > 0 {
 			for _, item := range outputFiles {
 				item.Status = "deleted"
@@ -596,7 +596,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
 	// also be environment variables / input arguments
 	filename, filenameOk := request.URL.Query()["filename"]
 	if filenameOk && ArrayContains(reservedCategoryNames, namespace) {
-		log.Printf("[DEBUG] Filename '%s' in URL with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List))
+		//log.Printf("[DEBUG] Filename '%s' in URL with reserved category name: %s. Listlength: %d", filename[0], namespace, len(fileResponse.List))
 
 		// Load from Github repo https://github.com/Shuffle/standards
 		filenameFound := false
@@ -650,7 +650,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
 						continue
 					}
 
-					log.Printf("\n\n\n[DEBUG] Decoded file '%s' with content:\n%s\n\n\n", *item.Path, string(decoded))
+					//log.Printf("[DEBUG] Decoded Github file '%s' with content:\n%s", *item.Path, string(decoded))
 
 					timeNow := time.Now().Unix()
 					fileId := "file_"+uuid.NewV4().String()
@@ -698,7 +698,7 @@ func HandleGetFileNamespace(resp http.ResponseWriter, request *http.Request) {
 						continue
 					}
 
-					log.Printf("\n\n[DEBUG] Uploaded file %s with ID %s in category %#v\n\n", file.Filename, fileId, namespace)
+					log.Printf("[DEBUG] Uploaded file %#v with ID %s in category %#v", file.Filename, fileId, namespace)
 
 					fileResponse.List = append(fileResponse.List, BaseFile{
 						Name: file.Filename,
@@ -1641,7 +1641,7 @@ func HandleCreateFile(resp http.ResponseWriter, request *http.Request) {
 		orgId := user.ActiveOrg.Id
 		files, err := FindSimilarFilename(ctx, curfile.Filename, orgId)
 		if err != nil {
-			log.Printf("[ERROR] Failed finding similar files: %s", err)
+			//log.Printf("[ERROR] Couldn't find any similar files: %s", err)
 		} else {
 
 			for _, item := range files {

go.mod

@@ -1,11 +1,9 @@
 module github.com/shuffle/shuffle-shared
 
-// Keep on 1.11 until AppEngine supports 1.17 or higher
-
 go 1.11
 
 //replace github.com/frikky/kin-openapi => ../kin-openapi
-replace github.com/shuffle/opensearch-go => ../opensearch-go
+//replace github.com/shuffle/opensearch-go => ../opensearch-go
 
 require (
 	cloud.google.com/go/datastore v1.4.0
@@ -16,7 +14,7 @@ require (
 	github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874
 	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013
 	github.com/frikky/kin-openapi v0.41.0
-	github.com/frikky/schemaless v0.0.9
+	github.com/frikky/schemaless v0.0.11
 	github.com/go-git/go-billy/v5 v5.5.0
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/google/go-github/v28 v28.1.1

kms.go

@@ -23,7 +23,8 @@ import (
 	openai "github.com/sashabaranov/go-openai"
 )
 
-var model = "gpt-4-turbo-preview"
+//var model = "gpt-4-turbo-preview"
+var model = "gpt-4o"
 
 func GetKmsCache(ctx context.Context, auth AppAuthenticationStorage, key string) (string, error) {
 	//log.Printf("\n\n[DEBUG] Getting KMS cache for key %s\n\n", key)

shared.go

@@ -7080,7 +7080,9 @@ func HandleGetUsers(resp http.ResponseWriter, request *http.Request) {
 		item.Executions = ExecutionInfo{}
 		item.Limits = UserLimits{}
 		item.PrivateApps = []WorkflowApp{}
-		item.MFA = MFAInfo{}
+		item.MFA = MFAInfo{
+			Active: item.MFA.Active,
+		}
 
 		if !user.SupportAccess {
 			item.LoginInfo = []LoginInfo{}