Merge pull request #27 from silinternational/develop

Release "Disable*" parameters for Google Groups

README.md

@@ -80,7 +80,8 @@ of the destination configuration required for Google Groups:
   "Destination": {
     "Type": "GoogleGroups",
     "ExtraJSON": {
-      "BatchSizePerMinute": 50,
+      "BatchSize": 10,
+      "BatchDelaySeconds": 3,
       "DelegatedAdminEmail": "[email protected]",
       "GoogleAuth": {
         "type": "service_account",
@@ -123,13 +124,18 @@ of the destination configuration required for Google Groups:
           "GroupEmail": "[email protected]",
           "Owners": ["[email protected]","[email protected]"],
           "Managers": ["[email protected]", "[email protected]"],
-          "ExtraOwners": ["[email protected]"]
+          "ExtraOwners": ["[email protected]"],
+          "DisableAdd": false,
+          "DisableUpdate": false,
+          "DisableDelete": false
       }
     }
   ]
 }
 ```
 
+Configurations for `BatchSize`, `BatchDelaySeconds`, `DisableAdd`, `DisableUpdate`, and `DisableDelete` are all option with defaults as shown in example.
+
 ### Google Users
 This destination can update User records in the Google Directory. The compare
 attribute is `primaryEmail`. A limited subset of user properties are available
@@ -161,7 +167,8 @@ Following is an example configuration listing all available fields:
   "Destination": {
     "Type": "GoogleUsers",
     "ExtraJSON": {
-      "BatchSizePerMinute": 50,
+      "BatchSize": 10,
+      "BatchDelaySeconds": 3,
       "DelegatedAdminEmail": "[email protected]",
       "GoogleAuth": {
         "type": "service_account",
@@ -317,13 +324,14 @@ as the `DelegatedAdminEmail` value under `Destination`/`ExtraJSON`.
       "Username": "syncuser",
       "Password": "apitoken",
       "ListClientsPageLimit": 100,
-      "BatchSizePerMinute": 50
+      "BatchSize": 50,
+      "BatchDelaySeconds": 60
     }
   }
 }
 ```
 
-`ListClientsPageLimit` and `BatchSizePerMinute` are optional. Their defaults are as shown in the example config.
+`ListClientsPageLimit`, `BatchSize` and `BatchDelaySeconds` are optional. Their defaults are as shown in the example config.
 
 ### Exporting logs from CloudWatch
 
@@ -346,3 +354,4 @@ stream. Note the single quotes around the log stream name to prevent the shell
 from interpreting the `$` character. `--output text` can be changed to 
 `--output json` if desired. Timestamps are available if needed, but omitted
 in this example by the `--query` string.
+

config.example.json

@@ -19,7 +19,8 @@
   "Destination": {
     "Type": "GoogleGroups",
     "ExtraJSON": {
-      "BatchSizePerMinute": 50,
+      "BatchSize": 10,
+      "BatchDelaySeconds": 3,
       "DelegatedAdminEmail": "[email protected]",
       "GoogleAuth": {
         "type": "service_account",
@@ -76,7 +77,10 @@
         ],
         "ExtraMembers":  [
           "[email protected]"
-        ]
+        ],
+        "DisableAdd": false,
+        "DisableUpdate": false,
+        "DisableDelete": false
       }
     }
   ]

googledest/google_groups.go

@@ -13,7 +13,8 @@ import (
 	"golang.org/x/net/context"
 )
 
-const DefaultBatchSizePerMinute = 50
+const DefaultBatchSize = 10
+const DefaultBatchDelaySeconds = 3
 const RoleMember = "MEMBER"
 const RoleOwner = "OWNER"
 const RoleManager = "MANAGER"
@@ -28,7 +29,8 @@ type GoogleGroups struct {
 	GoogleGroupsConfig GoogleGroupsConfig
 	AdminService       admin.Service
 	GroupSyncSet       GroupSyncSet
-	BatchSizePerMinute int
+	BatchSize          int
+	BatchDelaySeconds  int
 }
 
 type GroupSyncSet struct {
@@ -38,6 +40,9 @@ type GroupSyncSet struct {
 	Managers      []string
 	ExtraManagers []string
 	ExtraMembers  []string
+	DisableAdd    bool
+	DisableUpdate bool
+	DisableDelete bool
 }
 
 func NewGoogleGroupsDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) {
@@ -49,8 +54,11 @@ func NewGoogleGroupsDestination(destinationConfig personnel_sync.DestinationConf
 	}
 
 	// Defaults
-	if googleGroups.BatchSizePerMinute <= 0 {
-		googleGroups.BatchSizePerMinute = DefaultBatchSizePerMinute
+	if googleGroups.BatchSize <= 0 {
+		googleGroups.BatchSize = DefaultBatchSize
+	}
+	if googleGroups.BatchDelaySeconds <= 0 {
+		googleGroups.BatchDelaySeconds = DefaultBatchDelaySeconds
 	}
 
 	// Initialize AdminService object
@@ -160,29 +168,32 @@ func (g *GoogleGroups) ApplyChangeSet(
 	}
 
 	// One minute per batch
-	batchTimer := personnel_sync.NewBatchTimer(g.BatchSizePerMinute, int(60))
+	batchTimer := personnel_sync.NewBatchTimer(g.BatchSize, g.BatchDelaySeconds)
 
-	for email, role := range toBeCreated {
-		wg.Add(1)
-		go g.addMember(email, role, &results.Created, &wg, eventLog)
-		batchTimer.WaitOnBatch()
+	if !g.GroupSyncSet.DisableAdd {
+		for email, role := range toBeCreated {
+			wg.Add(1)
+			go g.addMember(email, role, &results.Created, &wg, eventLog)
+			batchTimer.WaitOnBatch()
+		}
 	}
 
-	for _, dp := range changes.Delete {
-		// Do not delete ExtraManagers, ExtraOwners, or ExtraMembers
-		if isExtraManager, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraManagers); isExtraManager {
-			continue
-		}
-		if isExtraOwner, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraOwners); isExtraOwner {
-			continue
-		}
-		if isExtraMember, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraMembers); isExtraMember {
-			continue
+	if !g.GroupSyncSet.DisableDelete {
+		for _, dp := range changes.Delete {
+			// Do not delete ExtraManagers, ExtraOwners, or ExtraMembers
+			if isExtraManager, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraManagers); isExtraManager {
+				continue
+			}
+			if isExtraOwner, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraOwners); isExtraOwner {
+				continue
+			}
+			if isExtraMember, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraMembers); isExtraMember {
+				continue
+			}
+			wg.Add(1)
+			go g.removeMember(dp.CompareValue, &results.Deleted, &wg, eventLog)
+			batchTimer.WaitOnBatch()
 		}
-
-		wg.Add(1)
-		go g.removeMember(dp.CompareValue, &results.Deleted, &wg, eventLog)
-		batchTimer.WaitOnBatch()
 	}
 
 	wg.Wait()

googledest/users.go

@@ -20,9 +20,10 @@ type GoogleUsersConfig struct {
 }
 
 type GoogleUsers struct {
-	GoogleUsersConfig  GoogleUsersConfig
-	AdminService       admin.Service
-	BatchSizePerMinute int
+	GoogleUsersConfig GoogleUsersConfig
+	AdminService      admin.Service
+	BatchSize         int
+	BatchDelaySeconds int
 }
 
 func NewGoogleUsersDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) {
@@ -34,8 +35,11 @@ func NewGoogleUsersDestination(destinationConfig personnel_sync.DestinationConfi
 	}
 
 	// Defaults
-	if googleUsers.BatchSizePerMinute <= 0 {
-		googleUsers.BatchSizePerMinute = DefaultBatchSizePerMinute
+	if googleUsers.BatchSize <= 0 {
+		googleUsers.BatchSize = DefaultBatchSize
+	}
+	if googleUsers.BatchDelaySeconds <= 0 {
+		googleUsers.BatchDelaySeconds = DefaultBatchDelaySeconds
 	}
 
 	// Initialize AdminService object
@@ -173,7 +177,7 @@ func (g *GoogleUsers) ApplyChangeSet(
 	var wg sync.WaitGroup
 
 	// One minute per batch
-	batchTimer := personnel_sync.NewBatchTimer(g.BatchSizePerMinute, int(60))
+	batchTimer := personnel_sync.NewBatchTimer(g.BatchSize, g.BatchDelaySeconds)
 
 	for _, toUpdate := range changes.Update {
 		wg.Add(1)

types.go

@@ -22,8 +22,11 @@ type SourceConfig struct {
 }
 
 type DestinationConfig struct {
-	Type      string
-	ExtraJSON json.RawMessage
+	Type          string
+	ExtraJSON     json.RawMessage
+	DisableAdd    bool
+	DisableUpdate bool
+	DisableDelete bool
 }
 
 const (

webhelpdesk/webhelpdesk.go

@@ -14,7 +14,8 @@ import (
 	personnel_sync "github.com/silinternational/personnel-sync"
 )
 
-const DefaultBatchSizePerMinute = 50
+const DefaultBatchSize = 50
+const DefaultBatchDelaySeconds = 60
 const DefaultListClientsPageLimit = 100
 const ClientsAPIPath = "/ra/Clients"
 
@@ -32,7 +33,8 @@ type WebHelpDesk struct {
 	Username             string
 	Password             string
 	ListClientsPageLimit int
-	BatchSizePerMinute   int
+	BatchSize            int
+	BatchDelaySeconds    int
 }
 
 func NewWebHelpDeskDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) {
@@ -44,8 +46,11 @@ func NewWebHelpDeskDestination(destinationConfig personnel_sync.DestinationConfi
 	}
 
 	// Set defaults for batch size per minute and page limit if not provided in ExtraJSON
-	if webHelpDesk.BatchSizePerMinute <= 0 {
-		webHelpDesk.BatchSizePerMinute = DefaultBatchSizePerMinute
+	if webHelpDesk.BatchSize <= 0 {
+		webHelpDesk.BatchSize = DefaultBatchSize
+	}
+	if webHelpDesk.BatchDelaySeconds <= 0 {
+		webHelpDesk.BatchDelaySeconds = DefaultBatchDelaySeconds
 	}
 
 	if webHelpDesk.ListClientsPageLimit == 0 {
@@ -121,7 +126,7 @@ func (w *WebHelpDesk) ApplyChangeSet(
 	var wg sync.WaitGroup
 
 	// One minute per batch
-	batchTimer := personnel_sync.NewBatchTimer(w.BatchSizePerMinute, int(60))
+	batchTimer := personnel_sync.NewBatchTimer(w.BatchSize, w.BatchDelaySeconds)
 
 	for _, cp := range changes.Create {
 		wg.Add(1)