diff --git a/Gopkg.lock b/Gopkg.lock index a565b08..e9c7e17 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -236,6 +236,7 @@ "golang.org/x/net/context", "golang.org/x/oauth2/google", "google.golang.org/api/admin/directory/v1", + "google.golang.org/api/option", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/README.md b/README.md index 6363bfd..f6360d1 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,8 @@ of the destination configuration required for Google Groups: "Destination": { "Type": "GoogleGroups", "ExtraJSON": { - "BatchSizePerMinute": 50, + "BatchSize": 10, + "BatchDelaySeconds": 3, "DelegatedAdminEmail": "delegated-admin@domain.com", "GoogleAuth": { "type": "service_account", @@ -123,13 +124,18 @@ of the destination configuration required for Google Groups: "GroupEmail": "group1@groups.domain.com", "Owners": ["person_a@domain.com","person_b@domain.com"], "Managers": ["another_person@domain.com", "yet-another-person@domain.com"], - "ExtraOwners": ["google-admin@domain.com"] + "ExtraOwners": ["google-admin@domain.com"], + "DisableAdd": false, + "DisableUpdate": false, + "DisableDelete": false } } ] } ``` +Configurations for `BatchSize`, `BatchDelaySeconds`, `DisableAdd`, `DisableUpdate`, and `DisableDelete` are all option with defaults as shown in example. + ### Google Users This destination can update User records in the Google Directory. The compare attribute is `primaryEmail`. A limited subset of user properties are available @@ -161,7 +167,8 @@ Following is an example configuration listing all available fields: "Destination": { "Type": "GoogleUsers", "ExtraJSON": { - "BatchSizePerMinute": 50, + "BatchSize": 10, + "BatchDelaySeconds": 3, "DelegatedAdminEmail": "admin@example.com", "GoogleAuth": { "type": "service_account", @@ -317,13 +324,14 @@ as the `DelegatedAdminEmail` value under `Destination`/`ExtraJSON`. "Username": "syncuser", "Password": "apitoken", "ListClientsPageLimit": 100, - "BatchSizePerMinute": 50 + "BatchSize": 50, + "BatchDelaySeconds": 60 } } } ``` -`ListClientsPageLimit` and `BatchSizePerMinute` are optional. Their defaults are as shown in the example config. +`ListClientsPageLimit`, `BatchSize` and `BatchDelaySeconds` are optional. Their defaults are as shown in the example config. ### Exporting logs from CloudWatch @@ -346,3 +354,4 @@ stream. Note the single quotes around the log stream name to prevent the shell from interpreting the `$` character. `--output text` can be changed to `--output json` if desired. Timestamps are available if needed, but omitted in this example by the `--query` string. + diff --git a/config.example.json b/config.example.json index 9726a62..6afc2f6 100644 --- a/config.example.json +++ b/config.example.json @@ -19,7 +19,8 @@ "Destination": { "Type": "GoogleGroups", "ExtraJSON": { - "BatchSizePerMinute": 50, + "BatchSize": 10, + "BatchDelaySeconds": 3, "DelegatedAdminEmail": "delegated-admin@domain.com", "GoogleAuth": { "type": "service_account", @@ -76,7 +77,10 @@ ], "ExtraMembers": [ "not-in-report@domain.com" - ] + ], + "DisableAdd": false, + "DisableUpdate": false, + "DisableDelete": false } } ] diff --git a/googledest/google_groups.go b/googledest/google_groups.go index 3112e79..8f8ad66 100644 --- a/googledest/google_groups.go +++ b/googledest/google_groups.go @@ -13,7 +13,8 @@ import ( "golang.org/x/net/context" ) -const DefaultBatchSizePerMinute = 50 +const DefaultBatchSize = 10 +const DefaultBatchDelaySeconds = 3 const RoleMember = "MEMBER" const RoleOwner = "OWNER" const RoleManager = "MANAGER" @@ -28,7 +29,8 @@ type GoogleGroups struct { GoogleGroupsConfig GoogleGroupsConfig AdminService admin.Service GroupSyncSet GroupSyncSet - BatchSizePerMinute int + BatchSize int + BatchDelaySeconds int } type GroupSyncSet struct { @@ -38,6 +40,9 @@ type GroupSyncSet struct { Managers []string ExtraManagers []string ExtraMembers []string + DisableAdd bool + DisableUpdate bool + DisableDelete bool } func NewGoogleGroupsDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) { @@ -49,8 +54,11 @@ func NewGoogleGroupsDestination(destinationConfig personnel_sync.DestinationConf } // Defaults - if googleGroups.BatchSizePerMinute <= 0 { - googleGroups.BatchSizePerMinute = DefaultBatchSizePerMinute + if googleGroups.BatchSize <= 0 { + googleGroups.BatchSize = DefaultBatchSize + } + if googleGroups.BatchDelaySeconds <= 0 { + googleGroups.BatchDelaySeconds = DefaultBatchDelaySeconds } // Initialize AdminService object @@ -160,29 +168,32 @@ func (g *GoogleGroups) ApplyChangeSet( } // One minute per batch - batchTimer := personnel_sync.NewBatchTimer(g.BatchSizePerMinute, int(60)) + batchTimer := personnel_sync.NewBatchTimer(g.BatchSize, g.BatchDelaySeconds) - for email, role := range toBeCreated { - wg.Add(1) - go g.addMember(email, role, &results.Created, &wg, eventLog) - batchTimer.WaitOnBatch() + if !g.GroupSyncSet.DisableAdd { + for email, role := range toBeCreated { + wg.Add(1) + go g.addMember(email, role, &results.Created, &wg, eventLog) + batchTimer.WaitOnBatch() + } } - for _, dp := range changes.Delete { - // Do not delete ExtraManagers, ExtraOwners, or ExtraMembers - if isExtraManager, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraManagers); isExtraManager { - continue - } - if isExtraOwner, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraOwners); isExtraOwner { - continue - } - if isExtraMember, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraMembers); isExtraMember { - continue + if !g.GroupSyncSet.DisableDelete { + for _, dp := range changes.Delete { + // Do not delete ExtraManagers, ExtraOwners, or ExtraMembers + if isExtraManager, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraManagers); isExtraManager { + continue + } + if isExtraOwner, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraOwners); isExtraOwner { + continue + } + if isExtraMember, _ := personnel_sync.InArray(dp.CompareValue, g.GroupSyncSet.ExtraMembers); isExtraMember { + continue + } + wg.Add(1) + go g.removeMember(dp.CompareValue, &results.Deleted, &wg, eventLog) + batchTimer.WaitOnBatch() } - - wg.Add(1) - go g.removeMember(dp.CompareValue, &results.Deleted, &wg, eventLog) - batchTimer.WaitOnBatch() } wg.Wait() diff --git a/googledest/users.go b/googledest/users.go index f478441..fdbcb8b 100644 --- a/googledest/users.go +++ b/googledest/users.go @@ -20,9 +20,10 @@ type GoogleUsersConfig struct { } type GoogleUsers struct { - GoogleUsersConfig GoogleUsersConfig - AdminService admin.Service - BatchSizePerMinute int + GoogleUsersConfig GoogleUsersConfig + AdminService admin.Service + BatchSize int + BatchDelaySeconds int } func NewGoogleUsersDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) { @@ -34,8 +35,11 @@ func NewGoogleUsersDestination(destinationConfig personnel_sync.DestinationConfi } // Defaults - if googleUsers.BatchSizePerMinute <= 0 { - googleUsers.BatchSizePerMinute = DefaultBatchSizePerMinute + if googleUsers.BatchSize <= 0 { + googleUsers.BatchSize = DefaultBatchSize + } + if googleUsers.BatchDelaySeconds <= 0 { + googleUsers.BatchDelaySeconds = DefaultBatchDelaySeconds } // Initialize AdminService object @@ -173,7 +177,7 @@ func (g *GoogleUsers) ApplyChangeSet( var wg sync.WaitGroup // One minute per batch - batchTimer := personnel_sync.NewBatchTimer(g.BatchSizePerMinute, int(60)) + batchTimer := personnel_sync.NewBatchTimer(g.BatchSize, g.BatchDelaySeconds) for _, toUpdate := range changes.Update { wg.Add(1) diff --git a/types.go b/types.go index 85b7925..4140900 100644 --- a/types.go +++ b/types.go @@ -22,8 +22,11 @@ type SourceConfig struct { } type DestinationConfig struct { - Type string - ExtraJSON json.RawMessage + Type string + ExtraJSON json.RawMessage + DisableAdd bool + DisableUpdate bool + DisableDelete bool } const ( diff --git a/webhelpdesk/webhelpdesk.go b/webhelpdesk/webhelpdesk.go index b6accc2..02f06cc 100644 --- a/webhelpdesk/webhelpdesk.go +++ b/webhelpdesk/webhelpdesk.go @@ -14,7 +14,8 @@ import ( personnel_sync "github.com/silinternational/personnel-sync" ) -const DefaultBatchSizePerMinute = 50 +const DefaultBatchSize = 50 +const DefaultBatchDelaySeconds = 60 const DefaultListClientsPageLimit = 100 const ClientsAPIPath = "/ra/Clients" @@ -32,7 +33,8 @@ type WebHelpDesk struct { Username string Password string ListClientsPageLimit int - BatchSizePerMinute int + BatchSize int + BatchDelaySeconds int } func NewWebHelpDeskDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) { @@ -44,8 +46,11 @@ func NewWebHelpDeskDestination(destinationConfig personnel_sync.DestinationConfi } // Set defaults for batch size per minute and page limit if not provided in ExtraJSON - if webHelpDesk.BatchSizePerMinute <= 0 { - webHelpDesk.BatchSizePerMinute = DefaultBatchSizePerMinute + if webHelpDesk.BatchSize <= 0 { + webHelpDesk.BatchSize = DefaultBatchSize + } + if webHelpDesk.BatchDelaySeconds <= 0 { + webHelpDesk.BatchDelaySeconds = DefaultBatchDelaySeconds } if webHelpDesk.ListClientsPageLimit == 0 { @@ -121,7 +126,7 @@ func (w *WebHelpDesk) ApplyChangeSet( var wg sync.WaitGroup // One minute per batch - batchTimer := personnel_sync.NewBatchTimer(w.BatchSizePerMinute, int(60)) + batchTimer := personnel_sync.NewBatchTimer(w.BatchSize, w.BatchDelaySeconds) for _, cp := range changes.Create { wg.Add(1)