Merge pull request #34 from silinternational/develop

Release 3.4.0 - Google Sheets destination

README.md

@@ -252,6 +252,69 @@ Note: `Source` fields should be adjusted to fit the actual source adapter.
 
 Configurations for `BatchSize`, `BatchDelaySeconds`, `DisableAdd`, `DisableUpdate`, and `DisableDelete` are all optional with defaults as shown in example.
 
+### Google Sheets
+The Google Sheets destination creates a copy of the source data in a Google Sheets
+document.
+
+If any of the disable options, DisableAdd, DisableDelete, or DisableUpdate are
+set to true, no sync will be performed.
+
+There must be at least two rows in the sheet to begin with. The first row must
+be pre-filled with field names. The second row must be present, but will be
+ignored and may be overwritten.
+
+The entire sheet will be overwritten with new data on every sync
+
+If not specified in the configuration, the sheet updated is "Sheet1"
+
+Example config:
+```json
+{
+  "Destination": {
+    "Type": "GoogleSheets",
+    "ExtraJSON": {
+      "DelegatedAdminEmail": "[email protected]",
+      "GoogleAuth": {
+        "type": "service_account",
+        "project_id": "abc-theme-123456",
+        "private_key_id": "abc123",
+        "private_key": "-----BEGIN PRIVATE KEY-----\nMIIabc...\nabc...\n...xyz\n-----END PRIVATE KEY-----\n",
+        "client_email": "[email protected]",
+        "client_id": "123456789012345678901",
+        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+        "token_uri": "https://oauth2.googleapis.com/token",
+        "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+        "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/my-sync-bot%40abc-theme-123456.iam.gserviceaccount.com"
+      }            
+    }
+  },
+  "AttributeMap": [
+    {
+      "Source": "email",
+      "Destination": "email"
+    },
+    {
+      "Source": "employee_id",
+      "Destination": "employee_id"
+    }
+  ],
+  "SyncSets": [
+    {
+      "Name": "Sync from Xyz API to Google Sheets",
+      "Source": {
+        "Paths": ["/user"]
+      },
+      "Destination": {
+        "SheetID": "putAnActualSheetIDHerejD70xAjqPnOCHlDK3YomH",
+        "SheetName": "Sheet2"
+      }
+    }
+  ]
+}
+```
+
+Note: `Source` fields should be adjusted to fit the actual source adapter.
+
 ### Google Users
 This destination can update User records in the Google Directory. The compare
 attribute is `primaryEmail`. A limited subset of user properties are available

cmd/syncpeeps.go

@@ -17,8 +17,7 @@ import (
 func main() {
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
-	now := time.Now().UTC()
-	log.Printf("Personnel sync started at %s", now.Format(time.RFC1123Z))
+	log.Printf("Personnel sync started at %s", time.Now().UTC().Format(time.RFC1123Z))
 
 	appConfig, err := personnel_sync.LoadConfig("")
 	if err != nil {
@@ -46,6 +45,8 @@ func main() {
 		destination, err = googledest.NewGoogleContactsDestination(appConfig.Destination)
 	case personnel_sync.DestinationTypeGoogleGroups:
 		destination, err = googledest.NewGoogleGroupsDestination(appConfig.Destination)
+	case personnel_sync.DestinationTypeGoogleSheets:
+		destination, err = googledest.NewGoogleSheetsDestination(appConfig.Destination)
 	case personnel_sync.DestinationTypeGoogleUsers:
 		destination, err = googledest.NewGoogleUsersDestination(appConfig.Destination)
 	case personnel_sync.DestinationTypeWebHelpDesk:
@@ -89,5 +90,6 @@ func main() {
 		}
 	}
 
+	log.Printf("Personnel sync completed at %s", time.Now().UTC().Format(time.RFC1123Z))
 	os.Exit(0)
 }

domain.go

@@ -15,6 +15,7 @@ const (
 	DefaultVerbosity              = 5
 	DestinationTypeGoogleContacts = "GoogleContacts"
 	DestinationTypeGoogleGroups   = "GoogleGroups"
+	DestinationTypeGoogleSheets   = "GoogleSheets"
 	DestinationTypeGoogleUsers    = "GoogleUsers"
 	DestinationTypeWebHelpDesk    = "WebHelpDesk"
 	SourceTypeRestAPI             = "RestAPI"
@@ -246,6 +247,8 @@ func SyncPeople(source Source, destination Destination, config AppConfig) Change
 	go processEventLog(eventLog)
 
 	results := destination.ApplyChangeSet(changeSet, eventLog)
+
+	time.Sleep(time.Millisecond * 10)
 	close(eventLog)
 
 	return results

go.mod

@@ -2,6 +2,8 @@ module github.com/silinternational/personnel-sync/v3
 
 go 1.14
 
+replace github.com/silinternational/personnel-sync/v3 => ./
+
 require (
 	github.com/Jeffail/gabs v1.4.0
 	github.com/aws/aws-lambda-go v1.16.0

googledest/admin.go

@@ -11,19 +11,6 @@ import (
 	admin "google.golang.org/api/admin/directory/v1"
 )
 
-type GoogleAuth struct {
-	Type                    string `json:"type"`
-	ProjectID               string `json:"project_id"`
-	PrivateKeyID            string `json:"private_key_id"`
-	PrivateKey              string `json:"private_key"`
-	ClientEmail             string `json:"client_email"`
-	ClientID                string `json:"client_id"`
-	AuthURI                 string `json:"auth_uri"`
-	TokenURI                string `json:"token_uri"`
-	AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`
-	ClientX509CertURL       string `json:"client_x509_cert_url"`
-}
-
 // initGoogleAdminService authenticates with the Google API and returns an admin.Service
 //  that has the requested scopes
 func initGoogleAdminService(auth GoogleAuth, adminEmail string, scopes ...string) (admin.Service, error) {

googledest/contacts.go

@@ -35,18 +35,12 @@ const (
 	contactFieldNotes          = "notes"
 )
 
-type GoogleContactsConfig struct {
-	DelegatedAdminEmail string
-	Domain              string
-	GoogleAuth          GoogleAuth
-	BatchSize           int
-	BatchDelaySeconds   int
-}
-
 type GoogleContacts struct {
-	DestinationConfig    personnel_sync.DestinationConfig
-	GoogleContactsConfig GoogleContactsConfig
-	Client               http.Client
+	BatchSize         int
+	BatchDelaySeconds int
+	DestinationConfig personnel_sync.DestinationConfig
+	GoogleConfig      GoogleConfig
+	Client            http.Client
 }
 
 type Entries struct {
@@ -116,19 +110,18 @@ func NewGoogleContactsDestination(destinationConfig personnel_sync.DestinationCo
 	}
 
 	var googleContacts GoogleContacts
-	// Unmarshal ExtraJSON into GoogleContactsConfig struct
-	err := json.Unmarshal(destinationConfig.ExtraJSON, &googleContacts.GoogleContactsConfig)
+	// Unmarshal ExtraJSON into GoogleConfig struct
+	err := json.Unmarshal(destinationConfig.ExtraJSON, &googleContacts.GoogleConfig)
 	if err != nil {
 		return &GoogleContacts{}, err
 	}
 
 	// Defaults
-	config := &googleContacts.GoogleContactsConfig
-	if config.BatchSize <= 0 {
-		config.BatchSize = DefaultBatchSize
+	if googleContacts.BatchSize <= 0 {
+		googleContacts.BatchSize = DefaultBatchSize
 	}
-	if config.BatchDelaySeconds <= 0 {
-		config.BatchDelaySeconds = DefaultBatchDelaySeconds
+	if googleContacts.BatchDelaySeconds <= 0 {
+		googleContacts.BatchDelaySeconds = DefaultBatchDelaySeconds
 	}
 
 	googleContacts.DestinationConfig = destinationConfig
@@ -156,7 +149,7 @@ func (g *GoogleContacts) ForSet(syncSetJson json.RawMessage) error {
 // ListUsers returns all users (contacts) in the destination
 func (g *GoogleContacts) ListUsers() ([]personnel_sync.Person, error) {
 	href := fmt.Sprintf("https://www.google.com/m8/feeds/contacts/%s/full?max-results=%d",
-		g.GoogleContactsConfig.Domain, MaxQuerySize)
+		g.GoogleConfig.Domain, MaxQuerySize)
 	body, err := g.httpRequest(http.MethodGet, href, "", map[string]string{})
 	if err != nil {
 		return []personnel_sync.Person{}, fmt.Errorf("failed to retrieve user list: %s", err)
@@ -182,8 +175,8 @@ func (g *GoogleContacts) ApplyChangeSet(
 	var results personnel_sync.ChangeResults
 	var wg sync.WaitGroup
 
-	batchTimer := personnel_sync.NewBatchTimer(g.GoogleContactsConfig.BatchSize,
-		g.GoogleContactsConfig.BatchDelaySeconds)
+	batchTimer := personnel_sync.NewBatchTimer(g.BatchSize,
+		g.BatchDelaySeconds)
 
 	if g.DestinationConfig.DisableAdd {
 		log.Println("Contact creation is disabled.")
@@ -321,7 +314,7 @@ func (g *GoogleContacts) addContact(
 
 	defer wg.Done()
 
-	href := "https://www.google.com/m8/feeds/contacts/" + g.GoogleContactsConfig.Domain + "/full"
+	href := "https://www.google.com/m8/feeds/contacts/" + g.GoogleConfig.Domain + "/full"
 	body := g.createBody(person)
 	headers := map[string]string{"Content-Type": "application/atom+xml"}
 	if _, err := g.httpRequest(http.MethodPost, href, body, headers); err != nil {
@@ -343,7 +336,7 @@ func (g *GoogleContacts) addContact(
 //  Authentication requires an email address that matches an actual GMail user (e.g. a machine account)
 //  that has appropriate access privileges
 func (g *GoogleContacts) initGoogleClient() error {
-	googleAuthJson, err := json.Marshal(g.GoogleContactsConfig.GoogleAuth)
+	googleAuthJson, err := json.Marshal(g.GoogleConfig.GoogleAuth)
 	if err != nil {
 		return fmt.Errorf("unable to marshal google auth data into json, error: %s", err)
 	}
@@ -353,7 +346,7 @@ func (g *GoogleContacts) initGoogleClient() error {
 		return fmt.Errorf("unable to parse client secret file to config: %s", err)
 	}
 
-	config.Subject = g.GoogleContactsConfig.DelegatedAdminEmail
+	config.Subject = g.GoogleConfig.DelegatedAdminEmail
 	g.Client = *config.Client(context.Background())
 
 	return nil

googledest/contacts_test.go

@@ -46,7 +46,9 @@ func TestNewGoogleContactsDestination(t *testing.T) {
 				ExtraJSON:     json.RawMessage(extraJSON),
 			},
 			want: GoogleContacts{
-				GoogleContactsConfig: GoogleContactsConfig{
+				BatchSize:         5,
+				BatchDelaySeconds: 1,
+				GoogleConfig: GoogleConfig{
 					DelegatedAdminEmail: "[email protected]",
 					Domain:              "example.com",
 					GoogleAuth: GoogleAuth{
@@ -61,8 +63,6 @@ func TestNewGoogleContactsDestination(t *testing.T) {
 						AuthProviderX509CertURL: "https://www.googleapis.com/oauth2/v1/certs",
 						ClientX509CertURL:       "https://www.googleapis.com/robot/v1/metadata/x509/my-sync-bot%40abc-theme-123456.iam.gserviceaccount.com",
 					},
-					BatchSize:         5,
-					BatchDelaySeconds: 1,
 				},
 			},
 			wantErr: false,
@@ -92,8 +92,8 @@ func TestNewGoogleContactsDestination(t *testing.T) {
 				return
 			}
 			g := got.(*GoogleContacts)
-			if !reflect.DeepEqual(g.GoogleContactsConfig, tt.want.GoogleContactsConfig) {
-				t.Errorf("incorrect GoogleContactsConfig \ngot: %#v, \nwant: %#v", got, tt.want)
+			if !reflect.DeepEqual(g.GoogleConfig, tt.want.GoogleConfig) {
+				t.Errorf("incorrect GoogleConfig \ngot: %#v, \nwant: %#v", got, tt.want)
 			}
 		})
 	}

googledest/googledest.go

@@ -0,0 +1,23 @@
+package googledest
+
+const DefaultBatchSize = 10
+const DefaultBatchDelaySeconds = 3
+
+type GoogleConfig struct {
+	DelegatedAdminEmail string
+	Domain              string
+	GoogleAuth          GoogleAuth
+}
+
+type GoogleAuth struct {
+	Type                    string `json:"type"`
+	ProjectID               string `json:"project_id"`
+	PrivateKeyID            string `json:"private_key_id"`
+	PrivateKey              string `json:"private_key"`
+	ClientEmail             string `json:"client_email"`
+	ClientID                string `json:"client_id"`
+	AuthURI                 string `json:"auth_uri"`
+	TokenURI                string `json:"token_uri"`
+	AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"`
+	ClientX509CertURL       string `json:"client_x509_cert_url"`
+}

googledest/google_groups.go → googledest/groups.go

@@ -13,24 +13,17 @@ import (
 	"golang.org/x/net/context"
 )
 
-const DefaultBatchSize = 10
-const DefaultBatchDelaySeconds = 3
 const RoleMember = "MEMBER"
 const RoleOwner = "OWNER"
 const RoleManager = "MANAGER"
 
-type GoogleGroupsConfig struct {
-	DelegatedAdminEmail string
-	GoogleAuth          GoogleAuth
-}
-
 type GoogleGroups struct {
-	DestinationConfig  personnel_sync.DestinationConfig
-	GoogleGroupsConfig GoogleGroupsConfig
-	AdminService       admin.Service
-	GroupSyncSet       GroupSyncSet
-	BatchSize          int
-	BatchDelaySeconds  int
+	DestinationConfig personnel_sync.DestinationConfig
+	GoogleConfig      GoogleConfig
+	AdminService      admin.Service
+	GroupSyncSet      GroupSyncSet
+	BatchSize         int
+	BatchDelaySeconds int
 }
 
 type GroupSyncSet struct {
@@ -48,7 +41,7 @@ type GroupSyncSet struct {
 func NewGoogleGroupsDestination(destinationConfig personnel_sync.DestinationConfig) (personnel_sync.Destination, error) {
 	var googleGroups GoogleGroups
 	// Unmarshal ExtraJSON into GoogleGroupsConfig struct
-	err := json.Unmarshal(destinationConfig.ExtraJSON, &googleGroups.GoogleGroupsConfig)
+	err := json.Unmarshal(destinationConfig.ExtraJSON, &googleGroups.GoogleConfig)
 	if err != nil {
 		return &GoogleGroups{}, err
 	}
@@ -63,8 +56,8 @@ func NewGoogleGroupsDestination(destinationConfig personnel_sync.DestinationConf
 
 	// Initialize AdminService object
 	googleGroups.AdminService, err = initGoogleAdminService(
-		googleGroups.GoogleGroupsConfig.GoogleAuth,
-		googleGroups.GoogleGroupsConfig.DelegatedAdminEmail,
+		googleGroups.GoogleConfig.GoogleAuth,
+		googleGroups.GoogleConfig.DelegatedAdminEmail,
 		admin.AdminDirectoryGroupScope,
 		admin.AdminDirectoryGroupMemberScope,
 	)

googledest/google_groups_test.go → googledest/groups_test.go

@@ -75,7 +75,7 @@ func TestGoogleGroups_ListUsers(t *testing.T) {
 
 	type fields struct {
 		DestinationConfig  personnel_sync.DestinationConfig
-		GoogleGroupsConfig GoogleGroupsConfig
+		GoogleGroupsConfig GoogleConfig
 		AdminService       admin.Service
 	}
 	tests := []struct {