Added support for external Google API OAuth2 client credentials

README.md

@@ -91,13 +91,30 @@ syncing many files. Currently only one file is uploaded at the time,
 the speed can be improved in the future by uploading several files concurrently.
 To learn more see usage and the examples below.
 
+## Client Credentials
+By default, gdrive uses OAuth2 client credentials (client id and secret) in
+order to allow it to use Google's Drive APIs on behalf of the user. The credentials
+are set inside gdrive by default. Unfortunately this means that the rate limit of
+API usage is shared amongst all gdrive users globally. It has become common for
+the rate limit to be exceeded, causing gdrive to stop stop working.
+
+To get around this, gdrive now supports loading external OAuth2 client credentials.
+
+Generate your new credentials in the Google API Console, name the credentials
+file `client_id.json` and move it to the gdrive config directory.
+Then delete `token_v2.json` to reauthenticate using the new credentials.
+
+One side effect to note is that existing sync directories will not work after the
+credentials change because sync sets credential specific properties on the files
+(appProperty).
+
 ### Service Account
-For server to server communication, where user interaction is not a viable option, 
+For server to server communication, where user interaction is not a viable option,
 is it possible to use a service account, as described in this [Google document](https://developers.google.com/identity/protocols/OAuth2ServiceAccount).
 If you want to use a service account, instead of being interactively prompted for
-authentication, you need to use the `--service-account <serviceAccountCredentials>` 
+authentication, you need to use the `--service-account <serviceAccountCredentials>`
 global option, where `serviceAccountCredentials` is a file in JSON format obtained
-through the Google API Console, and its location is relative to the config dir. 
+through the Google API Console, and its location is relative to the config dir.
 
 #### .gdriveignore
 Placing a .gdriveignore in the root of your sync directory can be used to
@@ -174,7 +191,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -f, --force           Overwrite existing file
   -r, --recursive       Download directory recursively, documents will be skipped
@@ -194,7 +211,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -f, --force       Overwrite existing file
   -r, --recursive   Download directories recursively, documents will be skipped
@@ -211,7 +228,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -r, --recursive               Upload directory recursively
   -p, --parent <parent>         Parent id, used to upload file to a specific directory, can be specified multiple times to give many parents
@@ -234,7 +251,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -p, --parent <parent>         Parent id, used to upload file to a specific directory, can be specified multiple times to give many parents
   --chunksize <chunksize>       Set chunk size in bytes, default: 8388608
@@ -254,7 +271,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -p, --parent <parent>         Parent id, used to upload file to a specific directory, can be specified multiple times to give many parents
   --name <name>                 Filename
@@ -274,7 +291,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --bytes   Show size in bytes
 ```
@@ -288,7 +305,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -p, --parent <parent>         Parent id of created directory, can be specified multiple times to give many parents
   --description <description>   Directory description
@@ -303,7 +320,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --role <role>     Share role: owner/writer/commenter/reader, default: reader
   --type <type>     Share type: user/group/domain/anyone, default: anyone
@@ -343,7 +360,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -r, --recursive   Delete directory and all it's content
 ```
@@ -357,7 +374,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --no-header   Dont print the header
 ```
@@ -371,7 +388,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --order <sortOrder>        Sort order. See https://godoc.org/google.golang.org/api/drive/v3#FilesListCall.OrderBy
   --path-width <pathWidth>   Width of path column, default: 60, minimum: 9, use 0 for full width
@@ -388,7 +405,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --keep-remote         Keep remote file when a conflict is encountered
   --keep-local          Keep local file when a conflict is encountered
@@ -408,7 +425,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --keep-remote             Keep remote file when a conflict is encountered
   --keep-local              Keep local file when a conflict is encountered
@@ -429,7 +446,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -m, --max <maxChanges>     Max changes to list, default: 100
   --since <pageToken>        Page token to start listing changes from
@@ -447,7 +464,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --name-width <nameWidth>   Width of name column, default: 40, minimum: 9, use 0 for full width
   --no-header                Dont print the header
@@ -463,7 +480,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -f, --force           Overwrite existing file
   --no-progress         Hide progress
@@ -492,7 +509,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -p, --parent <parent>   Parent id, used to upload file to a specific directory, can be specified multiple times to give many parents
   --no-progress           Hide progress
@@ -507,7 +524,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   -f, --force     Overwrite existing file
   --mime <mime>   Mime type of exported file
@@ -523,7 +540,7 @@ global:
   --refresh-token <refreshToken>   Oauth refresh token used to get access token (for advanced users)
   --access-token <accessToken>     Oauth access token, only recommended for short-lived requests because of short lifetime (for advanced users)
   --service-account <accountFile>  Oauth service account filename, used for server to server communication without user interaction (file is relative to config dir)
-  
+
 options:
   --bytes   Show size in bytes
 ```

auth/file_source.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"encoding/json"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
 	"io/ioutil"
 	"os"
 )
@@ -43,6 +44,15 @@ func ReadFile(path string) ([]byte, bool, error) {
 	return content, true, nil
 }
 
+func ReadClientCredentials(path string) (*oauth2.Config, bool, error) {
+
+	content, exists, err := ReadFile(path)
+	if err != nil || exists == false {
+		return nil, exists, err
+	}
+	conf, err := google.ConfigFromJSON(content)
+	return conf, exists, err
+}
 
 func ReadToken(path string) (*oauth2.Token, bool, error) {
 

auth/oauth.go

@@ -10,8 +10,7 @@ import (
 
 type authCodeFn func(string) func() string
 
-func NewFileSourceClient(clientId, clientSecret, tokenFile string, authFn authCodeFn) (*http.Client, error) {
-	conf := getConfig(clientId, clientSecret)
+func NewFileSourceClient(conf *oauth2.Config, tokenFile string, authFn authCodeFn) (*http.Client, error) {
 
 	// Read cached token
 	token, exists, err := ReadToken(tokenFile)
@@ -36,8 +35,7 @@ func NewFileSourceClient(clientId, clientSecret, tokenFile string, authFn authCo
 	), nil
 }
 
-func NewRefreshTokenClient(clientId, clientSecret, refreshToken string) *http.Client {
-	conf := getConfig(clientId, clientSecret)
+func NewRefreshTokenClient(conf *oauth2.Config, refreshToken string) *http.Client {
 
 	token := &oauth2.Token{
 		TokenType:    "Bearer",
@@ -51,8 +49,7 @@ func NewRefreshTokenClient(clientId, clientSecret, refreshToken string) *http.Cl
 	)
 }
 
-func NewAccessTokenClient(clientId, clientSecret, accessToken string) *http.Client {
-	conf := getConfig(clientId, clientSecret)
+func NewAccessTokenClient(conf *oauth2.Config, accessToken string) *http.Client {
 
 	token := &oauth2.Token{
 		TokenType:   "Bearer",
@@ -82,7 +79,7 @@ func NewServiceAccountClient(serviceAccountFile string) (*http.Client, error) {
 	return conf.Client(oauth2.NoContext), nil
 }
 
-func getConfig(clientId, clientSecret string) *oauth2.Config {
+func AssembleClientCredentials(clientId, clientSecret string) *oauth2.Config {
 	return &oauth2.Config{
 		ClientID:     clientId,
 		ClientSecret: clientSecret,

handlers_drive.go

@@ -17,8 +17,11 @@ import (
 const ClientId = "367116221053-7n0vf5akeru7on6o2fjinrecpdoe99eg.apps.googleusercontent.com"
 const ClientSecret = "1qsNodXNaWq1mQuBjUjmvhoO"
 const TokenFilename = "token_v2.json"
+const ClientCredentialsFilename = "client_id.json"
 const DefaultCacheFileName = "file_cache.json"
 
+var usingClientCredentialsFile = false
+
 func listHandler(ctx cli.Context) {
 	args := ctx.Args()
 	err := newDrive(args).List(drive.ListFilesArgs{
@@ -345,16 +348,31 @@ func getOauthClient(args cli.Arguments) (*http.Client, error) {
 		ExitF("Access token not needed when refresh token is provided")
 	}
 
+	configDir := getConfigDir(args)
+
+	clientCredentialsPath := ConfigFilePath(configDir, ClientCredentialsFilename)
+	clientCredentials, exists, err := auth.ReadClientCredentials(clientCredentialsPath)
+	if err != nil {
+		ExitF("Failed to read client credentials file: %s", err)
+	} else if !exists {
+		clientCredentials = auth.AssembleClientCredentials(ClientId, ClientSecret)
+	} else {
+		usingClientCredentialsFile = true
+		// Make sure the google drive scope is present
+		if len(clientCredentials.Scopes) == 0 {
+			clientCredentials.Scopes = append(clientCredentials.Scopes, "https://www.googleapis.com/auth/drive")
+		}
+	}
+
+
 	if args.String("refreshToken") != "" {
-		return auth.NewRefreshTokenClient(ClientId, ClientSecret, args.String("refreshToken")), nil
+		return auth.NewRefreshTokenClient(clientCredentials, args.String("refreshToken")), nil
 	}
 
 	if args.String("accessToken") != "" {
-		return auth.NewAccessTokenClient(ClientId, ClientSecret, args.String("accessToken")), nil
+		return auth.NewAccessTokenClient(clientCredentials, args.String("accessToken")), nil
 	}
 
-	configDir := getConfigDir(args)
-
 	if args.String("serviceAccount") != "" {
 		serviceAccountPath := ConfigFilePath(configDir, args.String("serviceAccount"))
 		serviceAccountClient, err := auth.NewServiceAccountClient(serviceAccountPath)
@@ -365,7 +383,7 @@ func getOauthClient(args cli.Arguments) (*http.Client, error) {
 	}
 
 	tokenPath := ConfigFilePath(configDir, TokenFilename)
-	return auth.NewFileSourceClient(ClientId, ClientSecret, tokenPath, authCodePrompt)
+	return auth.NewFileSourceClient(clientCredentials, tokenPath, authCodePrompt)
 }
 
 func getConfigDir(args cli.Arguments) string {
@@ -392,6 +410,12 @@ func newDrive(args cli.Arguments) *drive.Drive {
 
 func authCodePrompt(url string) func() string {
 	return func() string {
+		if (usingClientCredentialsFile == true) {
+			fmt.Println("Client credentials loaded from file\n")
+		} else {
+			fmt.Println("Client credentials file not found. Using built-in defaults\n")
+		}
+
 		fmt.Println("Authentication needed")
 		fmt.Println("Go to the following url in your browser:")
 		fmt.Printf("%s\n\n", url)