GAM 3.72

Fix typo, clean up doPrintGroupMembers

.github/ISSUE_TEMPLATE.txt

@@ -0,0 +1,14 @@
+The issue tracker is for reporting product deficiencies. "How do I?" questions should be posted to the discussion forum at https://groups.google.com/group/google-apps-manager. When in doubt, start at the discussion forum and return here only when instructed to do so.
+
+Please confirm the following:
+* I have upgraded to the latest GAM release from https://git.io/gamreleases and I still have this issue.
+* I am typing the command as described in the GAM Wiki at https://github.com/jay0lee/gam/wiki
+
+Full steps to reproduce the issue:
+1. 
+2.
+3.
+
+Expected outcome (what are you trying to do?):
+
+Actual outcome (what errors or bad behavior do you see instead?):

src/.gitignore

@@ -67,3 +67,6 @@ gamcache/
 gam/
 gam-64/
 *.zip
+*.msi
+*.wixobj
+*.wixpdb

src/GamCommands.txt

@@ -0,0 +1,819 @@
+This document describes the GAM command line syntax in  modified BNF, see https://en.wikipedia.org/wiki/Backus-Naur_Form
+Items on the command line are space separated, when an actual space character is required, it will be indicated by <Space>.
+If an item contains spaces, it should be surrounded by " or '.
+
+[] optional item
+() group items
+* item may appear zero or more times
++ item may appear one or more times
+| separates alternative items
+
+# Primatives
+<Digit> ::= 0|1|2|3|4|5|6|7|8|9
+<Number> ::= <Digit>+
+<Hex> ::= <Digit>|a|b|c|d|e|f|A|B|C|D|E|F
+<Space> ::= an actual space character
+<String> ::= a string of characters, surrounded by " or ' if it contains spaces
+<TrueValues> ::= true|on|yes|enabled|1
+<FalseValues>= false|off|no|disabled|0
+<DataTransferService> ::= googleplus|google+|gplus|g+|googledrive|gdrive|drive
+<ProductID> ::= Google-Apps|Google-Coordinate|Google-Drive-storage|Google-Vault
+<SKUID> ::= apps|gafb|gafw|gams|gau|unlimited|d4w|dfw|coordinate|vault|vfe|
+	    drive-20gb|drive20gb|20gb|drive-50gb|drive50gb|50gb|drive-200gb|drive200gb|200gb|drive-400gb|drive400gb|400gb|
+	    drive-1tb|drive1tb|1tb|drive-2tb|drive2tb|2tb|drive-4tb|drive4tb|4tb|drive-8tb|drive8tb|8tb|drive-16tb|drive16tb|16tb
+<Charset> ::= ascii|mbcs|utf-8|utf-8-sig|utf-16|<String>
+<FileFormat> ::= csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
+<Language> ::= ar|bn|bg|ca|zh-CN|zh-TW|hr|cs|da|nl|en|en-GB|et|fi|fr|de|el|gu|iw|is|in|it|ja|kn|ko|lv|lt|ms|ml|mr|no|or|fa|pl|pt-BR|pt-PT|ro|ru|sr|sk|sl|es|sv|tl|ta|te|th|tr|uk|ur|vi
+
+# Basic items built from primatives
+<Boolean> ::= <TrueValues>|<FalseValues>
+<ByteCount> ::= <Number>[m|k|b]
+<CIDRnetmask> ::= <Number>.<Number>.<Number>.<Number>/<Number>
+<ColorHex> ::= #<Hex><Hex><Hex><Hex><Hex><Hex>
+<DomainName> ::= <String>(.<String>)+
+<EmailAddress> ::= <String>@<DomainName>
+<Year> ::= <Digit><Digit><Digit><Digit>
+<Month> ::= <Digit><Digit>
+<Day> ::= <Digit><Digit>
+<Hour> ::= <Digit><Digit>
+<Minute> ::= <Digit><Digit>
+<Second> ::= <Digit><Digit>
+<MilliSeconds> ::= <Digit><Digit><Digit>
+<Date> ::= <Year>-<Month>-<Day>
+<DateTime> ::= <Year>-<Month>-<Day>(<Space>|T)<Hour>:<Minute>
+<Time> ::= <Year>-<Month>-<Day>(<Space>|T)<Hour>:<Minute>[:<Second>[.<MilliSeconds>[Z]]]
+<RegularExpression> ::= <Python Regular Expression, see: https://docs.python.org/2/library/re.html>
+<Tag> ::= <String>
+<UniqueID> ::= uid:<String>
+
+# Named items
+<AccessToken> ::= <String>
+<ACLScope> ::= [user:]<EmailAddress>|group:<EmailAddress>|domain[:<DomainName>]|default
+<CalendarACLRole> ::= editor|freebusy|freebusyreader|owner|reader|writer
+<CalendarColorIndex> ::== <Number in range 1-24>
+<CalendarItem> ::= <EmailAddress>|<String>
+<ClientID> ::= <String>
+<CourseAlias> ::= <String>
+<CourseID> ::= <Number>|d:<CourseAlias>
+<CourseParticipantType> ::= teacher|teachers|student|students
+<CrOSID> ::= <String>
+<CrOSItem> ::= <CrOSID>|(query:<QueryCrOS>)|(query <QueryCrOS>)
+<DestEmailAddress> ::= <EmailAddress>
+<DomainAlias> ::= <String>
+<DriveFileACLRole> :: =commenter|editor|owner|reader|writer
+<DriveFileID> ::= <String>
+<DriveFileURL> :: = https://docs.google.com/a/<DomainName>/document/d/<DriveFileID>/<String>
+<DriveFileItem> ::= <DriveFileID>|<DriveFileURL>
+<DriveFileName> ::= <String>
+<DriveFolderID> ::= <String>
+<DriveFolderName> ::= <String>
+<EmailItem> ::= <EmailAddress>|<UniqueID>|<String>
+<EventColorIndex> ::== <Number in range 1-11>
+<EventID> ::= <String>
+<FieldName> ::= <String>
+<FileName> ::= <String>
+<FileNamePattern> ::= <String>
+<FilterID> ::= <Sttring>
+<FolderName> ::= <String>
+<GroupItem> ::= <EmailAddress>|<UniqueID>|<String>
+<GuardianID> ::= <String>
+<HostName> ::= <String>
+<Key> ::= <String>
+<LabelID> ::= <String>
+<LabelName> ::= <String>
+<LabelReplacement> ::= <String>
+<Marker> ::= <String>
+<MobileID> ::= <String>
+<MobileItem> ::= <MobileID>|(query:<QueryMobile>)|(query <QueryMobile>)
+<Name> ::= <String>
+<NotificationID> ::= <String>
+<OrgUnitID> ::= <String>
+<OrgUnitPath> ::= /|(/<String)+
+<ParameterKey> ::= <String>
+<ParameterValue> ::= <String>
+<PermissionID> ::= id:<String>|<EmailAddress>|anyone|anyonewithlink
+<PrinterID> ::= <String>
+<PrintJobAge> ::= <Number>[m|h|d]
+<PrintJobID> ::= <String>
+<PrintJobStatus> ::= done|error|held|in_progress|queued|submitted
+<PropertyKey> ::= <String>
+<PropertyValue> ::= <String>
+<QueryContact> ::= <String> See: https://developers.google.com/google-apps/contacts/v3/reference#contacts-query-parameters-reference
+<QueryCrOS> ::= <String> See: https://support.google.com/chrome/a/answer/1698333?hl=en
+<QueryDriveFile> :: = <String> See: https://developers.google.com/drive/v2/web/search-parameters 
+<QueryGmail> ::= <String> See: https://support.google.com/mail/answer/7190
+<QueryMobile> ::= <String> See: https://support.google.com/a/answer/1408863?hl=en#search
+<QueryPrinter> ::= <String> See: https://developers.google.com/cloud-print/docs/appInterfaces#search
+<QueryPrintJob> ::= <String> See: https://developers.google.com/cloud-print/docs/appInterfaces#parameters_3
+<QueryUser> :: = <String> See: https://developers.google.com/admin-sdk/directory/v1/guides/search-users
+<RequestID> ::= <String>
+<ResourceID> ::= <String>
+<RoleItem> ::= <UniqueID>|<String>
+<RoleAssignmentID> ::= <String>
+<SchemaName> ::= <String>
+<Section> ::= <String>
+<StudentID> ::= <String>
+<TeacherID> ::= <String>
+<Timezone> ::= <String>
+<Title> ::= <String>
+<URI> ::= <String>
+<URL> ::= <String>
+<UserItem> ::= <EmailAddress>|<UniqueID>|<String>
+
+
+<CrOSFieldName> ::=
+	activetimeranges|timeranges|
+	annotatedassetid|assedid|asset|
+	annotatedlocation|location|
+	annotateduser|user|
+	bootmode|
+	deviceid|
+	ethernetmacaddress|
+	firmwareversion|
+	lastenrollmenttime|
+	lastsync|
+	macaddress|
+	meid|
+	model|
+	notes|
+	ordernumber|
+	orgunitpath|org|ou|
+	osversion|
+	platformversion|
+	recentusers|
+	serialnumber|
+	status|
+	supportenddate|
+	willautorenew
+
+<CrOSOrderByFieldName> ::=
+	lastsync|location|notes|serialnumber|status|supportenddate|user
+
+<DriveFieldName> ::=
+	appdatacontents|
+	cancomment|
+	canreadrevisions|
+	copyable|
+	createddate|createdtime|
+	description|
+	editable|
+	explicitlytrashed|
+	fileextension|
+	filesize|
+	foldercolorrgb|
+	fullfileextension|
+	headrevisionid|
+	iconlink|
+	id|
+	lastmodifyinguser|
+	lastmodifyingusername|
+	lastviewedbyme|lastviewedbymedate|lastviewedbymetime|lastviewedbyuser|
+	md5|md5checksum|md5sum|
+	mime|mimetype|
+	modifiedbyme|modifiedbymedate|modifiedbymetime|modifiedbyuser|
+	modifieddate|modifiedtime|
+	name|
+	originalfilename|
+	ownedbyme|
+	ownernames|
+	owners|
+	parents|
+	permissions|
+	quotabytesused|quotaused|
+	restricted|
+	shareable|
+	shared|
+	sharedwithmedate|sharedwithmetime|
+	sharinguser|
+	size|
+	spaces|
+	starred|
+	thumbnaillink|
+	title|
+	trashed
+	userpermission|
+	version|
+	viewed|
+	viewerscancopycontent|
+	webcontentlink|
+	webviewlink|
+	writerscanshare
+
+<GroupFieldName> ::=
+	admincreated|
+	aliases|
+	allowexternalmembers|
+	allowgooglecommunication|
+	allowwebposting|
+	archiveonly|
+	customreplyto|
+	defaultmessagedenynotificationtext|
+	description|
+	email|
+	id|
+	includeinglobaladdresslist|gal|
+	isarchived|
+	maxmessagebytes|
+	memberscanpostasthegroup|
+	messagedisplayfont|
+	messagemoderationlevel|
+	name
+	primarylanguage|
+	replyto|
+	sendmessagedenynotification|
+	showingroupdirectory|
+	spammoderationlevel|
+	whocanadd|
+	whocancontactowner|
+	whocaninvite|
+	whocanjoin|
+	whocanleavegroup|
+	whocanpostmessage|
+	whocanviewgroup|
+	whocanviewmembership
+
+<GuardianState> ::=
+	complete|
+	pending
+
+<MembersFieldName> ::=
+	email|
+	group|
+	id|
+	name|
+	role|
+	type
+
+<MobileOrderByFieldName> ::=
+	deviceid|email|lastsync|model|name|os|status|type
+
+<OrgUnitFieldName> ::=
+	description|id|inherit|name|orgunitpath|parent|parentid|inherit
+
+<PrintJobOrderByFieldName> ::=
+	create_time|status|title
+
+<UserFieldName> ::=
+	addresses|address|
+	agreedtoterms|agreed2terms|
+	changepasswordatnextlogin|changepassword|
+	creationtime|
+	deletiontime|
+	email|emails|otheremail|otheremails|
+	externalids|externalid|
+	familyname|firstname|fullname|givenname|lastname|name|
+	id|
+	ims|im|
+	includeinglobaladdresslist|gal|
+	ipwhitelisted|
+	isdelegatedadmin|admin|isadmin|
+	ismailboxsetup|
+	lastlogintime|
+	noneditablealiases|aliases|nicknames|
+	notes|note|
+	organizations|organization|
+	orgunitpath|org|ou|
+	phones|phone|
+	primaryemail|username|
+	relations|relation|
+	suspended|
+	thumbnailphotourl|photo|photourl|
+	websites|website|
+	custom all|<SchemaNameList>
+
+<UserOrderByFieldName> ::=
+	familyname|lastname|givenname|firstname|email
+
+# Named Lists
+# Lists can be in the following formats
+# Items, separated by commas, without spaces or commas in the items themselves: item(,item)*
+# Items, separated by spaces, without spaces or commas in the items themselves: "item( item)*"
+# Items, separated by commas, with spaces or commas in the items themselves: "'it em'(,'it em')*"
+# Items, separated by spaces, with spaces or commas in the items themselves: "'it em'( 'it em')*"
+
+<ACLList> ::== '<ACLScope>(,<ACLScope>)*'
+<CalendarList> ::= '<CalendarItem>(,<CalendarItem>)*'
+<CourseAliasList> ::= '<CourseAlias>(,<CourseAlias>)*'
+<CourseIDList> ::= '<CourseID>(,<CourseID>)*'
+<CrOSFieldNameList> ::= '<CrOSFieldName(,<CrOSFieldName>)*'
+<CrOSList> ::= '<CrOSID>(,<CrOSID>)*'
+<DriveFileList> ::= '<DriveFileItem>(,<DriveFileItem>)*'
+<EmailAddressList> ::= '<EmailAddress>(,<EmailAddress>)*'
+<EventIDList> ::= '<EventID>(,<EventID>)*'
+<FileFormatList> ::= '<FileFormat>(,<FileFormat)*'
+<FilterIDList> ::= '<FilterID>(,<FilterID>)*'
+<GroupFieldNameList> ::= '<GroupFieldName(,<GroupFieldName>)*'
+<GroupList> ::= '<GroupItem>(,<GroupItem>)*'
+<GuardianStateList> ::= '<GuardianState>(,<GuardianState>)*'
+<LabelNameList> ::= '<LabelName>(,<LabelName)*'
+<MembersFieldNameList> ::= '<MembersFieldName(,<MembersFieldName>)*'
+<MobileList> ::= '<MobileId>(,<MobileId>)*'
+<OrgUnitList> ::== '<OrgUnitPath>(,<OrgUnitPath>)*'
+<PrinterIDList> ::= '<PrinterID>(,<PrinterID>)*'
+<ProductIDList> ::= '(<ProductID>|SKUID>)(,<ProductID>|SKUID>)*'
+<PrintJobIDList> ::= '<PrintJobID>(,<PrintJobID>)*'
+<ResourceIDList> ::= '<ResourceID>(,<ResourceID>)*'
+<SKUIDList> ='<SKUID>(,<SKUID>)*'
+<SchemaNameList> ::= '<SchemaName>(,<SchemaName>)*'
+<UserFieldNameList> ::= '<UserFieldName(,<UserFieldName>)*'
+<UserList> ::= '<UserItem>(,<UserItem>)*'
+
+# Specify a collection of ChromeOS devices by directly specifying them
+<CrOSTypeEntity> ::=
+		(all cros)|
+		(cros <CrOSList>)|
+# Specify a collection of Users by directly specifying them or by specifiying items that will yield a list of users
+<UserTypeEntity> ::=
+		(all users)|
+		(user <UserItem>)|
+		(users <UserList>)|
+		(group <GroupItem)|
+		(ou|org <OrgUnitPath)|
+		(ou_and_children|ou_and_child <OrgUnitPath>)|
+		(courseparticipants <CourseID>)|
+		(students <CourseID>)|
+		(teachers <CourseID>)|
+		(file <FileName>)|
+		(csvfile <FileName>:<FieldName>)|
+		(license|licenses|licence|licences <SKUIDList>)|
+		(query <QueryUser>)
+
+
+# Item attributes
+<CalendarAttributes> ::=
+	(selected <Boolean>)|(hidden <Boolean>)|(summary <String>)|(colorindex|colorid <CalendarColorIndex>)|(backgroundcolor <ColorHex>)|(foregroundcolor <ColorHex>)|
+	(reminder clear|(email|sms|pop <Number>))|
+	(notification clear|(email|sms eventcreation|eventchange|eventcancellation|eventresponse|agenda))
+
+<CalendarSettings> ::==
+	(summary <String>)|(description <String>)|(location <String>)|(timezone <String>)
+
+<CourseAttributes> ::=
+	(name <String>)|(section <string>)|(heading <String>)|(room <String>)|(description <String>)|
+	(state|status active|archived|provisioned|declined)
+
+<CrOSAttributes> ::=
+	(asset|assetid|tag <String>)|
+	(location <String>)|
+	(notes <String>)|
+	(org|ou <OrgUnitPath>)|
+	(status active|deprovisioned|inactive|returnapproved|returnrequested|shipped|unknown)|
+	(user <Name>)
+
+<DriveFileAddAttributes> ::=
+	(localfile <FileName>)|
+	(convert)|(ocr)|(ocrlanguage <Language>)|(restricted|restrict)|(starred|star)|(trashed|trash)|(viewed|view)|
+	(lastviewedbyme <Time>)|(modifieddate|modifiedtime <Time>)|(description <String>)|(mimetype gdoc|gdocument|gdrawing|gfolder|gdirectory|gform|gfusion|gpresentation|gscript|gsite|gsheet|gspreadsheet)|
+	(parentid <DriveFolderID>)|(parentname <FolderName>)|writerscantshare
+<DriveFileUpdateAttributes> ::=
+	(localfile <FileName>)|
+	(convert)|(ocr)|(ocrlanguage <Language>)|(restricted|restrict <Boolean>)|(starred|star <Boolean>)|(trashed|trash <Boolean>)|(viewed|view <Boolean>)|
+	(lastviewedbyme <Time>)|(modifieddate <Time>)|(description <String>)|(mimetype gdoc|gdocument|gdrawing|gfolder|gdirectory|gform|gfusion|gpresentation|gscript|gsite|gsheet|gspreadsheet)|
+	(parentid <DriveFolderID>)|(parentname <FolderName>)|writerscantshare
+<EventAttributes> ::=
+	(anyonecanaddself)|(guestscantinviteothers)|(guestscantseeothers)|(notifyattendees)|(available)|(visibility default|public|prvate)|(tentative)|
+	(attendee <EmailAddress>)|(optionalattendee <EmailAddress>)|
+	(description <String>)|(summary <String>)|(location <String>)|(id <String>)|
+	(source <String> <URL>)|(privateproperty <PropertyKey> <PropertyValue>)|(sharedproperty <PropertyKey> <PropertyValue>)|
+	(recurrence <RRULE, EXRULE, RDATE and EXDATE line>)|
+	(start allday <Date>)|(start <Time>)|(end allday <Date>)|(end <Time>)|(timezone <Timezone>)|
+	(noreminders|(reminder email|popup|sms <Number>))|
+	(colorindex|colorid <EventColorIndex>)
+
+<GroupAttributes> ::=
+	(allowexternalmembers <Boolean>)|
+	(allowgooglecommunication <Boolean>)|
+	(allowwebposting <Boolean>)|
+	(archiveonly <Boolean>)|
+	(customfootertext <String>)|
+	(customreplyto <EmailAddress>)|
+	(defaultmessagedenynotificationtext <String>)|
+	(description <String>)|
+	(gal|includeInGlobalAddressList <Boolean>)|
+	(includecustomfooter <Boolean>)|
+	(isarchived <Boolean>)|
+	(maxmessagebytes <ByteCount>)|
+	(memberscanpostasthegroup <Boolean>)|
+	(messagedisplayfont DEFAULT_FONT|FIXED_WIDTH_FONT)|
+	(messagemoderationlevel MODERATE_ALL_MESSAGES|MODERATE_NON_MEMBERS|MODERATE_NEW_MEMBERS|MODERATE_NONE)|
+	(name <String>)|
+	(primarylanguage <Language>)|
+	(replyto REPLY_TO_CUSTOM|REPLY_TO_SENDER|REPLY_TO_LIST|REPLY_TO_OWNER|REPLY_TO_IGNORE|REPLY_TO_MANAGERS)|
+	(sendmessagedenynotification <Boolean>)|
+	(showingroupdirectory <Boolean>)|
+	(spammoderationlevel ALLOW|MODERATE|SILENTLY_MODERATE|REJECT)|
+	(whocanadd ALL_MEMBERS_CAN_ADD|ALL_MANAGERS_CAN_ADD|NONE_CAN_ADD)|
+	(whocancontactowner ANYONE_CAN_CONTACT|ALL_IN_DOMAIN_CAN_CONTACT|ALL_MEMBERS_CAN_CONTACT|ALL_MANAGERS_CAN_CONTACT)|
+	(whocaninvite ALL_MEMBERS_CAN_INVITE|ALL_MANAGERS_CAN_INVITE|NONE_CAN_INVITE)|
+	(whocanjoin ANYONE_CAN_JOIN|ALL_IN_DOMAIN_CAN_JOIN|INVITED_CAN_JOIN|CAN_REQUEST_TO_JOIN)|
+	(whocanleavegroup ALL_MANAGERS_CAN_LEAVE|ALL_MEMBERS_CAN_LEAVE|NONE_CAN_LEAVE)|
+	(whocanpostmessage NONE_CAN_POST|ALL_MANAGERS_CAN_POST|ALL_MEMBERS_CAN_POST|ALL_IN_DOMAIN_CAN_POST|ANYONE_CAN_POST)|
+	(whocanviewgroup ANYONE_CAN_VIEW|ALL_IN_DOMAIN_CAN_VIEW|ALL_MEMBERS_CAN_VIEW|ALL_MANAGERS_CAN_VIEW)|
+	(whocanviewmembership ALL_IN_DOMAIN_CAN_VIEW|ALL_MEMBERS_CAN_VIEW|ALL_MANAGERS_CAN_VIEW)
+
+<MobileAttributes> ::=
+	(model <String>)|(os <String>)|(useragent <String>)|
+	(action admin_remote_wipe|wipe|admin_account_wipe|accountwipe|wipeaccount|approve|block|cancel_remote_wipe_then_activate|cancel_remote_wipe_then_block)
+
+<PrinterAttributes> ::= (currentquota <Number>)|(dailyquota <Number>)|
+	(defaultdisplayname <String>)|(description <String>)|(displayname <String>)|(firmware <String>)|(gcpversion <String>)|
+	(istosaccepted <Boolean>)|(manufacturer <String>)|(model <String>)|(name <String>)|(ownerid <EmailAddress>)|(proxy <String>)|(public <Boolean>)|
+	(quotaenabled <Boolean>)|(status <Number>)|(type <String>)|(uuid <String>)|
+	(setupurl <URL>)|(supporturl <URL>)|(updateurl <URL>) 
+
+<SchemaFieldDefinition> ::=
+	field <FieldName> (type bool|date|double|email|int64|phone|string) [multivalued|multivalue] [indexed] [restricted] [range <Number> <Number>] endfield
+
+<UserAttributes> ::=
+	(address|addresses clear|(type work|home|other|(custom <String>) [unstructured|formatted <String>] [pobox <String>] [extendedaddress <String>] [streetaddress <String>]
+		[locality <String>] [region <String>] [postalcode <String>] [country <String>] [countrycode <String>] notprimary|primary))|
+	(admin <Boolean>)|
+	(agreed2terms|agreedtoterms <Boolean>)|
+	(changepassword|changepasswordatnextlogin <Boolean>)|
+	(crypt|sha|sha1|sha-1|md5|nohash)|
+	(customerid <String>)|
+	(email|primaryemail|username <EmailAddress>)|
+	(emails|otheremail|otheremails clear|(work|home|other|<String> <String>))|
+	(externalid|externalids clear|(account|customer|network|organization|<String> <String>))|
+	(firstname|givenname <String>)|
+	(gal|includeinglobaladdresslist <Boolean>)|
+	(im|ims clear|(type work|home|other|(custom <String>) protocol aim|gtalk|icq|jabber|msn|net_meeting|qq|skype|yahoo|(custom_protocol <String>) <String> [notprimary|primary]))|
+	(ipwhitelisted <Boolean>)|
+	(lastname|familyname <String>)|
+	(note|notes clear|([text_plain|text_html] <String>|(file <FileName>)))|
+	(organization|organizations clear|([type domain_only|school|unknown|work] [customtype <String>] [name <String>] [title <String>] [department <String>] [symbol <String>]
+		[costcenter <String>]  [location <String>] [description <String>] [domain <String>] notprimary|primary))|
+	(org|ou|orgunitpath <OrgUnitPath>)
+	(password random|<String>)|
+	(phone|phones clear|([type work|home|other|work_fax|home_fax|other_fax|main|company_main|assistant|mobile|work_mobile|pager|work_pager|car|radio|callback|isdn|telex|tty_tdd|grand_central|(custom <String>)
+		[value <String>] notprimary|primary))|
+	(relation|relations clear|(spouse|child|mother|father|parent|brother|sister|friend|relative|domestic_partner|manager|assistant|referred_by|partner|<String> <String>))|
+	(suspended <Boolean>)|
+	(website|websites clear|(home_page|blog|profile|work|home|other|ftp|reservations|app_install_page|<String> <URL> [notprimary|primary]))|
+	(<SchemaName>.<FieldName> [multivalued|multivalue|value] <String>)
+
+
+gam version
+gam help
+
+gam batch <FileName>|- [charset <Charset>]
+gam csv <FileName>|- [charset <Charset>] gam <GAM argument list>
+
+gam oauth|oauth2 create|request
+gam oauth|oauth2 delete|revoke
+gam oauth|oauth2 info|verify [<AccessToken>]
+
+gam whatis <EmailItem>
+
+gam report users|user [todrive]
+	[date <Date>] [(user all|<UserItem>)] [filter|filters <String>] [fields|parameters <String>]
+gam report customers|customer|domain [todrive]
+	[date <Date>] [fields|parameters <String>]
+gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token [todrive]
+	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
+
+gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
+gam delete admin <RoleAssignmentId>
+gam print admins [todrive] [user <UserItem>] [role <RoleItem>]
+gam print adminroles|roles [todrive]
+
+gam create domain <DomainName>
+gam update domain <DomainName> primary
+gam delete domain <DomainName>
+gam info domain [<DomainName>]
+gam print domains [todrive]
+
+gam create domainalias|aliasdomain <DomainAlias> <DomainName>
+gam delete domainalias|aliasdomain <DomainAlias>
+gam info domainalias|aliasdomain <DomainAlias>
+gam print domainaliases|aliasdomains [todrive]
+
+
+gam info customer
+gam update customer [adminsecondaryemail|alternateemail <EmailAddress>] [language <LanguageCode] [phone|phonenumber <String>]
+	[contact|contactname <String>] [name|organizationname <String>]
+	[address1|addressline1 <String>] [address2|addressline2 <String>] [address3|addressline3 <String>]
+	[locality <String>] [region <String>] [postalcode <String>] [country|countrycode <String>]
+
+gam create datatransfer|transfer <OldOwnerID> <DataTransferService> <NewOwnerID> (<ParameterKey> <ParameterValue>)*
+gam info datatransfer|transfer <TransferID>
+gam print datatransfers|transfers [todrive] [olduser|oldowner <UserItem>] [newuser|newowner <UserItem>] [status <String>]
+
+gam print transferapps
+
+gam update instance logo <FileName>
+gam update instance sso_settings [enabled <Boolean>] [sign_on_uri <URI>] [sign_out_uri <URI>] [password_uri <URI>] [whitelist <CIDRnetmask>] [use_domain_specific_issuer <Boolean>]
+gam update instance sso_key <FileName>
+gam info instance [logo <FileName>]
+
+gam create org|ou <Name> [description <String>] [parent <OrgUnitPath>] [inherit|noinherit]
+gam update org|ou <OrgUnitPath> [name <Name>] [description <String>] [parent <OrgUnitPath>] [inherit|noinherit]
+gam update org|ou <OrgUnitPath> add|move <CrOSTypeEntity>|<UserTypeEntity>
+gam delete org|ou <OrgUnitPath>
+gam info org|ou <OrgUnitPath> [nousers] [children|child]
+gam print orgs|ous [todrive] [toplevelonly] [from_parent <OrgUnitPath>] [allfields|<OrgUnitFieldName>*]
+
+gam create alias|nickname <EmailAddress> user|group|target <UniqueID>|<EmailAddress>
+gam update alias|nickname <EmailAddress> user|group|target <UniqueID>|<EmailAddress>
+gam delete alias|nickname [user|group|target] <UniqueID>|<EmailAddress>
+gam info alias|nickname <EmailAddress>
+gam print aliases|nicknames [todrive]
+
+gam audit uploadkey <ValueReadFromStdin>
+gam audit activity request <EmailAddress>
+gam audit activity delete <EmailAddress> <RequestID>
+gam audit activity download <EmailAddress> <RequestID>
+gam audit activity status [<EmailAddress> <RequestID>]
+
+gam audit export request <EmailAddress> [begin <DateTime>] [end <DateTime>] [search <QueryGmail>] [headersonly] [includedeleted]
+gam audit export delete <EmailAddress> <RequestID>
+gam audit export download <EmailAddress> <RequestID>
+gam audit export status [<EmailAddress> <RequestID>]
+gam audit export watch <EmailAddress> <RequestID>
+
+gam audit monitor create <EmailAddress> <DestEmailAddress> [begin <DateTime>] [end <DateTime>] [incoming_headers] [outgoing_headers] [nochats] [nodrafts] [chat_headers] [draft_headers]
+gam audit monitor delete <EmailAddress> <DestEmailAddress>
+gam audit monitor list <EmailAddress>
+
+gam calendar <CalendarItem> add <CalendarACLRole> ([user] <EmailAddress>)|(group <EmailAddress>)|(domain [<DomainName>])|default
+gam calendar <CalendarItem> update <CalendarACLRole> ([user] <EmailAddress>)|(group <EmailAddress>)|(domain [<DomainName>])|default
+gam calendar <CalendarItem> del|delete <CalendarACLRole> <EmailAddress>|(domain [<DomainName>])|default
+gam calendar <CalendarItem> showacl
+
+gam calendar <CalendarItem> addevent <EventAttributes>+
+gam calendar <CalendarItem> wipe
+
+gam update cros <CrOSItem> <CrOSAttributes>+
+gam info cros <CrOSItem> [nolists] [listlimit <Number>]
+	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>]
+
+gam print cros [todrive] [query <QueryCrOS>]
+	[orderby <CrOSOrderByFieldName> [ascending|descending]] [nolists] [listlimit <Number>]
+	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>]
+gam <CrOSTypeEntity> print
+
+Summary of printing:
+gam print cros
+Prints a header row and deviceId for all CrOS devices.
+gam <CrOSTypeEntity> print cros
+Prints no header row and deviceId for specified CrOS devices.
+gam print cros ... basic|full
+Prints a header row and selected fields for specified CrOS devices.
+
+The basic argument yields these column headers: deviceId,annotatedAssetId,annotatedLocation,annotatedUser,lastSync,notes,serialNumber,status
+The full argument yields all column headers including two headers, recentUsers and activeTimeRanges,
+that repeat with two subvalues each, yielding a large number of columns that make the output hard to process.
+The nolists argument suppresses these two headers; if you want these headers in a more manageable form use the following arguments.
+The listlimit <Number> argument limits the number of repetitions to <Number>. If <Number> equals zero, there is no limit.
+If recentusers is specified as a field, each pair of values for recentUsers is put on a separate row with all of the other headers.
+If timeranges is specified as a field, each  pair of values for activeTimeRanges is put on a separate row with all of the other headers.
+
+gam update mobile <MobileItem> <MobileAttributes>+
+gam delete mobile <MobileItem>
+gam info mobile <MobileItem>
+gam print mobile [todrive] [query <QueryMobile>] [basic|full] [orderby <MobileOrderByFieldName> [ascending|descending]]
+
+gam create group <EmailAddress> <GroupAttributes>*
+gam update group <GroupItem> [admincreated <Boolean>] [email <EmailAddress>] <GroupAttributes>*
+gam update group <GroupItem> add [owner|manager|member] [notsuspended] <UserTypeEntity>
+gam update group <GroupItem> delete|remove [owner|manager|member] <UserTypeEntity>
+gam update group <GroupItem> sync [owner|manager|member] [notsuspended] <UserTypeEntity>
+gam update group <GroupItem> update [owner|manager|member] <UserTypeEntity>
+gam delete group <GroupItem>
+gam info group <GroupItem> [nousers] [noaliases] [groups]
+gam update group <GroupItem> clear [owner] [manager] [member]
+
+gam print groups [todrive] ([domain <DomainName>] [member <UserItem>])
+	[maxresults <Number>] [delimiter <String>]
+	[members] [owners] [managers] [settings] <GroupFieldName>* [fields <GroupFieldNameList>]
+
+
+gam print group-members|groups-members [todrive] ([domain <DomainName>] [member <UserItem>])|[group <GroupItem>]
+	[membernames] [fields <MembersFieldNameList>]
+
+gam print license|licenses|licence|licences [todrive] [products|product <ProductIDList>] [skus|sku <SKUIDList>]
+
+gam update notification|notifications [(id all)|(id <NotificationID>)*] unread|read
+gam delete notification|notifications [(id all)|(id <NotificationID>)*]
+gam info notification|notifications [unreadonly]
+
+gam create resource <ResourceID> <Name> [description <String>] [type <String>]
+gam update resource <ResourceID> [name <Name>] [description <String>] [type <String>]
+gam delete resource <ResourceID>
+gam info resource <ResourceID>
+gam print resources [todrive] [allfields] [id] [name] [description] [email] [type]
+
+gam create schema|schemas <SchemaName> <SchemaFieldDefinition>+
+gam update schema <SchemaName> <SchemaFieldDefinition>* (deletefield <FieldName>)*
+gam delete schema <SchemaName>
+gam info schema <SchemaName>
+gam show schema|schemas
+gam print schema|schemas
+
+gam create user <EmailAddress> <UserAttrubutes>*
+gam update user <UserItem> <UserAttributes>+
+gam delete user <UserItem>
+gam undelete user <UserItem> [org|ou <OrgUnitPath>]
+gam info user [<UserItem>] [noaliases] [nogroups] [nolicenses|nolicences] [noschemas] [schemas <SchemaNameList>] [userview] [skus|sku <SKUIDList>]
+
+gam print users [todrive] ([domain <DomainName>] [query <QueryUser>] [deleted_only|only_deleted])
+	[groups] [license|licenses|licence|licences] [emailpart|emailparts|username]
+	[orderby <UserOrderByFieldName> [ascending|descending]] [userview]
+	[basic|full|allfields | <UserFieldName>* | fields <UserFieldNameList>]
+gam <UserTypeEntity> print
+
+Summary of printing:
+gam print users
+Prints a header row and primaryEmail for all users.
+gam <UserTypeEntity> print users
+Prints no header row and primaryEmail for specified users.
+
+gam create verify|verification <DomainName>
+gam update verify|verification <DomainName> cname|txt|text|site|file
+gam info verify|verification
+
+gam create course id|alias <CourseAlias> [teacher <UserItem>] <CourseAttributes>*
+gam update course <CourseID> <CourseAttributes>+
+gam delete course <CourseID>
+gam info course <CourseID>
+gam print courses [todrive] [teacher] [student] [alias|aliases] [delimiter <String>]
+
+gam course <CourseID> add alias <CourseAlias>
+gam course <CourseID> delete alias <CourseAlias>
+gam course <CourseID> add teachers|students <UserItem>
+gam course <CourseID> delete|remove teachers|students <UserItem>
+gam course <CourseID> sync teachers|students <UserTypeEntity>
+gam print course-participants [todrive] (course|class <CourseID>)*|([teacher <UserItem>] [student <UserItem>]) [show all|students|teachers]
+
+gam create guardian|guardianinvite|inviteguardian <EmailAddress> <UserItem>
+gam delete guardian|guardians <GuardianID>|<EmailAddress> <UserItem>
+gam show guardian|guardians [invitedguardian <GuardianID>] [student <UserItem>] [invitations] [states <GuardianStateList>] [<UserTypeEntity>]
+gam print guardian|guardians [todrive] [invitedguardian <GuardianID>] [student <UserItem>] [invitations] [states <GuardianStateList>] [<UserTypeEntity>]
+
+gam printer register
+gam update printer <PrinterID> <PrinterAttributes>+
+gam delete printer <PrinterID>
+gam info printer <PrinterID> [everything]
+gam print printers [todrive] [query <QueryPrinter>] [type <String>] [status <String>] [extrafields <String>]
+
+gam printer <PrinterID> add user|manager|owner <EmailAddress>|[domain:]<DomainName>|public
+gam printer <PrinterID> delete <EmailAddress>|[domain:]<DomainName>|public
+gam printer <PrinterID> showacl
+gam printjob <PrintJobID> cancel
+gam printjob <PrintJobID> delete
+gam printjob <PrintJobID> resubmit <PrinterID>
+
+gam printjob <PrinterID>|any fetch
+	[olderthan|newerthan <PrintJobAge>] [query <QueryPrintJob>]
+	[status <PrintJobStatus>]
+	[orderby <PrintJobOrderByFieldName> [ascending|descending]]
+	[owner|user <EmailAddress>]
+	[limit <Number>] [drivedir|(targetfolder <FilePath>)]
+gam printjob <PrinterID> submit <FileName>|<URL> [name|title <String>] (tag <String>)*
+gam print printjobs [todrive] [printer|printerid <PrinterID>]
+	[olderthan|newerthan <PrintJobAge>] [query <QueryPrintJob>]
+	[status <PrintJobStatus>]
+	[orderby <PrintJobOrderByFieldName> [ascending|descending]]
+	[owner|user <EmailAddress>]
+	[limit <Number>]
+
+gam <UserTypeEntity> delete|del asp|asps|applicationspecificpasswords <AspID>
+gam <UserTypeEntity> show asps|asp|applicationspecificpasswords
+
+gam <UserTypeEntity> update backupcodes|backupcode|verificationcodes
+gam <UserTypeEntity> delete|del backupcodes|backupcode|verificationcodes
+gam <UserTypeEntity> show backupcodes|backupcode|verificationcodes
+
+gam <UserTypeEntity> add calendar <CalendarItem> <CalendarAttributes>*
+gam <UserTypeEntity> update calendar <CalendarItem> <CalendarAttributes>+
+gam <UserTypeEntity> delete|del calendar <CalendarItem>
+gam <UserTypeEntity> show calendars
+gam <UserTypeEntity> info calendar <CalendarItem>|primary
+gam <UserTypeEntity> print calendars [todrive]
+
+gam <UserTypeEntity> show calsettings
+gam <UserTypeEntity> update calattendees csv <FileName> [dryrun] [start <Date>] [end <Date>] [allevents]
+gam <UserTypeEntity> transfer seccals <UserItem> [keepuser]
+
+gam <UserTypeEntity> print|show driveactivity [todrive] [fileid <DriveFileID>] [folderid <DriveFolderID>]
+gam <UserTypeEntity> print|show drivesettings [todrive]
+gam <UserTypeEntity> print|show filelist [todrive] [query <QueryDriveFile>] [allfields|<DriveFieldName>*]
+
+gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
+gam <UserTypeEntity> show filerevisions <DriveFileID>
+gam <UserTypeEntity> show filetree
+
+gam <UserTypeEntity> add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>*
+gam <UserTypeEntity> update drivefile (id <DriveFileID)|(drivefilename <DriveFileName>)|(query <QueryDriveFile) [copy] [newfilename <DriveFileName>] <DriveFileUpdateAttributes>*
+gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>) [format <FileFormatList>] [targetfolder <FilePath>] [revision <Number>]
+gam <UserTypeEntity> delete|del drivefile <DriveFileID>|<DriveFileURL>|(query:<QueryDriveFile>) [purge|untrash]
+gam <UserTypeEntity> transfer drive <UserItem> [keepuser]
+gam <UserTypeEntity> delete|del emptydrivefolders
+gam <UserTypeEntity> empty drivetrash
+
+gam <UserTypeEntity> add drivefileacl <DriveFileID> anyone|(user <UserItem>)|(group <GroupItem>)|(domain <DomainName>)
+	(role <DriveFileACLRole>) [withlink] [sendmail] [emailmessage <String>]
+gam <UserTypeEntity> update drivefileacl <DriveFileID> <PermissionID>
+	(role <DriveFileACLRole>) [withlink] [transferownership <Boolean>]
+gam <UserTypeEntity> delete|del drivefileacl <DriveFileID> <PermissionID>
+gam <UserTypeEntity> show drivefileacl <DriveFileID>
+
+gam <UserTypeEntity> delete|del alias|aliases
+
+gam <UserTypeEntity> delete|del group|groups
+
+gam <UserTypeEntity> add license <SKUID>
+gam <UserTypeEntity> update license <SKUID> [from] <SKUID>
+gam <UserTypeEntity> delete|del license <SKUID>
+
+gam <UserTypeEntity> update photo <FileNamePattern>
+gam <UserTypeEntity> delete|del photo
+gam <UserTypeEntity> get photo [drivedir|(targetfolder <FilePath>)] [noshow]
+
+gam <UserTypeEntity> profile share|shared|unshare|unshared
+gam <UserTypeEntity> show profile
+
+gam <UserTypeEntity> delete|del token|tokens clientid <ClientID>
+gam <UserTypeEntity> show tokens|token [clientid <ClientID>]
+gam <UserTypeEntity> print tokens|token [todrive] [clientid <ClientID>]
+gam print tokens|token [todrive] [clientid <ClientID>] [<UserTypeEntity>]
+
+gam <UserTypeEntity> update user <UserAttrubutes>
+
+gam <UserTypeEntity> deprovision|deprov
+#
+# Update user Gmail mailbox
+#
+gam <UserTypeEntity> [add] label|labels <Name> [messagelistvisibility hide|show] [labellistvisibility hide|show|showifunread]
+gam <UserTypeEntity> update labelsettings <LabelName> [name <Name>] [messagelistvisibility hide|show] [labellistvisibility hide|show|showifunread]
+gam <UserTypeEntity> update label|labels [search <RegularExpression>] [replace <LabelReplacement>] [merge]
+gam <UserTypeEntity> delete|del label|labels <LabelName>|regex:<RegularExpression>|--ALL_LABELS--
+gam <UserTypeEntity> show labels|label [onlyuser] [showcounts]
+
+gam <UserTypeEntity> delete messages query <QueryGmail> [doit] [max_to_delete|max_to_process <Number>]
+gam <UserTypeEntity> modify messages query <QueryGmail> [doit] [max_to_modify|max_to_process <Number>] (addlabel <LabelName>)* (removelabel <LabelName>)*
+gam <UserTypeEntity> trash messages query <QueryGmail> [doit] [max_to_trash|max_to_process <Number>]
+gam <UserTypeEntity> untrash messages query <QueryGmail> [doit] [max_to_untrash|max_to_process <Number>]
+#
+# Update user Gmail settings
+#
+gam <UserTypeEntity> show gmailprofile [todrive]
+gam <UserTypeEntity> show gplusprofile [todrive]
+gam <UserTypeEntity> arrows <Boolean>
+
+gam <UserTypeEntity> add delegate|delegates <EmailAddress>
+gam <UserTypeEntity> delegate|delegates to <EmailAddress>
+gam <UserTypeEntity> delete|del delegate|delegates <EmailAddress>
+gam <UserTypeEntity> show delegates|delegate [csv]
+gam <UserTypeEntity> print delegates [todrive]
+
+gam <UserTypeEntity> [add] filter [from <EmailAddress>] [to <EmailAddress>] [subject <String>] [haswords|query <List>] [nowords|negatedquery <List>] [musthaveattachment|hasattachment] [excludechats] [size larger|smaller <ByteCount>]
+	[label <LabelID>] [important|notimportant] [star] [trash] [markread] [archive] [neverspam] [forward <EmailAddress>]
+gam <UserTypeEntity> delete filters <FilterIDEntity>
+gam <UserTypeEntity> show filters
+gam <UserTypeEntity> info filters <FilterIDEntity>
+gam <UserTypeEntity> print filters [todrive]
+
+gam <UserTypeEntity> forward <FalseValues>
+gam <UserTypeEntity> forward <TrueValues> keep|leaveininbox|archive|delete|trash|markread <EmailAddress>
+gam <UserTypeEntity> show forward
+gam <UserTypeEntity> print forward [todrive]
+
+gam <UserTypeEntity> add forwardingaddress|forwardingaddresses <EmailAddress>
+gam <UserTypeEntity> delete forwardingaddress|forwardingaddresses <EmailAddress>
+gam <UserTypeEntity> show forwardingaddress|forwardingaddresses
+gam <UserTypeEntity> info forwardingaddress|forwardingaddresses <EmailAddress>
+gam <UserTypeEntity> print forwardingaddress|forwardingaddresses [todrive]
+
+gam <UserTypeEntity> imap|imap4 <Boolean> [noautoexpunge] [expungebehavior archive|deleteforever|trash] [maxfoldersize 0|1000|2000|5000|10000]
+gam <UserTypeEntity> show imap|imap4
+
+gam <UserTypeEntity> pop|pop3 <Boolean> [for allmail|newmail|mailfromnowon|fromnowown] [action keep|leaveininbox|archive|delete|trash|markread]
+gam <UserTypeEntity> show pop|pop3
+
+gam <UserTypeEntity> language <Language>
+
+gam <UserTypeEntity> pagesize 25|50|100
+
+gam <UserTypeEntity> [add] sendas <EmailAddress> <Name> [signature|sig <String>|(file <FileName> [charset <CharSet>]) (replace <RegularExpression> <String>)*] [replyto <EmailAddress>] [default] [treatasalias <Boolean>]
+gam <UserTypeEntity> update sendas <EmailAddress> [name <Name>] [signature|sig <String>|(file <FileName> [charset <CharSet>]) (replace <RegularExpression> <String>)*] [replyto <EmailAddress>] [default] [treatasalias <Boolean>]
+gam <UserTypeEntity> delete sendas <EmailAddress>
+gam <UserTypeEntity> show sendas [format]
+gam <UserTypeEntity> info sendas <EmailAddress> [format]
+gam <UserTypeEntity> print sendas [todrive]
+
+gam <UserTypeEntity> shortcuts <Boolean>
+
+gam <UserTypeEntity> signature|sig <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)* [name <String>] [replyto <EmailAddress>] [default] [treatasalias <Boolean>]
+gam <UserTypeEntity> show signature|sig [format]
+
+gam <UserTypeEntity> snippets <Boolean>
+
+gam <UserTypeEntity> utf|utf8|utf-8|unicode <Boolean>
+
+gam <UserTypeEntity> vacation <FalseValues>
+gam <UserTypeEntity> vacation <TrueValues> subject <String> (message <String>)|(file <FileName> [charset <CharSet>]) (replace <Tag> <String>)* [html]
+	[contactsonly] [domainonly] [startdate <Date>] [enddate <Date>]
+gam <UserTypeEntity> show vacation [format]
+
+gam <UserTypeEntity> webclips <Boolean>

src/admin-settings-v2.json

@@ -1,11 +1,11 @@
 {
  "kind": "discovery#restDescription",
  "discoveryVersion": "v1",
- "id": "admin-settings:v1",
+ "id": "admin-settings:v2",
  "name": "admin-settings",
- "version": "v1",
- "revision": "20130823",
- "title": "Admin Settings API (read-only calls)",
+ "version": "v2",
+ "revision": "20160616",
+ "title": "Admin Settings API",
  "description": "Lets you access Google Apps Admin Settings",
  "ownerDomain": "google.com",
  "ownerName": "Google",

src/build.bat

@@ -1,17 +0,0 @@
-rmdir /q /s gam
-rmdir /q /s gam-64
-rmdir /q /s build
-rmdir /q /s dist
-del /q /f gam-%1-windows.zip
-del /q /f gam-%1-windows-x64.zip
-
-c:\python27-32\scripts\pyinstaller -F --distpath=gam gam.spec
-xcopy LICENSE gam\
-xcopy whatsnew.txt gam\
-del gam\w9xpopen.exe
-"%ProgramFiles(x86)%\7-Zip\7z.exe" a -tzip gam-%1-windows.zip gam\ -xr!.svn
-
-c:\python27\scripts\pyinstaller -F --distpath=gam-64 gam.spec
-xcopy LICENSE gam-64\
-xcopy whatsnew.txt gam-64\
-"%ProgramFiles(x86)%\7-Zip\7z.exe" a -tzip gam-%1-windows-x64.zip gam-64\ -xr!.svn

src/email-audit-v1.json

@@ -0,0 +1,34 @@
+{
+ "kind": "discovery#restDescription",
+ "discoveryVersion": "v1",
+ "id": "email-audit:v1",
+ "name": "email-audit",
+ "version": "v1",
+ "revision": "20130823",
+ "title": "Email Audit API",
+ "description": "Lets you perform Google Apps email audits",
+ "ownerDomain": "google.com",
+ "ownerName": "Google",
+ "icons": {
+  "x16": "http://www.google.com/images/icons/product/search-16.gif",
+  "x32": "http://www.google.com/images/icons/product/search-32.gif"
+ },
+ "documentationLink": "https://developers.google.com/admin-sdk/email-audit",
+ "protocol": "rest",
+ "baseUrl": "https://apps-apis.google.com/",
+ "rootUrl": "https://apps-apis.google.com/",
+ "servicePath": "/a/feeds/compliance/audit/",
+ "auth": {
+  "oauth2": {
+   "scopes": {
+    "https://apps-apis.google.com/a/feeds/compliance/audit/": {
+     "description": "Manage email audits"
+    }
+   }
+  }
+ },
+ "schemas": {
+ },
+ "resources": {
+ }
+}

src/email-settings-v2.json

@@ -0,0 +1,34 @@
+{
+ "kind": "discovery#restDescription",
+ "discoveryVersion": "v1",
+ "id": "email-settings:v2",
+ "name": "email-settings",
+ "version": "v2",
+ "revision": "20160616",
+ "title": "Email Settings API",
+ "description": "Lets you manage Google Apps Email Settings",
+ "ownerDomain": "google.com",
+ "ownerName": "Google",
+ "icons": {
+  "x16": "http://www.google.com/images/icons/product/search-16.gif",
+  "x32": "http://www.google.com/images/icons/product/search-32.gif"
+ },
+ "documentationLink": "https://developers.google.com/admin-sdk/email-settings",
+ "protocol": "rest",
+ "baseUrl": "https://apps-apis.google.com/",
+ "rootUrl": "https://apps-apis.google.com/",
+ "servicePath": "/a/feeds/emailsettings/2.0/",
+ "auth": {
+  "oauth2": {
+   "scopes": {
+    "https://apps-apis.google.com/a/feeds/emailsettings/2.0/": {
+     "description": "Manage email settings"
+    }
+   }
+  }
+ },
+ "schemas": {
+ },
+ "resources": {
+ }
+}

src/gam.wxs

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi" >
+  <Product
+      Id="*"
+      Name="GAM"
+      Language="1033"
+      Version="$(env.GAMVERSION)"
+      Manufacturer="Jay Lee - jay0lee@gmail.com"
+      UpgradeCode="15C5FD61-B04C-4E04-A26D-CD8424C19D9F">
+    <Package
+        InstallerVersion="200" Compressed="yes" InstallScope="perMachine" />
+
+    <MajorUpgrade
+        DowngradeErrorMessage=
+            "A newer version of [ProductName] is already installed."
+        Schedule="afterInstallExecute" />
+    <MediaTemplate EmbedCab="yes" />
+
+    <Property Id="WIXUI_INSTALLDIR" Value="INSTALLFOLDER" />
+    <WixVariable Id="WixUILicenseRtf" Value="LICENSE.rtf" />
+    <UIRef Id="WixUI_InstallDir" />
+
+    <Feature
+        Id="gam"
+        Title="GAM"
+        Level="1">
+      <ComponentGroupRef Id="ProductComponents" />
+    </Feature>
+  </Product>
+
+  <Fragment>
+    <Directory Id="TARGETDIR" Name="SourceDir">
+      <Directory Id="ROOTDRIVE">
+          <Directory Id="INSTALLFOLDER" Name="GAM" />
+        </Directory>
+    </Directory>
+  </Fragment>
+
+  <Fragment>
+    <!-- Group of components that are our main application items -->
+    <ComponentGroup
+        Id="ProductComponents"
+        Directory="INSTALLFOLDER"
+        Source="gam-64">
+      <Component Id="gam_exe" Guid="886abc07-73c5-4acc-9f71-58daf62aabc1">
+        <File Name="gam.exe" KeyPath="yes" />
+      </Component>
+      <Component Id="license" Guid="7a15de2e-fb91-4d0a-b8bf-c8b19c68f569">
+        <File Name="LICENSE" KeyPath="yes" />
+      </Component>
+      <Component Id="whatsnew_txt" Guid="6aa9863c-90d9-412f-9b73-fda82549a950">
+        <File Name="whatsnew.txt" KeyPath="yes" />
+      </Component>
+    </ComponentGroup>
+  </Fragment>
+
+  <Fragment>
+    <InstallUISequence>
+      <ExecuteAction />
+      <Show Dialog="WelcomeDlg" Before="ProgressDlg" />
+      <!-- <Show Dialog="ProgressDlg" After="" /> -->
+    </InstallUISequence>
+  </Fragment>
+</Wix>

src/googleapiclient/__init__.py

@@ -12,4 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.5.0"
+__version__ = "1.5.1"
+
+# Set default logging handler to avoid "No handler found" warnings.
+import logging
+
+try:  # Python 2.7+
+    from logging import NullHandler
+except ImportError:
+    class NullHandler(logging.Handler):
+        def emit(self, record):
+            pass
+
+logging.getLogger(__name__).addHandler(NullHandler())

src/googleapiclient/discovery.py

@@ -82,6 +82,9 @@ URITEMPLATE = re.compile('{[^}]*}')
 VARNAME = re.compile('[a-zA-Z0-9_-]+')
 DISCOVERY_URI = ('https://www.googleapis.com/discovery/v1/apis/'
                  '{api}/{apiVersion}/rest')
+V1_DISCOVERY_URI = DISCOVERY_URI
+V2_DISCOVERY_URI = ('https://{api}.googleapis.com/$discovery/rest?'
+                    'version={apiVersion}')
 DEFAULT_METHOD_DOC = 'A description of how to use this function'
 HTTP_PAYLOAD_METHODS = frozenset(['PUT', 'POST', 'PATCH'])
 _MEDIA_SIZE_BIT_SHIFTS = {'KB': 10, 'MB': 20, 'GB': 30, 'TB': 40}
@@ -196,21 +199,23 @@ def build(serviceName,
   if http is None:
     http = httplib2.Http()
 
-  requested_url = uritemplate.expand(discoveryServiceUrl, params)
+  for discovery_url in (discoveryServiceUrl, V2_DISCOVERY_URI,):
+    requested_url = uritemplate.expand(discovery_url, params)
 
-  try:
-    content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
-                                      cache)
-  except HttpError as e:
-    if e.resp.status == http_client.NOT_FOUND:
-      raise UnknownApiNameOrVersion("name: %s  version: %s" % (serviceName,
-                                                               version))
-    else:
-      raise e
+    try:
+      content = _retrieve_discovery_doc(requested_url, http, cache_discovery,
+                                        cache)
+      return build_from_document(content, base=discovery_url, http=http,
+          developerKey=developerKey, model=model, requestBuilder=requestBuilder,
+          credentials=credentials)
+    except HttpError as e:
+      if e.resp.status == http_client.NOT_FOUND:
+        continue
+      else:
+        raise e
 
-  return build_from_document(content, base=discoveryServiceUrl, http=http,
-      developerKey=developerKey, model=model, requestBuilder=requestBuilder,
-      credentials=credentials)
+  raise UnknownApiNameOrVersion(
+        "name: %s  version: %s" % (serviceName, version))
 
 
 def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):

src/googleapiclient/discovery_cache/__init__.py

@@ -19,6 +19,9 @@ from __future__ import absolute_import
 import logging
 import datetime
 
+
+LOGGER = logging.getLogger(__name__)
+
 DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24  # 1 day
 
 
@@ -38,5 +41,5 @@ def autodetect():
       from . import file_cache
       return file_cache.cache
     except Exception as e:
-      logging.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
       return None

src/googleapiclient/discovery_cache/appengine_memcache.py

@@ -23,6 +23,9 @@ from google.appengine.api import memcache
 from . import base
 from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
 
+
+LOGGER = logging.getLogger(__name__)
+
 NAMESPACE = 'google-api-client'
 
 
@@ -41,12 +44,12 @@ class Cache(base.Cache):
     try:
       return memcache.get(url, namespace=NAMESPACE)
     except Exception as e:
-      logging.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
 
   def set(self, url, content):
     try:
       memcache.set(url, content, time=int(self._max_age), namespace=NAMESPACE)
     except Exception as e:
-      logging.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
 
 cache = Cache(max_age=DISCOVERY_DOC_MAX_AGE)

src/googleapiclient/discovery_cache/file_cache.py

@@ -29,12 +29,16 @@ import os
 import tempfile
 import threading
 
-from oauth2client.contrib.locked_file import LockedFile
+try:
+  from oauth2client.contrib.locked_file import LockedFile
+except ImportError:
+  # oauth2client < 2.0.0
+  from oauth2client.locked_file import LockedFile
 
 from . import base
 from ..discovery_cache import DISCOVERY_DOC_MAX_AGE
 
-logger = logging.getLogger(__name__)
+LOGGER = logging.getLogger(__name__)
 
 FILENAME = 'google-api-python-client-discovery-doc.cache'
 EPOCH = datetime.datetime.utcfromtimestamp(0)
@@ -84,7 +88,7 @@ class Cache(base.Cache):
         # If we can not obtain the lock, other process or thread must
         # have initialized the file.
       except Exception as e:
-        logging.warning(e, exc_info=True)
+        LOGGER.warning(e, exc_info=True)
       finally:
         f.unlock_and_close()
 
@@ -100,10 +104,10 @@ class Cache(base.Cache):
             return content
         return None
       else:
-        logger.debug('Could not obtain a lock for the cache file.')
+        LOGGER.debug('Could not obtain a lock for the cache file.')
         return None
     except Exception as e:
-      logger.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
     finally:
       f.unlock_and_close()
 
@@ -122,9 +126,9 @@ class Cache(base.Cache):
         f.file_handle().seek(0)
         json.dump(cache, f.file_handle())
       else:
-        logger.debug('Could not obtain a lock for the cache file.')
+        LOGGER.debug('Could not obtain a lock for the cache file.')
     except Exception as e:
-      logger.warning(e, exc_info=True)
+      LOGGER.warning(e, exc_info=True)
     finally:
       f.unlock_and_close()
 

src/googleapiclient/http.py

@@ -20,6 +20,7 @@ actuall HTTP request.
 """
 from __future__ import absolute_import
 import six
+from six.moves import http_client
 from six.moves import range
 
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
@@ -36,11 +37,19 @@ import logging
 import mimetypes
 import os
 import random
-import ssl
+import socket
 import sys
 import time
 import uuid
 
+# TODO(issue 221): Remove this conditional import jibbajabba.
+try:
+  import ssl
+except ImportError:
+  _ssl_SSLError = object()
+else:
+  _ssl_SSLError = ssl.SSLError
+
 from email.generator import Generator
 from email.mime.multipart import MIMEMultipart
 from email.mime.nonmultipart import MIMENonMultipart
@@ -57,10 +66,57 @@ from googleapiclient.model import JsonModel
 from oauth2client import util
 
 
+LOGGER = logging.getLogger(__name__)
+
 DEFAULT_CHUNK_SIZE = 512*1024
 
 MAX_URI_LENGTH = 2048
 
+_TOO_MANY_REQUESTS = 429
+
+
+def _should_retry_response(resp_status, content):
+  """Determines whether a response should be retried.
+
+  Args:
+    resp_status: The response status received.
+    content: The response content body. 
+
+  Returns:
+    True if the response should be retried, otherwise False.
+  """
+  # Retry on 5xx errors.
+  if resp_status >= 500:
+    return True
+
+  # Retry on 429 errors.
+  if resp_status == _TOO_MANY_REQUESTS:
+    return True
+
+  # For 403 errors, we have to check for the `reason` in the response to
+  # determine if we should retry.
+  if resp_status == six.moves.http_client.FORBIDDEN:
+    # If there's no details about the 403 type, don't retry.
+    if not content:
+      return False
+
+    # Content is in JSON format.
+    try:
+      data = json.loads(content.decode('utf-8'))
+      reason = data['error']['errors'][0]['reason']
+    except (UnicodeDecodeError, ValueError, KeyError):
+      LOGGER.warning('Invalid JSON content from response: %s', content)
+      return False
+
+    LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason)
+
+    # Only retry on rate limit related failures.
+    if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ):
+      return True
+
+  # Everything else is a success or non-retriable so break.
+  return False
+
 
 def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
                    **kwargs):
@@ -82,21 +138,37 @@ def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args,
     resp, content - Response from the http request (may be HTTP 5xx).
   """
   resp = None
+  content = None
   for retry_num in range(num_retries + 1):
     if retry_num > 0:
-      sleep(rand() * 2**retry_num)
-      logging.warning(
-          'Retry #%d for %s: %s %s%s' % (retry_num, req_type, method, uri,
-          ', following status: %d' % resp.status if resp else ''))
+      # Sleep before retrying.
+      sleep_time = rand() * 2 ** retry_num
+      LOGGER.warning(
+          'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s',
+          sleep_time, retry_num, num_retries, req_type, method, uri,
+          resp.status if resp else exception)
+      sleep(sleep_time)
 
     try:
+      exception = None
       resp, content = http.request(uri, method, *args, **kwargs)
-    except ssl.SSLError:
-      if retry_num == num_retries:
+    # Retry on SSL errors and socket timeout errors.
+    except _ssl_SSLError as ssl_error:
+      exception = ssl_error
+    except socket.error as socket_error:
+      # errno's contents differ by platform, so we have to match by name.
+      if socket.errno.errorcode.get(socket_error.errno) not in (
+          'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED', ):
         raise
+      exception = socket_error
+
+    if exception:
+      if retry_num == num_retries:
+        raise exception
       else:
         continue
-    if resp.status < 500:
+
+    if not _should_retry_response(resp.status, content):
       break
 
   return resp, content
@@ -882,7 +954,7 @@ class HttpRequest(object):
     for retry_num in range(num_retries + 1):
       if retry_num > 0:
         self._sleep(self._rand() * 2**retry_num)
-        logging.warning(
+        LOGGER.warning(
             'Retry #%d for media upload: %s %s, following status: %d'
             % (retry_num, self.method, self.uri, resp.status))
 
@@ -1632,7 +1704,7 @@ def tunnel_patch(http):
       headers = {}
     if method == 'PATCH':
       if 'oauth_token' in headers.get('authorization', ''):
-        logging.warning(
+        LOGGER.warning(
             'OAuth 1.0 request made with Credentials after tunnel_patch.')
       headers['x-http-method-override'] = "PATCH"
       method = 'POST'

src/googleapiclient/model.py

@@ -33,6 +33,8 @@ from googleapiclient import __version__
 from googleapiclient.errors import HttpError
 
 
+LOGGER = logging.getLogger(__name__)
+
 dump_request_response = False
 
 
@@ -105,18 +107,18 @@ class BaseModel(Model):
   def _log_request(self, headers, path_params, query, body):
     """Logs debugging information about the request if requested."""
     if dump_request_response:
-      logging.info('--request-start--')
-      logging.info('-headers-start-')
+      LOGGER.info('--request-start--')
+      LOGGER.info('-headers-start-')
       for h, v in six.iteritems(headers):
-        logging.info('%s: %s', h, v)
-      logging.info('-headers-end-')
-      logging.info('-path-parameters-start-')
+        LOGGER.info('%s: %s', h, v)
+      LOGGER.info('-headers-end-')
+      LOGGER.info('-path-parameters-start-')
       for h, v in six.iteritems(path_params):
-        logging.info('%s: %s', h, v)
-      logging.info('-path-parameters-end-')
-      logging.info('body: %s', body)
-      logging.info('query: %s', query)
-      logging.info('--request-end--')
+        LOGGER.info('%s: %s', h, v)
+      LOGGER.info('-path-parameters-end-')
+      LOGGER.info('body: %s', body)
+      LOGGER.info('query: %s', query)
+      LOGGER.info('--request-end--')
 
   def request(self, headers, path_params, query_params, body_value):
     """Updates outgoing requests with a serialized body.
@@ -176,12 +178,12 @@ class BaseModel(Model):
   def _log_response(self, resp, content):
     """Logs debugging information about the response if requested."""
     if dump_request_response:
-      logging.info('--response-start--')
+      LOGGER.info('--response-start--')
       for h, v in six.iteritems(resp):
-        logging.info('%s: %s', h, v)
+        LOGGER.info('%s: %s', h, v)
       if content:
-        logging.info(content)
-      logging.info('--response-end--')
+        LOGGER.info(content)
+      LOGGER.info('--response-end--')
 
   def response(self, resp, content):
     """Convert the response wire format into a Python object.
@@ -206,7 +208,7 @@ class BaseModel(Model):
         return self.no_content_response
       return self.deserialize(content)
     else:
-      logging.debug('Content from bad request was: %s' % content)
+      LOGGER.debug('Content from bad request was: %s' % content)
       raise HttpError(resp, content)
 
   def serialize(self, body_value):

src/linux-build.sh

@@ -0,0 +1,10 @@
+rm -rf gam
+rm -rf build
+rm -rf  dist
+rm -rf gam-$1-linux-$(arch).tar.xz
+
+pyinstaller --clean -F --distpath=gam linux-gam.spec
+cp LICENSE gam
+cp whatsnew.txt gam
+
+tar cfJ gam-$1-linux-$(arch).tar.xz gam/ 

src/linux-gam.spec

@@ -0,0 +1,26 @@
+# -*- mode: python -*-
+a = Analysis(['gam.py'],
+             hiddenimports=[],
+             hookspath=None,
+             excludes=['_tkinter'],
+             runtime_hooks=None)
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')]
+a.datas += [('admin-settings-v2.json', 'admin-settings-v2.json', 'DATA')]
+a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
+a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')]
+a.datas += [('email-settings-v2.json', 'email-settings-v2.json', 'DATA')]
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )

src/macos-build.sh

@@ -0,0 +1,10 @@
+rmdir /q /s gam
+rmdir /q /s build
+rmdir /q /s dist
+rm -rf gam-$1-macos.tar.xz
+
+pyinstaller --clean -F --distpath=gam macos-gam.spec
+cp LICENSE gam
+cp whatsnew.txt gam
+
+tar cfJ gam-$1-macos.tar.xz gam/ 

src/macos-gam.spec

@@ -0,0 +1,26 @@
+# -*- mode: python -*-
+a = Analysis(['gam.py'],
+             hiddenimports=[],
+             hookspath=None,
+             excludes=['_tkinter'],
+             runtime_hooks=None)
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')]
+a.datas += [('admin-settings-v2.json', 'admin-settings-v2.json', 'DATA')]
+a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
+a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')]
+a.datas += [('email-settings-v2.json', 'email-settings-v2.json', 'DATA')]
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )

src/oauth2client/__init__.py

@@ -14,7 +14,7 @@
 
 """Client library for using OAuth2, especially with Google APIs."""
 
-__version__ = '2.0.1'
+__version__ = '3.0.0'
 
 GOOGLE_AUTH_URI = 'https://accounts.google.com/o/oauth2/v2/auth'
 GOOGLE_DEVICE_URI = 'https://accounts.google.com/o/oauth2/device/code'

src/oauth2client/_helpers.py

@@ -15,6 +15,7 @@
 
 import base64
 import json
+
 import six
 
 
@@ -67,7 +68,7 @@ def _to_bytes(value, encoding='ascii'):
     if isinstance(result, six.binary_type):
         return result
     else:
-        raise ValueError('%r could not be converted to bytes' % (value,))
+        raise ValueError('{0!r} could not be converted to bytes'.format(value))
 
 
 def _from_bytes(value):
@@ -88,7 +89,8 @@ def _from_bytes(value):
     if isinstance(result, six.text_type):
         return result
     else:
-        raise ValueError('%r could not be converted to unicode' % (value,))
+        raise ValueError(
+            '{0!r} could not be converted to unicode'.format(value))
 
 
 def _urlsafe_b64encode(raw_bytes):

src/oauth2client/_openssl_crypt.py

@@ -13,12 +13,9 @@
 # limitations under the License.
 """OpenSSL Crypto-related routines for oauth2client."""
 
-import base64
-
 from OpenSSL import crypto
 
-from oauth2client._helpers import _parse_pem_key
-from oauth2client._helpers import _to_bytes
+from oauth2client import _helpers
 
 
 class OpenSSLVerifier(object):
@@ -45,8 +42,8 @@ class OpenSSLVerifier(object):
             True if message was signed by the private key associated with the
             public key that this object was constructed with.
         """
-        message = _to_bytes(message, encoding='utf-8')
-        signature = _to_bytes(signature, encoding='utf-8')
+        message = _helpers._to_bytes(message, encoding='utf-8')
+        signature = _helpers._to_bytes(signature, encoding='utf-8')
         try:
             crypto.verify(self._pubkey, signature, message, 'sha256')
             return True
@@ -68,7 +65,7 @@ class OpenSSLVerifier(object):
         Raises:
             OpenSSL.crypto.Error: if the key_pem can't be parsed.
         """
-        key_pem = _to_bytes(key_pem)
+        key_pem = _helpers._to_bytes(key_pem)
         if is_x509_cert:
             pubkey = crypto.load_certificate(crypto.FILETYPE_PEM, key_pem)
         else:
@@ -96,7 +93,7 @@ class OpenSSLSigner(object):
         Returns:
             string, The signature of the message for the given key.
         """
-        message = _to_bytes(message, encoding='utf-8')
+        message = _helpers._to_bytes(message, encoding='utf-8')
         return crypto.sign(self._key, message, 'sha256')
 
     @staticmethod
@@ -113,12 +110,12 @@ class OpenSSLSigner(object):
         Raises:
             OpenSSL.crypto.Error if the key can't be parsed.
         """
-        key = _to_bytes(key)
-        parsed_pem_key = _parse_pem_key(key)
+        key = _helpers._to_bytes(key)
+        parsed_pem_key = _helpers._parse_pem_key(key)
         if parsed_pem_key:
             pkey = crypto.load_privatekey(crypto.FILETYPE_PEM, parsed_pem_key)
         else:
-            password = _to_bytes(password, encoding='utf-8')
+            password = _helpers._to_bytes(password, encoding='utf-8')
             pkey = crypto.load_pkcs12(key, password).get_privatekey()
         return OpenSSLSigner(pkey)
 
@@ -133,7 +130,7 @@ def pkcs12_key_as_pem(private_key_bytes, private_key_password):
     Returns:
         String. PEM contents of ``private_key_bytes``.
     """
-    private_key_password = _to_bytes(private_key_password)
+    private_key_password = _helpers._to_bytes(private_key_password)
     pkcs12 = crypto.load_pkcs12(private_key_bytes, private_key_password)
     return crypto.dump_privatekey(crypto.FILETYPE_PEM,
                                   pkcs12.get_privatekey())

src/oauth2client/_pure_python_crypt.py

@@ -26,8 +26,7 @@ from pyasn1_modules.rfc5208 import PrivateKeyInfo
 import rsa
 import six
 
-from oauth2client._helpers import _from_bytes
-from oauth2client._helpers import _to_bytes
+from oauth2client import _helpers
 
 
 _PKCS12_ERROR = r"""\
@@ -86,7 +85,7 @@ class RsaVerifier(object):
             True if message was signed by the private key associated with the
             public key that this object was constructed with.
         """
-        message = _to_bytes(message, encoding='utf-8')
+        message = _helpers._to_bytes(message, encoding='utf-8')
         try:
             return rsa.pkcs1.verify(message, signature, self._pubkey)
         except (ValueError, rsa.pkcs1.VerificationError):
@@ -111,7 +110,7 @@ class RsaVerifier(object):
                         "-----BEGIN CERTIFICATE-----" error, otherwise fails
                         to find "-----BEGIN RSA PUBLIC KEY-----".
         """
-        key_pem = _to_bytes(key_pem)
+        key_pem = _helpers._to_bytes(key_pem)
         if is_x509_cert:
             der = rsa.pem.load_pem(key_pem, 'CERTIFICATE')
             asn1_cert, remaining = decoder.decode(der, asn1Spec=Certificate())
@@ -145,7 +144,7 @@ class RsaSigner(object):
         Returns:
             string, The signature of the message for the given key.
         """
-        message = _to_bytes(message, encoding='utf-8')
+        message = _helpers._to_bytes(message, encoding='utf-8')
         return rsa.pkcs1.sign(message, self._key, 'SHA-256')
 
     @classmethod
@@ -164,7 +163,7 @@ class RsaSigner(object):
             ValueError if the key cannot be parsed as PKCS#1 or PKCS#8 in
             PEM format.
         """
-        key = _from_bytes(key)  # pem expects str in Py3
+        key = _helpers._from_bytes(key)  # pem expects str in Py3
         marker_id, key_bytes = pem.readPemBlocksFromFile(
             six.StringIO(key), _PKCS1_MARKER, _PKCS8_MARKER)
 

src/oauth2client/_pycrypto_crypt.py

@@ -13,14 +13,12 @@
 # limitations under the License.
 """pyCrypto Crypto-related routines for oauth2client."""
 
-from Crypto.PublicKey import RSA
 from Crypto.Hash import SHA256
+from Crypto.PublicKey import RSA
 from Crypto.Signature import PKCS1_v1_5
 from Crypto.Util.asn1 import DerSequence
 
-from oauth2client._helpers import _parse_pem_key
-from oauth2client._helpers import _to_bytes
-from oauth2client._helpers import _urlsafe_b64decode
+from oauth2client import _helpers
 
 
 class PyCryptoVerifier(object):
@@ -47,7 +45,7 @@ class PyCryptoVerifier(object):
             True if message was signed by the private key associated with the
             public key that this object was constructed with.
         """
-        message = _to_bytes(message, encoding='utf-8')
+        message = _helpers._to_bytes(message, encoding='utf-8')
         return PKCS1_v1_5.new(self._pubkey).verify(
             SHA256.new(message), signature)
 
@@ -64,9 +62,9 @@ class PyCryptoVerifier(object):
             Verifier instance.
         """
         if is_x509_cert:
-            key_pem = _to_bytes(key_pem)
+            key_pem = _helpers._to_bytes(key_pem)
             pemLines = key_pem.replace(b' ', b'').split()
-            certDer = _urlsafe_b64decode(b''.join(pemLines[1:-1]))
+            certDer = _helpers._urlsafe_b64decode(b''.join(pemLines[1:-1]))
             certSeq = DerSequence()
             certSeq.decode(certDer)
             tbsSeq = DerSequence()
@@ -97,7 +95,7 @@ class PyCryptoSigner(object):
         Returns:
             string, The signature of the message for the given key.
         """
-        message = _to_bytes(message, encoding='utf-8')
+        message = _helpers._to_bytes(message, encoding='utf-8')
         return PKCS1_v1_5.new(self._key).sign(SHA256.new(message))
 
     @staticmethod
@@ -115,7 +113,7 @@ class PyCryptoSigner(object):
         Raises:
             NotImplementedError if the key isn't in PEM format.
         """
-        parsed_pem_key = _parse_pem_key(_to_bytes(key))
+        parsed_pem_key = _helpers._parse_pem_key(_helpers._to_bytes(key))
         if parsed_pem_key:
             pkey = RSA.importKey(parsed_pem_key)
         else:

src/oauth2client/client.py

@@ -17,32 +17,25 @@
 Tools for interacting with OAuth 2.0 protected resources.
 """
 
-import base64
 import collections
 import copy
 import datetime
 import json
 import logging
 import os
+import shutil
 import socket
 import sys
 import tempfile
-import time
-import shutil
+
 import six
 from six.moves import http_client
 from six.moves import urllib
 
-import httplib2
-from oauth2client import GOOGLE_AUTH_URI
-from oauth2client import GOOGLE_DEVICE_URI
-from oauth2client import GOOGLE_REVOKE_URI
-from oauth2client import GOOGLE_TOKEN_URI
-from oauth2client import GOOGLE_TOKEN_INFO_URI
-from oauth2client._helpers import _from_bytes
-from oauth2client._helpers import _to_bytes
-from oauth2client._helpers import _urlsafe_b64decode
+import oauth2client
+from oauth2client import _helpers
 from oauth2client import clientsecrets
+from oauth2client import transport
 from oauth2client import util
 
 
@@ -53,9 +46,8 @@ HAS_CRYPTO = False
 try:
     from oauth2client import crypt
     HAS_CRYPTO = True
-    if crypt.OpenSSLVerifier is not None:
-        HAS_OPENSSL = True
-except ImportError:
+    HAS_OPENSSL = crypt.OpenSSLVerifier is not None
+except ImportError:  # pragma: NO COVER
     pass
 
 
@@ -73,9 +65,6 @@ ID_TOKEN_VERIFICATON_CERTS = ID_TOKEN_VERIFICATION_CERTS
 # Constant to use for the out of band OAuth 2.0 flow.
 OOB_CALLBACK_URN = 'urn:ietf:wg:oauth:2.0:oob'
 
-# Google Data client libraries may need to set this to [401, 403].
-REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,)
-
 # The value representing user credentials.
 AUTHORIZED_USER = 'authorized_user'
 
@@ -113,6 +102,14 @@ DEFAULT_ENV_NAME = 'UNKNOWN'
 # If set to True _get_environment avoid GCE check (_detect_gce_environment)
 NO_GCE_CHECK = os.environ.setdefault('NO_GCE_CHECK', 'False')
 
+# Timeout in seconds to wait for the GCE metadata server when detecting the
+# GCE environment.
+try:
+    GCE_METADATA_TIMEOUT = int(
+        os.environ.setdefault('GCE_METADATA_TIMEOUT', '3'))
+except ValueError:  # pragma: NO COVER
+    GCE_METADATA_TIMEOUT = 3
+
 _SERVER_SOFTWARE = 'SERVER_SOFTWARE'
 _GCE_METADATA_HOST = '169.254.169.254'
 _METADATA_FLAVOR_HEADER = 'Metadata-Flavor'
@@ -122,6 +119,12 @@ _DESIRED_METADATA_FLAVOR = 'Google'
 # easier testing (by replacing with a stub).
 _UTCNOW = datetime.datetime.utcnow
 
+# NOTE: These names were previously defined in this module but have been
+#       moved into `oauth2client.transport`,
+clean_headers = transport.clean_headers
+MemoryCache = transport.MemoryCache
+REFRESH_STATUS_CODES = transport.REFRESH_STATUS_CODES
+
 
 class SETTINGS(object):
     """Settings namespace for globally defined values."""
@@ -179,26 +182,6 @@ class CryptoUnavailableError(Error, NotImplementedError):
     """Raised when a crypto library is required, but none is available."""
 
 
-def _abstract():
-    raise NotImplementedError('You need to override this function')
-
-
-class MemoryCache(object):
-    """httplib2 Cache implementation which only caches locally."""
-
-    def __init__(self):
-        self.cache = {}
-
-    def get(self, key):
-        return self.cache.get(key)
-
-    def set(self, key, value):
-        self.cache[key] = value
-
-    def delete(self, key):
-        self.cache.pop(key, None)
-
-
 def _parse_expiry(expiry):
     if expiry and isinstance(expiry, datetime.datetime):
         return expiry.strftime(EXPIRY_FORMAT)
@@ -229,7 +212,7 @@ class Credentials(object):
             http: httplib2.Http, an http object to be used to make the refresh
                   request.
         """
-        _abstract()
+        raise NotImplementedError
 
     def refresh(self, http):
         """Forces a refresh of the access_token.
@@ -238,7 +221,7 @@ class Credentials(object):
             http: httplib2.Http, an http object to be used to make the refresh
                   request.
         """
-        _abstract()
+        raise NotImplementedError
 
     def revoke(self, http):
         """Revokes a refresh_token and makes the credentials void.
@@ -247,7 +230,7 @@ class Credentials(object):
             http: httplib2.Http, an http object to be used to make the revoke
                   request.
         """
-        _abstract()
+        raise NotImplementedError
 
     def apply(self, headers):
         """Add the authorization to the headers.
@@ -255,7 +238,7 @@ class Credentials(object):
         Args:
             headers: dict, the headers to add the Authorization header to.
         """
-        _abstract()
+        raise NotImplementedError
 
     def _to_json(self, strip, to_serialize=None):
         """Utility function that creates JSON repr. of a Credentials object.
@@ -264,8 +247,8 @@ class Credentials(object):
             strip: array, An array of names of members to exclude from the
                    JSON.
             to_serialize: dict, (Optional) The properties for this object
-                          that will be serialized. This allows callers to modify
-                          before serializing.
+                          that will be serialized. This allows callers to
+                          modify before serializing.
 
         Returns:
             string, a JSON representation of this instance, suitable to pass to
@@ -274,6 +257,9 @@ class Credentials(object):
         curr_type = self.__class__
         if to_serialize is None:
             to_serialize = copy.copy(self.__dict__)
+        else:
+            # Assumes it is a str->str dictionary, so we don't deep copy.
+            to_serialize = copy.copy(to_serialize)
         for member in strip:
             if member in to_serialize:
                 del to_serialize[member]
@@ -311,7 +297,7 @@ class Credentials(object):
             An instance of the subclass of Credentials that was serialized with
             to_json().
         """
-        json_data_as_unicode = _from_bytes(json_data)
+        json_data_as_unicode = _helpers._from_bytes(json_data)
         data = json.loads(json_data_as_unicode)
         # Find and call the right classmethod from_json() to restore
         # the object.
@@ -361,7 +347,8 @@ class Storage(object):
 
         Args:
             lock: An optional threading.Lock-like object. Must implement at
-                  least acquire() and release(). Does not need to be re-entrant.
+                  least acquire() and release(). Does not need to be
+                  re-entrant.
         """
         self._lock = lock
 
@@ -390,7 +377,7 @@ class Storage(object):
         Returns:
             oauth2client.client.Credentials
         """
-        _abstract()
+        raise NotImplementedError
 
     def locked_put(self, credentials):
         """Write a credential.
@@ -400,14 +387,14 @@ class Storage(object):
         Args:
             credentials: Credentials, the credentials to store.
         """
-        _abstract()
+        raise NotImplementedError
 
     def locked_delete(self):
         """Delete a credential.
 
         The Storage lock must be held when this is called.
         """
-        _abstract()
+        raise NotImplementedError
 
     def get(self):
         """Retrieve credential.
@@ -453,32 +440,6 @@ class Storage(object):
             self.release_lock()
 
 
-def clean_headers(headers):
-    """Forces header keys and values to be strings, i.e not unicode.
-
-    The httplib module just concats the header keys and values in a way that
-    may make the message header a unicode string, which, if it then tries to
-    contatenate to a binary request body may result in a unicode decode error.
-
-    Args:
-        headers: dict, A dictionary of headers.
-
-    Returns:
-        The same dictionary but with all the keys converted to strings.
-    """
-    clean = {}
-    try:
-        for k, v in six.iteritems(headers):
-            if not isinstance(k, six.binary_type):
-                k = str(k)
-            if not isinstance(v, six.binary_type):
-                v = str(v)
-            clean[_to_bytes(k)] = _to_bytes(v)
-    except UnicodeEncodeError:
-        raise NonAsciiHeaderError(k, ': ', v)
-    return clean
-
-
 def _update_query_params(uri, params):
     """Updates a URI with new query parameters.
 
@@ -586,67 +547,7 @@ class OAuth2Credentials(Credentials):
         that adds in the Authorization header and then calls the original
         version of 'request()'.
         """
-        request_orig = http.request
-
-        # The closure that will replace 'httplib2.Http.request'.
-        def new_request(uri, method='GET', body=None, headers=None,
-                        redirections=httplib2.DEFAULT_MAX_REDIRECTS,
-                        connection_type=None):
-            if not self.access_token:
-                logger.info('Attempting refresh to obtain '
-                            'initial access_token')
-                self._refresh(request_orig)
-
-            # Clone and modify the request headers to add the appropriate
-            # Authorization header.
-            if headers is None:
-                headers = {}
-            else:
-                headers = dict(headers)
-            self.apply(headers)
-
-            if self.user_agent is not None:
-                if 'user-agent' in headers:
-                    headers['user-agent'] = (self.user_agent + ' ' +
-                                             headers['user-agent'])
-                else:
-                    headers['user-agent'] = self.user_agent
-
-            body_stream_position = None
-            if all(getattr(body, stream_prop, None) for stream_prop in
-                   ('read', 'seek', 'tell')):
-                body_stream_position = body.tell()
-
-            resp, content = request_orig(uri, method, body,
-                                         clean_headers(headers),
-                                         redirections, connection_type)
-
-            # A stored token may expire between the time it is retrieved and
-            # the time the request is made, so we may need to try twice.
-            max_refresh_attempts = 2
-            for refresh_attempt in range(max_refresh_attempts):
-                if resp.status not in REFRESH_STATUS_CODES:
-                    break
-                logger.info('Refreshing due to a %s (attempt %s/%s)',
-                            resp.status, refresh_attempt + 1,
-                            max_refresh_attempts)
-                self._refresh(request_orig)
-                self.apply(headers)
-                if body_stream_position is not None:
-                    body.seek(body_stream_position)
-
-                resp, content = request_orig(uri, method, body,
-                                             clean_headers(headers),
-                                             redirections, connection_type)
-
-            return (resp, content)
-
-        # Replace the request method with our own closure.
-        http.request = new_request
-
-        # Set credentials as a property of the request method.
-        setattr(http.request, 'credentials', self)
-
+        transport.wrap_http_for_auth(self, http)
         return http
 
     def refresh(self, http):
@@ -721,7 +622,7 @@ class OAuth2Credentials(Credentials):
         Returns:
             An instance of a Credentials subclass.
         """
-        data = json.loads(_from_bytes(json_data))
+        data = json.loads(_helpers._from_bytes(json_data))
         if (data.get('token_expiry') and
                 not isinstance(data['token_expiry'], datetime.datetime)):
             try:
@@ -772,7 +673,7 @@ class OAuth2Credentials(Credentials):
         """
         if not self.access_token or self.access_token_expired:
             if not http:
-                http = httplib2.Http()
+                http = transport.get_http_object()
             self.refresh(http)
         return AccessTokenInfo(access_token=self.access_token,
                                expires_in=self._expires_in())
@@ -894,7 +795,7 @@ class OAuth2Credentials(Credentials):
         logger.info('Refreshing access_token')
         resp, content = http_request(
             self.token_uri, method='POST', body=body, headers=headers)
-        content = _from_bytes(content)
+        content = _helpers._from_bytes(content)
         if resp.status == http_client.OK:
             d = json.loads(content)
             self.token_response = d
@@ -918,7 +819,7 @@ class OAuth2Credentials(Credentials):
             # An {'error':...} response body means the token is expired or
             # revoked, so we flag the credentials as such.
             logger.info('Failed to retrieve access token: %s', content)
-            error_msg = 'Invalid response %s.' % resp['status']
+            error_msg = 'Invalid response {0}.'.format(resp['status'])
             try:
                 d = json.loads(content)
                 if 'error' in d:
@@ -926,7 +827,7 @@ class OAuth2Credentials(Credentials):
                     if 'error_description' in d:
                         error_msg += ': ' + d['error_description']
                     self.invalid = True
-                    if self.store:
+                    if self.store is not None:
                         self.store.locked_put(self)
             except (TypeError, ValueError):
                 pass
@@ -963,9 +864,9 @@ class OAuth2Credentials(Credentials):
         if resp.status == http_client.OK:
             self.invalid = True
         else:
-            error_msg = 'Invalid response %s.' % resp.status
+            error_msg = 'Invalid response {0}.'.format(resp.status)
             try:
-                d = json.loads(_from_bytes(content))
+                d = json.loads(_helpers._from_bytes(content))
                 if 'error' in d:
                     error_msg = d['error']
             except (TypeError, ValueError):
@@ -1004,12 +905,12 @@ class OAuth2Credentials(Credentials):
         token_info_uri = _update_query_params(self.token_info_uri,
                                               query_params)
         resp, content = http_request(token_info_uri)
-        content = _from_bytes(content)
+        content = _helpers._from_bytes(content)
         if resp.status == http_client.OK:
             d = json.loads(content)
             self.scopes = set(util.string_to_scopes(d.get('scope', '')))
         else:
-            error_msg = 'Invalid response %s.' % (resp.status,)
+            error_msg = 'Invalid response {0}.'.format(resp.status)
             try:
                 d = json.loads(content)
                 if 'error_description' in d:
@@ -1070,7 +971,7 @@ class AccessTokenCredentials(OAuth2Credentials):
 
     @classmethod
     def from_json(cls, json_data):
-        data = json.loads(_from_bytes(json_data))
+        data = json.loads(_helpers._from_bytes(json_data))
         retval = AccessTokenCredentials(
             data['access_token'],
             data['user_agent'])
@@ -1105,7 +1006,7 @@ def _detect_gce_environment():
     #       the metadata resolution was particularly slow. The latter case is
     #       "unlikely".
     connection = six.moves.http_client.HTTPConnection(
-        _GCE_METADATA_HOST, timeout=1)
+        _GCE_METADATA_HOST, timeout=GCE_METADATA_TIMEOUT)
 
     try:
         headers = {_METADATA_FLAVOR_HEADER: _DESIRED_METADATA_FLAVOR}
@@ -1186,14 +1087,14 @@ class GoogleCredentials(OAuth2Credentials):
         print(response)
     """
 
-    NON_SERIALIZED_MEMBERS =  (
+    NON_SERIALIZED_MEMBERS = (
         frozenset(['_private_key']) |
         OAuth2Credentials.NON_SERIALIZED_MEMBERS)
     """Members that aren't serialized when object is converted to JSON."""
 
     def __init__(self, access_token, client_id, client_secret, refresh_token,
                  token_expiry, token_uri, user_agent,
-                 revoke_uri=GOOGLE_REVOKE_URI):
+                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
         """Create an instance of GoogleCredentials.
 
         This constructor is not usually called by the user, instead
@@ -1211,8 +1112,8 @@ class GoogleCredentials(OAuth2Credentials):
             user_agent: string, The HTTP User-Agent to provide for this
                         application.
             revoke_uri: string, URI for revoke endpoint. Defaults to
-                        GOOGLE_REVOKE_URI; a token can't be revoked if this
-                        is None.
+                        oauth2client.GOOGLE_REVOKE_URI; a token can't be
+                        revoked if this is None.
         """
         super(GoogleCredentials, self).__init__(
             access_token, client_id, client_secret, refresh_token,
@@ -1237,14 +1138,17 @@ class GoogleCredentials(OAuth2Credentials):
     def from_json(cls, json_data):
         # TODO(issue 388): eliminate the circularity that is the reason for
         #                  this non-top-level import.
-        from oauth2client.service_account import ServiceAccountCredentials
-        data = json.loads(_from_bytes(json_data))
+        from oauth2client import service_account
+        data = json.loads(_helpers._from_bytes(json_data))
 
         # We handle service_account.ServiceAccountCredentials since it is a
         # possible return type of GoogleCredentials.get_application_default()
         if (data['_module'] == 'oauth2client.service_account' and
-            data['_class'] == 'ServiceAccountCredentials'):
-            return ServiceAccountCredentials.from_json(data)
+                data['_class'] == 'ServiceAccountCredentials'):
+            return service_account.ServiceAccountCredentials.from_json(data)
+        elif (data['_module'] == 'oauth2client.service_account' and
+                data['_class'] == '_JWTAccessCredentials'):
+            return service_account._JWTAccessCredentials.from_json(data)
 
         token_expiry = _parse_expiry(data.get('token_expiry'))
         google_credentials = cls(
@@ -1348,10 +1252,10 @@ class GoogleCredentials(OAuth2Credentials):
         """Gets credentials implicitly from the environment.
 
         Checks environment in order of precedence:
-        - Google App Engine (production and testing)
         - Environment variable GOOGLE_APPLICATION_CREDENTIALS pointing to
           a file with stored credentials information.
         - Stored "well known" file associated with `gcloud` command line tool.
+        - Google App Engine (production and testing)
         - Google Compute Engine production environment.
 
         Raises:
@@ -1360,8 +1264,8 @@ class GoogleCredentials(OAuth2Credentials):
         """
         # Environ checks (in order).
         environ_checkers = [
-            cls._implicit_credentials_from_gae,
             cls._implicit_credentials_from_files,
+            cls._implicit_credentials_from_gae,
             cls._implicit_credentials_from_gce,
         ]
 
@@ -1446,7 +1350,8 @@ def save_to_well_known_file(credentials, well_known_file=None):
 
     config_dir = os.path.dirname(well_known_file)
     if not os.path.isdir(config_dir):
-        raise OSError('Config directory does not exist: %s' % config_dir)
+        raise OSError(
+            'Config directory does not exist: {0}'.format(config_dir))
 
     credentials_data = credentials.serialization_data
     _save_private_file(well_known_file, credentials_data)
@@ -1454,8 +1359,7 @@ def save_to_well_known_file(credentials, well_known_file=None):
 
 def _get_environment_variable_file():
     application_default_credential_filename = (
-      os.environ.get(GOOGLE_APPLICATION_CREDENTIALS,
-                     None))
+        os.environ.get(GOOGLE_APPLICATION_CREDENTIALS, None))
 
     if application_default_credential_filename:
         if os.path.isfile(application_default_credential_filename):
@@ -1521,11 +1425,11 @@ def _get_application_default_credential_from_file(filename):
             client_secret=client_credentials['client_secret'],
             refresh_token=client_credentials['refresh_token'],
             token_expiry=None,
-            token_uri=GOOGLE_TOKEN_URI,
+            token_uri=oauth2client.GOOGLE_TOKEN_URI,
             user_agent='Python client library')
     else:  # client_credentials['type'] == SERVICE_ACCOUNT
-        from oauth2client.service_account import ServiceAccountCredentials
-        return ServiceAccountCredentials.from_json_keyfile_dict(
+        from oauth2client import service_account
+        return service_account._JWTAccessCredentials.from_json_keyfile_dict(
             client_credentials)
 
 
@@ -1538,8 +1442,8 @@ def _raise_exception_for_reading_json(credential_file,
                                       extra_help,
                                       error):
     raise ApplicationDefaultCredentialsError(
-      'An error was encountered while reading json file: ' +
-      credential_file + extra_help + ': ' + str(error))
+        'An error was encountered while reading json file: ' +
+        credential_file + extra_help + ': ' + str(error))
 
 
 def _get_application_default_credential_GAE():
@@ -1567,8 +1471,8 @@ class AssertionCredentials(GoogleCredentials):
 
     @util.positional(2)
     def __init__(self, assertion_type, user_agent=None,
-                 token_uri=GOOGLE_TOKEN_URI,
-                 revoke_uri=GOOGLE_REVOKE_URI,
+                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
                  **unused_kwargs):
         """Constructor for AssertionFlowCredentials.
 
@@ -1605,7 +1509,7 @@ class AssertionCredentials(GoogleCredentials):
 
     def _generate_assertion(self):
         """Generate assertion string to be used in the access token request."""
-        _abstract()
+        raise NotImplementedError
 
     def _revoke(self, http_request):
         """Revokes the access_token and deletes the store if available.
@@ -1630,7 +1534,7 @@ class AssertionCredentials(GoogleCredentials):
         raise NotImplementedError('This method is abstract.')
 
 
-def _RequireCryptoOrDie():
+def _require_crypto_or_die():
     """Ensure we have a crypto library, or throw CryptoUnavailableError.
 
     The oauth2client.crypt module requires either PyCrypto or PyOpenSSL
@@ -1641,11 +1545,6 @@ def _RequireCryptoOrDie():
         raise CryptoUnavailableError('No crypto library available')
 
 
-# Only used in verify_id_token(), which is always calling to the same URI
-# for the certs.
-_cached_http = httplib2.Http(MemoryCache())
-
-
 @util.positional(2)
 def verify_id_token(id_token, audience, http=None,
                     cert_uri=ID_TOKEN_VERIFICATION_CERTS):
@@ -1669,16 +1568,16 @@ def verify_id_token(id_token, audience, http=None,
         oauth2client.crypt.AppIdentityError: if the JWT fails to verify.
         CryptoUnavailableError: if no crypto library is available.
     """
-    _RequireCryptoOrDie()
+    _require_crypto_or_die()
     if http is None:
-        http = _cached_http
+        http = transport.get_cached_http()
 
     resp, content = http.request(cert_uri)
     if resp.status == http_client.OK:
-        certs = json.loads(_from_bytes(content))
+        certs = json.loads(_helpers._from_bytes(content))
         return crypt.verify_signed_jwt_with_certs(id_token, certs, audience)
     else:
-        raise VerifyJwtTokenError('Status code: %d' % resp.status)
+        raise VerifyJwtTokenError('Status code: {0}'.format(resp.status))
 
 
 def _extract_id_token(id_token):
@@ -1699,9 +1598,10 @@ def _extract_id_token(id_token):
 
     if len(segments) != 3:
         raise VerifyJwtTokenError(
-            'Wrong number of segments in token: %s' % id_token)
+            'Wrong number of segments in token: {0}'.format(id_token))
 
-    return json.loads(_from_bytes(_urlsafe_b64decode(segments[1])))
+    return json.loads(
+        _helpers._from_bytes(_helpers._urlsafe_b64decode(segments[1])))
 
 
 def _parse_exchange_token_response(content):
@@ -1718,7 +1618,7 @@ def _parse_exchange_token_response(content):
         i.e. {}. That basically indicates a failure.
     """
     resp = {}
-    content = _from_bytes(content)
+    content = _helpers._from_bytes(content)
     try:
         resp = json.loads(content)
     except Exception:
@@ -1736,11 +1636,12 @@ def _parse_exchange_token_response(content):
 @util.positional(4)
 def credentials_from_code(client_id, client_secret, scope, code,
                           redirect_uri='postmessage', http=None,
-                          user_agent=None, token_uri=GOOGLE_TOKEN_URI,
-                          auth_uri=GOOGLE_AUTH_URI,
-                          revoke_uri=GOOGLE_REVOKE_URI,
-                          device_uri=GOOGLE_DEVICE_URI,
-                          token_info_uri=GOOGLE_TOKEN_INFO_URI):
+                          user_agent=None,
+                          token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                          auth_uri=oauth2client.GOOGLE_AUTH_URI,
+                          revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
+                          device_uri=oauth2client.GOOGLE_DEVICE_URI,
+                          token_info_uri=oauth2client.GOOGLE_TOKEN_INFO_URI):
     """Exchanges an authorization code for an OAuth2Credentials object.
 
     Args:
@@ -1864,11 +1765,38 @@ class DeviceFlowInfo(collections.namedtuple('DeviceFlowInfo', (
         })
         if 'expires_in' in response:
             kwargs['user_code_expiry'] = (
-                datetime.datetime.now() +
+                _UTCNOW() +
                 datetime.timedelta(seconds=int(response['expires_in'])))
         return cls(**kwargs)
 
 
+def _oauth2_web_server_flow_params(kwargs):
+    """Configures redirect URI parameters for OAuth2WebServerFlow."""
+    params = {
+        'access_type': 'offline',
+        'response_type': 'code',
+    }
+
+    params.update(kwargs)
+
+    # Check for the presence of the deprecated approval_prompt param and
+    # warn appropriately.
+    approval_prompt = params.get('approval_prompt')
+    if approval_prompt is not None:
+        logger.warning(
+            'The approval_prompt parameter for OAuth2WebServerFlow is '
+            'deprecated. Please use the prompt parameter instead.')
+
+        if approval_prompt == 'force':
+            logger.warning(
+                'approval_prompt="force" has been adjusted to '
+                'prompt="consent"')
+            params['prompt'] = 'consent'
+            del params['approval_prompt']
+
+    return params
+
+
 class OAuth2WebServerFlow(Flow):
     """Does the Web Server Flow for OAuth 2.0.
 
@@ -1881,18 +1809,18 @@ class OAuth2WebServerFlow(Flow):
                  scope=None,
                  redirect_uri=None,
                  user_agent=None,
-                 auth_uri=GOOGLE_AUTH_URI,
-                 token_uri=GOOGLE_TOKEN_URI,
-                 revoke_uri=GOOGLE_REVOKE_URI,
+                 auth_uri=oauth2client.GOOGLE_AUTH_URI,
+                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
                  login_hint=None,
-                 device_uri=GOOGLE_DEVICE_URI,
-                 token_info_uri=GOOGLE_TOKEN_INFO_URI,
+                 device_uri=oauth2client.GOOGLE_DEVICE_URI,
+                 token_info_uri=oauth2client.GOOGLE_TOKEN_INFO_URI,
                  authorization_header=None,
                  **kwargs):
         """Constructor for OAuth2WebServerFlow.
 
         The kwargs argument is used to set extra query parameters on the
-        auth_uri. For example, the access_type and approval_prompt
+        auth_uri. For example, the access_type and prompt
         query parameters can be set via kwargs.
 
         Args:
@@ -1944,11 +1872,7 @@ class OAuth2WebServerFlow(Flow):
         self.device_uri = device_uri
         self.token_info_uri = token_info_uri
         self.authorization_header = authorization_header
-        self.params = {
-            'access_type': 'offline',
-            'response_type': 'code',
-        }
-        self.params.update(kwargs)
+        self.params = _oauth2_web_server_flow_params(kwargs)
 
     @util.positional(1)
     def step1_get_authorize_url(self, redirect_uri=None, state=None):
@@ -2014,25 +1938,25 @@ class OAuth2WebServerFlow(Flow):
             headers['user-agent'] = self.user_agent
 
         if http is None:
-            http = httplib2.Http()
+            http = transport.get_http_object()
 
         resp, content = http.request(self.device_uri, method='POST', body=body,
                                      headers=headers)
-        content = _from_bytes(content)
+        content = _helpers._from_bytes(content)
         if resp.status == http_client.OK:
             try:
                 flow_info = json.loads(content)
-            except ValueError as e:
+            except ValueError as exc:
                 raise OAuth2DeviceCodeError(
-                    'Could not parse server response as JSON: "%s", '
-                    'error: "%s"' % (content, e))
+                    'Could not parse server response as JSON: "{0}", '
+                    'error: "{1}"'.format(content, exc))
             return DeviceFlowInfo.FromResponse(flow_info)
         else:
-            error_msg = 'Invalid response %s.' % resp.status
+            error_msg = 'Invalid response {0}.'.format(resp.status)
             try:
-                d = json.loads(content)
-                if 'error' in d:
-                    error_msg += ' Error: %s' % d['error']
+                error_dict = json.loads(content)
+                if 'error' in error_dict:
+                    error_msg += ' Error: {0}'.format(error_dict['error'])
             except ValueError:
                 # Couldn't decode a JSON response, stick with the
                 # default message.
@@ -2069,7 +1993,7 @@ class OAuth2WebServerFlow(Flow):
 
         if code is None:
             code = device_flow_info.device_code
-        elif not isinstance(code, six.string_types):
+        elif not isinstance(code, (six.string_types, six.binary_type)):
             if 'code' not in code:
                 raise FlowExchangeError(code.get(
                     'error', 'No code was supplied in the query parameters.'))
@@ -2097,7 +2021,7 @@ class OAuth2WebServerFlow(Flow):
             headers['user-agent'] = self.user_agent
 
         if http is None:
-            http = httplib2.Http()
+            http = transport.get_http_object()
 
         resp, content = http.request(self.token_uri, method='POST', body=body,
                                      headers=headers)
@@ -2108,7 +2032,7 @@ class OAuth2WebServerFlow(Flow):
             if not refresh_token:
                 logger.info(
                     'Received token response with no refresh_token. Consider '
-                    "reauthenticating with approval_prompt='force'.")
+                    "reauthenticating with prompt='consent'.")
             token_expiry = None
             if 'expires_in' in d:
                 delta = datetime.timedelta(seconds=int(d['expires_in']))
@@ -2132,7 +2056,7 @@ class OAuth2WebServerFlow(Flow):
                 error_msg = (str(d['error']) +
                              str(d.get('error_description', '')))
             else:
-                error_msg = 'Invalid response: %s.' % str(resp.status)
+                error_msg = 'Invalid response: {0}.'.format(str(resp.status))
             raise FlowExchangeError(error_msg)
 
 
@@ -2196,11 +2120,14 @@ def flow_from_clientsecrets(filename, scope, redirect_uri=None,
                 client_info['client_id'], client_info['client_secret'],
                 scope, **constructor_kwargs)
 
-    except clientsecrets.InvalidClientSecretsError:
-        if message:
+    except clientsecrets.InvalidClientSecretsError as e:
+        if message is not None:
+            if e.args:
+                message = ('The client secrets were invalid: '
+                           '\n{0}\n{1}'.format(e, message))
             sys.exit(message)
         else:
             raise
     else:
         raise UnknownClientSecretsFlowError(
-            'This OAuth 2.0 flow is unsupported: %r' % client_type)
+            'This OAuth 2.0 flow is unsupported: {0!r}'.format(client_type))

src/oauth2client/clientsecrets.py

@@ -19,8 +19,8 @@ an OAuth 2.0 protected service.
 """
 
 import json
-import six
 
+import six
 
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
 
@@ -93,17 +93,17 @@ def _validate_clientsecrets(clientsecrets_dict):
 
     if client_type not in VALID_CLIENT:
         raise InvalidClientSecretsError(
-            'Unknown client type: %s.' % (client_type,))
+            'Unknown client type: {0}.'.format(client_type))
 
     for prop_name in VALID_CLIENT[client_type]['required']:
         if prop_name not in client_info:
             raise InvalidClientSecretsError(
-                'Missing property "%s" in a client type of "%s".' %
-                (prop_name, client_type))
+                'Missing property "{0}" in a client type of "{1}".'.format(
+                    prop_name, client_type))
     for prop_name in VALID_CLIENT[client_type]['string']:
         if client_info[prop_name].startswith('[['):
             raise InvalidClientSecretsError(
-                'Property "%s" is not configured.' % prop_name)
+                'Property "{0}" is not configured.'.format(prop_name))
     return client_type, client_info
 
 

src/oauth2client/contrib/_appengine_ndb.py

@@ -76,9 +76,9 @@ class FlowNDBProperty(ndb.PickleProperty):
         """
         _LOGGER.info('validate: Got type %s', type(value))
         if value is not None and not isinstance(value, client.Flow):
-            raise TypeError('Property %s must be convertible to a flow '
-                            'instance; received: %s.' % (self._name,
-                                                         value))
+            raise TypeError(
+                'Property {0} must be convertible to a flow '
+                'instance; received: {1}.'.format(self._name, value))
 
 
 class CredentialsNDBProperty(ndb.BlobProperty):
@@ -104,9 +104,9 @@ class CredentialsNDBProperty(ndb.BlobProperty):
         """
         _LOGGER.info('validate: Got type %s', type(value))
         if value is not None and not isinstance(value, client.Credentials):
-            raise TypeError('Property %s must be convertible to a '
-                            'credentials instance; received: %s.' %
-                            (self._name, value))
+            raise TypeError(
+                'Property {0} must be convertible to a credentials '
+                'instance; received: {1}.'.format(self._name, value))
 
     def _to_base_type(self, value):
         """Converts our validated value to a JSON serialized string.

src/oauth2client/contrib/_fcntl_opener.py

@@ -0,0 +1,81 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import fcntl
+import time
+
+from oauth2client.contrib import locked_file
+
+
+class _FcntlOpener(locked_file._Opener):
+    """Open, lock, and unlock a file using fcntl.lockf."""
+
+    def open_and_lock(self, timeout, delay):
+        """Open the file and lock it.
+
+        Args:
+            timeout: float, How long to try to lock for.
+            delay: float, How long to wait between retries
+
+        Raises:
+            AlreadyLockedException: if the lock is already acquired.
+            IOError: if the open fails.
+            CredentialsFileSymbolicLinkError: if the file is a symbolic
+                                              link.
+        """
+        if self._locked:
+            raise locked_file.AlreadyLockedException(
+                'File {0} is already locked'.format(self._filename))
+        start_time = time.time()
+
+        locked_file.validate_file(self._filename)
+        try:
+            self._fh = open(self._filename, self._mode)
+        except IOError as e:
+            # If we can't access with _mode, try _fallback_mode and
+            # don't lock.
+            if e.errno in (errno.EPERM, errno.EACCES):
+                self._fh = open(self._filename, self._fallback_mode)
+                return
+
+        # We opened in _mode, try to lock the file.
+        while True:
+            try:
+                fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
+                self._locked = True
+                return
+            except IOError as e:
+                # If not retrying, then just pass on the error.
+                if timeout == 0:
+                    raise
+                if e.errno != errno.EACCES:
+                    raise
+                # We could not acquire the lock. Try again.
+                if (time.time() - start_time) >= timeout:
+                    locked_file.logger.warn('Could not lock %s in %s seconds',
+                                            self._filename, timeout)
+                    if self._fh:
+                        self._fh.close()
+                    self._fh = open(self._filename, self._fallback_mode)
+                    return
+                time.sleep(delay)
+
+    def unlock_and_close(self):
+        """Close and unlock the file using the fcntl.lockf primitive."""
+        if self._locked:
+            fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
+        self._locked = False
+        if self._fh:
+            self._fh.close()

src/oauth2client/contrib/_metadata.py

@@ -0,0 +1,123 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Provides helper methods for talking to the Compute Engine metadata server.
+
+See https://cloud.google.com/compute/docs/metadata
+"""
+
+import datetime
+import json
+
+import httplib2
+from six.moves import http_client
+from six.moves.urllib import parse as urlparse
+
+from oauth2client import _helpers
+from oauth2client import client
+from oauth2client import util
+
+
+METADATA_ROOT = 'http://metadata.google.internal/computeMetadata/v1/'
+METADATA_HEADERS = {'Metadata-Flavor': 'Google'}
+
+
+def get(http_request, path, root=METADATA_ROOT, recursive=None):
+    """Fetch a resource from the metadata server.
+
+    Args:
+        path: A string indicating the resource to retrieve. For example,
+            'instance/service-accounts/defualt'
+        http_request: A callable that matches the method
+            signature of httplib2.Http.request. Used to make the request to the
+            metadataserver.
+        root: A string indicating the full path to the metadata server root.
+        recursive: A boolean indicating whether to do a recursive query of
+            metadata. See
+            https://cloud.google.com/compute/docs/metadata#aggcontents
+
+    Returns:
+        A dictionary if the metadata server returns JSON, otherwise a string.
+
+    Raises:
+        httplib2.Httplib2Error if an error corrured while retrieving metadata.
+    """
+    url = urlparse.urljoin(root, path)
+    url = util._add_query_parameter(url, 'recursive', recursive)
+
+    response, content = http_request(
+        url,
+        headers=METADATA_HEADERS
+    )
+
+    if response.status == http_client.OK:
+        decoded = _helpers._from_bytes(content)
+        if response['content-type'] == 'application/json':
+            return json.loads(decoded)
+        else:
+            return decoded
+    else:
+        raise httplib2.HttpLib2Error(
+            'Failed to retrieve {0} from the Google Compute Engine'
+            'metadata service. Response:\n{1}'.format(url, response))
+
+
+def get_service_account_info(http_request, service_account='default'):
+    """Get information about a service account from the metadata server.
+
+    Args:
+        service_account: An email specifying the service account for which to
+            look up information. Default will be information for the "default"
+            service account of the current compute engine instance.
+        http_request: A callable that matches the method
+            signature of httplib2.Http.request. Used to make the request to the
+            metadata server.
+    Returns:
+         A dictionary with information about the specified service account,
+         for example:
+
+            {
+                'email': '...',
+                'scopes': ['scope', ...],
+                'aliases': ['default', '...']
+            }
+    """
+    return get(
+        http_request,
+        'instance/service-accounts/{0}/'.format(service_account),
+        recursive=True)
+
+
+def get_token(http_request, service_account='default'):
+    """Fetch an oauth token for the
+
+    Args:
+        service_account: An email specifying the service account this token
+            should represent. Default will be a token for the "default" service
+            account of the current compute engine instance.
+        http_request: A callable that matches the method
+            signature of httplib2.Http.request. Used to make the request to the
+            metadataserver.
+
+    Returns:
+         A tuple of (access token, token expiration), where access token is the
+         access token as a string and token expiration is a datetime object
+         that indicates when the access token will expire.
+    """
+    token_json = get(
+        http_request,
+        'instance/service-accounts/{0}/token'.format(service_account))
+    token_expiry = client._UTCNOW() + datetime.timedelta(
+        seconds=token_json['expires_in'])
+    return token_json['access_token'], token_expiry

src/oauth2client/contrib/_win32_opener.py

@@ -0,0 +1,106 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import errno
+import time
+
+import pywintypes
+import win32con
+import win32file
+
+from oauth2client.contrib import locked_file
+
+
+class _Win32Opener(locked_file._Opener):
+    """Open, lock, and unlock a file using windows primitives."""
+
+    # Error #33:
+    #  'The process cannot access the file because another process'
+    FILE_IN_USE_ERROR = 33
+
+    # Error #158:
+    #  'The segment is already unlocked.'
+    FILE_ALREADY_UNLOCKED_ERROR = 158
+
+    def open_and_lock(self, timeout, delay):
+        """Open the file and lock it.
+
+        Args:
+            timeout: float, How long to try to lock for.
+            delay: float, How long to wait between retries
+
+        Raises:
+            AlreadyLockedException: if the lock is already acquired.
+            IOError: if the open fails.
+            CredentialsFileSymbolicLinkError: if the file is a symbolic
+                                              link.
+        """
+        if self._locked:
+            raise locked_file.AlreadyLockedException(
+                'File {0} is already locked'.format(self._filename))
+        start_time = time.time()
+
+        locked_file.validate_file(self._filename)
+        try:
+            self._fh = open(self._filename, self._mode)
+        except IOError as e:
+            # If we can't access with _mode, try _fallback_mode
+            # and don't lock.
+            if e.errno == errno.EACCES:
+                self._fh = open(self._filename, self._fallback_mode)
+                return
+
+        # We opened in _mode, try to lock the file.
+        while True:
+            try:
+                hfile = win32file._get_osfhandle(self._fh.fileno())
+                win32file.LockFileEx(
+                    hfile,
+                    (win32con.LOCKFILE_FAIL_IMMEDIATELY |
+                     win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
+                    pywintypes.OVERLAPPED())
+                self._locked = True
+                return
+            except pywintypes.error as e:
+                if timeout == 0:
+                    raise
+
+                # If the error is not that the file is already
+                # in use, raise.
+                if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
+                    raise
+
+                # We could not acquire the lock. Try again.
+                if (time.time() - start_time) >= timeout:
+                    locked_file.logger.warn('Could not lock %s in %s seconds',
+                                            self._filename, timeout)
+                    if self._fh:
+                        self._fh.close()
+                    self._fh = open(self._filename, self._fallback_mode)
+                    return
+                time.sleep(delay)
+
+    def unlock_and_close(self):
+        """Close and unlock the file using the win32 primitive."""
+        if self._locked:
+            try:
+                hfile = win32file._get_osfhandle(self._fh.fileno())
+                win32file.UnlockFileEx(hfile, 0, -0x10000,
+                                       pywintypes.OVERLAPPED())
+            except pywintypes.error as e:
+                if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
+                    raise
+        self._locked = False
+        if self._fh:
+            self._fh.close()

src/oauth2client/contrib/appengine.py

@@ -24,26 +24,18 @@ import os
 import pickle
 import threading
 
-import httplib2
-import webapp2 as webapp
-
 from google.appengine.api import app_identity
 from google.appengine.api import memcache
 from google.appengine.api import users
 from google.appengine.ext import db
 from google.appengine.ext.webapp.util import login_required
+import httplib2
+import webapp2 as webapp
 
-from oauth2client import GOOGLE_AUTH_URI
-from oauth2client import GOOGLE_REVOKE_URI
-from oauth2client import GOOGLE_TOKEN_URI
+import oauth2client
+from oauth2client import client
 from oauth2client import clientsecrets
 from oauth2client import util
-from oauth2client.client import AccessTokenRefreshError
-from oauth2client.client import AssertionCredentials
-from oauth2client.client import Credentials
-from oauth2client.client import Flow
-from oauth2client.client import OAuth2WebServerFlow
-from oauth2client.client import Storage
 from oauth2client.contrib import xsrfutil
 
 # This is a temporary fix for a Google internal issue.
@@ -61,7 +53,7 @@ OAUTH2CLIENT_NAMESPACE = 'oauth2client#ns'
 
 XSRF_MEMCACHE_ID = 'xsrf_secret_key'
 
-if _appengine_ndb is None:
+if _appengine_ndb is None:  # pragma: NO COVER
     CredentialsNDBModel = None
     CredentialsNDBProperty = None
     FlowNDBProperty = None
@@ -89,14 +81,6 @@ def _safe_html(s):
     return cgi.escape(s, quote=1).replace("'", ''')
 
 
-class InvalidClientSecretsError(Exception):
-    """The client_secrets.json file is malformed or missing required fields."""
-
-
-class InvalidXsrfTokenError(Exception):
-    """The XSRF token is invalid or expired."""
-
-
 class SiteXsrfSecretKey(db.Model):
     """Storage for the sites XSRF secret key.
 
@@ -134,7 +118,7 @@ def xsrf_secret_key():
     return str(secret)
 
 
-class AppAssertionCredentials(AssertionCredentials):
+class AppAssertionCredentials(client.AssertionCredentials):
     """Credentials object for App Engine Assertion Grants
 
     This object will allow an App Engine application to identify itself to
@@ -193,7 +177,7 @@ class AppAssertionCredentials(AssertionCredentials):
             (token, _) = app_identity.get_access_token(
                 scopes, service_account_id=self.service_account_id)
         except app_identity.Error as e:
-            raise AccessTokenRefreshError(str(e))
+            raise client.AccessTokenRefreshError(str(e))
         self.access_token = token
 
     @property
@@ -244,7 +228,7 @@ class FlowProperty(db.Property):
     """
 
     # Tell what the user type is.
-    data_type = Flow
+    data_type = client.Flow
 
     # For writing to datastore.
     def get_value_for_datastore(self, model_instance):
@@ -259,10 +243,10 @@ class FlowProperty(db.Property):
         return pickle.loads(value)
 
     def validate(self, value):
-        if value is not None and not isinstance(value, Flow):
-            raise db.BadValueError('Property %s must be convertible '
-                                   'to a FlowThreeLegged instance (%s)' %
-                                   (self.name, value))
+        if value is not None and not isinstance(value, client.Flow):
+            raise db.BadValueError(
+                'Property {0} must be convertible '
+                'to a FlowThreeLegged instance ({1})'.format(self.name, value))
         return super(FlowProperty, self).validate(value)
 
     def empty(self, value):
@@ -273,11 +257,11 @@ class CredentialsProperty(db.Property):
     """App Engine datastore Property for Credentials.
 
     Utility property that allows easy storage and retrieval of
-    oath2client.Credentials
+    oauth2client.Credentials
     """
 
     # Tell what the user type is.
-    data_type = Credentials
+    data_type = client.Credentials
 
     # For writing to datastore.
     def get_value_for_datastore(self, model_instance):
@@ -298,7 +282,7 @@ class CredentialsProperty(db.Property):
         if len(value) == 0:
             return None
         try:
-            credentials = Credentials.new_from_json(value)
+            credentials = client.Credentials.new_from_json(value)
         except ValueError:
             credentials = None
         return credentials
@@ -306,14 +290,14 @@ class CredentialsProperty(db.Property):
     def validate(self, value):
         value = super(CredentialsProperty, self).validate(value)
         logger.info("validate: Got type " + str(type(value)))
-        if value is not None and not isinstance(value, Credentials):
-            raise db.BadValueError('Property %s must be convertible '
-                                   'to a Credentials instance (%s)' %
-                                   (self.name, value))
+        if value is not None and not isinstance(value, client.Credentials):
+            raise db.BadValueError(
+                'Property {0} must be convertible '
+                'to a Credentials instance ({1})'.format(self.name, value))
         return value
 
 
-class StorageByKeyName(Storage):
+class StorageByKeyName(client.Storage):
     """Store and retrieve a credential to and from the App Engine datastore.
 
     This Storage helper presumes the Credentials have been stored as a
@@ -365,8 +349,8 @@ class StorageByKeyName(Storage):
             elif issubclass(self._model, db.Model):
                 return False
 
-        raise TypeError('Model class not an NDB or DB model: %s.' %
-                        (self._model,))
+        raise TypeError(
+            'Model class not an NDB or DB model: {0}.'.format(self._model))
 
     def _get_entity(self):
         """Retrieve entity from datastore.
@@ -405,7 +389,7 @@ class StorageByKeyName(Storage):
         if self._cache:
             json = self._cache.get(self._key_name)
             if json:
-                credentials = Credentials.new_from_json(json)
+                credentials = client.Credentials.new_from_json(json)
         if credentials is None:
             entity = self._get_entity()
             if entity is not None:
@@ -476,18 +460,15 @@ def _parse_state_value(state, user):
         state: string, The value of the state parameter.
         user: google.appengine.api.users.User, The current user.
 
-    Raises:
-        InvalidXsrfTokenError: if the XSRF token is invalid.
-
     Returns:
-        The redirect URI.
+        The redirect URI, or None if XSRF token is not valid.
     """
     uri, token = state.rsplit(':', 1)
-    if not xsrfutil.validate_token(xsrf_secret_key(), token, user.user_id(),
-                                   action_id=uri):
-        raise InvalidXsrfTokenError()
-
-    return uri
+    if xsrfutil.validate_token(xsrf_secret_key(), token, user.user_id(),
+                               action_id=uri):
+        return uri
+    else:
+        return None
 
 
 class OAuth2Decorator(object):
@@ -544,9 +525,9 @@ class OAuth2Decorator(object):
 
     @util.positional(4)
     def __init__(self, client_id, client_secret, scope,
-                 auth_uri=GOOGLE_AUTH_URI,
-                 token_uri=GOOGLE_TOKEN_URI,
-                 revoke_uri=GOOGLE_REVOKE_URI,
+                 auth_uri=oauth2client.GOOGLE_AUTH_URI,
+                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
                  user_agent=None,
                  message=None,
                  callback_path='/oauth2callback',
@@ -665,7 +646,7 @@ class OAuth2Decorator(object):
                 return request_handler.redirect(self.authorize_url())
             try:
                 resp = method(request_handler, *args, **kwargs)
-            except AccessTokenRefreshError:
+            except client.AccessTokenRefreshError:
                 return request_handler.redirect(self.authorize_url())
             finally:
                 self.credentials = None
@@ -686,7 +667,7 @@ class OAuth2Decorator(object):
         if self.flow is None:
             redirect_uri = request_handler.request.relative_url(
                 self._callback_path)  # Usually /oauth2callback
-            self.flow = OAuth2WebServerFlow(
+            self.flow = client.OAuth2WebServerFlow(
                 self._client_id, self._client_secret, self._scope,
                 redirect_uri=redirect_uri, user_agent=self._user_agent,
                 auth_uri=self._auth_uri, token_uri=self._token_uri,
@@ -802,8 +783,8 @@ class OAuth2Decorator(object):
                 if error:
                     errormsg = self.request.get('error_description', error)
                     self.response.out.write(
-                        'The authorization request failed: %s' %
-                        _safe_html(errormsg))
+                        'The authorization request failed: {0}'.format(
+                            _safe_html(errormsg)))
                 else:
                     user = users.get_current_user()
                     decorator._create_flow(self)
@@ -815,6 +796,10 @@ class OAuth2Decorator(object):
                         user=user).put(credentials)
                     redirect_uri = _parse_state_value(
                         str(self.request.get('state')), user)
+                    if redirect_uri is None:
+                        self.response.out.write(
+                            'The authorization request failed')
+                        return
 
                     if (decorator._token_response_param and
                             credentials.token_response):
@@ -885,7 +870,7 @@ class OAuth2DecoratorFromClientSecrets(OAuth2Decorator):
                                                           cache=cache)
         if client_type not in (clientsecrets.TYPE_WEB,
                                clientsecrets.TYPE_INSTALLED):
-            raise InvalidClientSecretsError(
+            raise clientsecrets.InvalidClientSecretsError(
                 "OAuth2Decorator doesn't support this OAuth 2.0 flow.")
 
         constructor_kwargs = dict(kwargs)

src/oauth2client/contrib/devshell.py

@@ -19,13 +19,9 @@ import json
 import os
 import socket
 
-from oauth2client._helpers import _to_bytes
+from oauth2client import _helpers
 from oauth2client import client
 
-# Expose utcnow() at module level to allow for
-# easier testing (by replacing with a stub).
-_UTCNOW = datetime.datetime.utcnow
-
 DEVSHELL_ENV = 'DEVSHELL_CLIENT_PORT'
 
 
@@ -83,8 +79,8 @@ def _SendRecv():
     sock.connect(('localhost', port))
 
     data = CREDENTIAL_INFO_REQUEST_JSON
-    msg = '%s\n%s' % (len(data), data)
-    sock.sendall(_to_bytes(msg, encoding='utf-8'))
+    msg = '{0}\n{1}'.format(len(data), data)
+    sock.sendall(_helpers._to_bytes(msg, encoding='utf-8'))
 
     header = sock.recv(6).decode()
     if '\n' not in header:
@@ -127,7 +123,7 @@ class DevshellCredentials(client.GoogleCredentials):
         expires_in = self.devshell_response.expires_in
         if expires_in is not None:
             delta = datetime.timedelta(seconds=expires_in)
-            self.token_expiry = _UTCNOW() + delta
+            self.token_expiry = client._UTCNOW() + delta
         else:
             self.token_expiry = None
 

src/oauth2client/contrib/dictionary_storage.py

@@ -14,11 +14,10 @@
 
 """Dictionary storage for OAuth2 Credentials."""
 
-from oauth2client.client import OAuth2Credentials
-from oauth2client.client import Storage
+from oauth2client import client
 
 
-class DictionaryStorage(Storage):
+class DictionaryStorage(client.Storage):
     """Store and retrieve credentials to and from a dictionary-like object.
 
     Args:
@@ -46,7 +45,7 @@ class DictionaryStorage(Storage):
         if serialized is None:
             return None
 
-        credentials = OAuth2Credentials.from_json(serialized)
+        credentials = client.OAuth2Credentials.from_json(serialized)
         credentials.set_store(self)
 
         return credentials

src/oauth2client/contrib/django_orm.py

@@ -1,173 +0,0 @@
-# Copyright 2014 Google Inc. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""OAuth 2.0 utilities for Django.
-
-Utilities for using OAuth 2.0 in conjunction with
-the Django datastore.
-"""
-
-import oauth2client
-import base64
-import pickle
-import six
-
-from django.db import models
-from django.utils.encoding import smart_bytes, smart_text
-from oauth2client.client import Storage as BaseStorage
-
-
-__author__ = 'jcgregorio@google.com (Joe Gregorio)'
-
-
-class CredentialsField(six.with_metaclass(models.SubfieldBase, models.Field)):
-
-    def __init__(self, *args, **kwargs):
-        if 'null' not in kwargs:
-            kwargs['null'] = True
-        super(CredentialsField, self).__init__(*args, **kwargs)
-
-    def get_internal_type(self):
-        return 'TextField'
-
-    def to_python(self, value):
-        if value is None:
-            return None
-        if isinstance(value, oauth2client.client.Credentials):
-            return value
-        return pickle.loads(base64.b64decode(smart_bytes(value)))
-
-    def get_prep_value(self, value):
-        if value is None:
-            return None
-        return smart_text(base64.b64encode(pickle.dumps(value)))
-
-    def value_to_string(self, obj):
-        """Convert the field value from the provided model to a string.
-
-        Used during model serialization.
-
-        Args:
-            obj: db.Model, model object
-
-        Returns:
-            string, the serialized field value
-        """
-        value = self._get_val_from_obj(obj)
-        return self.get_prep_value(value)
-
-
-class FlowField(six.with_metaclass(models.SubfieldBase, models.Field)):
-
-    def __init__(self, *args, **kwargs):
-        if 'null' not in kwargs:
-            kwargs['null'] = True
-        super(FlowField, self).__init__(*args, **kwargs)
-
-    def get_internal_type(self):
-        return 'TextField'
-
-    def to_python(self, value):
-        if value is None:
-            return None
-        if isinstance(value, oauth2client.client.Flow):
-            return value
-        return pickle.loads(base64.b64decode(value))
-
-    def get_prep_value(self, value):
-        if value is None:
-            return None
-        return smart_text(base64.b64encode(pickle.dumps(value)))
-
-    def value_to_string(self, obj):
-        """Convert the field value from the provided model to a string.
-
-        Used during model serialization.
-
-        Args:
-            obj: db.Model, model object
-
-        Returns:
-            string, the serialized field value
-        """
-        value = self._get_val_from_obj(obj)
-        return self.get_prep_value(value)
-
-
-class Storage(BaseStorage):
-    """Store and retrieve a single credential to and from the Django datastore.
-
-    This Storage helper presumes the Credentials
-    have been stored as a CredenialsField
-    on a db model class.
-    """
-
-    def __init__(self, model_class, key_name, key_value, property_name):
-        """Constructor for Storage.
-
-        Args:
-            model: db.Model, model class
-            key_name: string, key name for the entity that has the credentials
-            key_value: string, key value for the entity that has the
-                       credentials
-            property_name: string, name of the property that is an
-                           CredentialsProperty
-        """
-        super(Storage, self).__init__()
-        self.model_class = model_class
-        self.key_name = key_name
-        self.key_value = key_value
-        self.property_name = property_name
-
-    def locked_get(self):
-        """Retrieve stored credential.
-
-        Returns:
-            oauth2client.Credentials
-        """
-        credential = None
-
-        query = {self.key_name: self.key_value}
-        entities = self.model_class.objects.filter(**query)
-        if len(entities) > 0:
-            credential = getattr(entities[0], self.property_name)
-            if credential and hasattr(credential, 'set_store'):
-                credential.set_store(self)
-        return credential
-
-    def locked_put(self, credentials, overwrite=False):
-        """Write a Credentials to the Django datastore.
-
-        Args:
-            credentials: Credentials, the credentials to store.
-            overwrite: Boolean, indicates whether you would like these
-                       credentials to overwrite any existing stored
-                       credentials.
-        """
-        args = {self.key_name: self.key_value}
-
-        if overwrite:
-            (entity,
-             unused_is_new) = self.model_class.objects.get_or_create(**args)
-        else:
-            entity = self.model_class(**args)
-
-        setattr(entity, self.property_name, credentials)
-        entity.save()
-
-    def locked_delete(self):
-        """Delete Credentials from the datastore."""
-
-        query = {self.key_name: self.key_value}
-        entities = self.model_class.objects.filter(**query).delete()

src/oauth2client/contrib/django_util/__init__.py

@@ -12,17 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Utilities for the Django web framework
+"""Utilities for the Django web framework.
 
 Provides Django views and helpers the make using the OAuth2 web server
-flow easier. It includes an ``oauth_required`` decorator to automatically ensure
-that user credentials are available, and an ``oauth_enabled`` decorator to check
-if the user has authorized, and helper shortcuts to create the authorization
-URL otherwise.
+flow easier. It includes an ``oauth_required`` decorator to automatically
+ensure that user credentials are available, and an ``oauth_enabled`` decorator
+to check if the user has authorized, and helper shortcuts to create the
+authorization URL otherwise.
 
+There are two basic use cases supported. The first is using Google OAuth as the
+primary form of authentication, which is the simpler approach recommended
+for applications without their own user system.
+
+The second use case is adding Google OAuth credentials to an
+existing Django model containing a Django user field. Most of the
+configuration is the same, except for `GOOGLE_OAUTH_MODEL_STORAGE` in
+settings.py. See "Adding Credentials To An Existing Django User System" for
+usage differences.
+
+Only Django versions 1.8+ are supported.
 
 Configuration
-=============
+===============
 
 To configure, you'll need a set of OAuth2 web application credentials from
 `Google Developer's Console <https://console.developers.google.com/project/_/apiui/credential>`.
@@ -35,9 +46,13 @@ Add the helper to your INSTALLED_APPS:
 
     INSTALLED_APPS = (
         # other apps
+        "django.contrib.sessions.middleware"
         "oauth2client.contrib.django_util"
     )
 
+This helper also requires the Django Session Middleware, so
+``django.contrib.sessions.middleware`` should be in INSTALLED_APPS as well.
+
 Add the client secrets created earlier to the settings. You can either
 specify the path to the credentials file in JSON format
 
@@ -88,8 +103,8 @@ Add the oauth2 routes to your application's urls.py urlpatterns.
    urlpatterns += [url(r'^oauth2/', include(oauth2_urls))]
 
 To require OAuth2 credentials for a view, use the `oauth2_required` decorator.
-This creates a credentials object with an id_token, and allows you to create an
-`http` object to build service clients with. These are all attached to the
+This creates a credentials object with an id_token, and allows you to create
+an `http` object to build service clients with. These are all attached to the
 request.oauth
 
 .. code-block:: python
@@ -105,7 +120,10 @@ request.oauth
                     http=request.oauth.http,
                    developerKey=API_KEY)
       events = service.events().list(calendarId='primary').execute()['items']
-      return HttpResponse("email: %s , calendar: %s" % (email, str(events)))
+      return HttpResponse("email: {0} , calendar: {1}".format(
+                           email,str(events)))
+      return HttpResponse(
+          "email: {0} , calendar: {1}".format(email, str(events)))
 
 To make OAuth2 optional and provide an authorization link in your own views.
 
@@ -120,11 +138,12 @@ To make OAuth2 optional and provide an authorization link in your own views.
        if request.oauth.has_credentials():
            # this could be passed into a view
            # request.oauth.http is also initialized
-           return HttpResponse("User email: %s"
-            % request.oauth.credentials.id_token['email'])
+           return HttpResponse("User email: {0}".format(
+               request.oauth.credentials.id_token['email']))
        else:
-           return HttpResponse('Here is an OAuth Authorize link:
-           <a href="%s">Authorize</a>' % request.oauth.get_authorize_redirect())
+           return HttpResponse(
+               'Here is an OAuth Authorize link: <a href="{0}">Authorize'
+               '</a>'.format(request.oauth.get_authorize_redirect()))
 
 If a view needs a scope not included in the default scopes specified in
 the settings, you can use [incremental auth](https://developers.google.com/identity/sign-in/web/incremental-auth)
@@ -143,8 +162,9 @@ and specify additional scopes in the decorator arguments.
            events = service.files().list().execute()['items']
            return HttpResponse(str(events))
        else:
-           return HttpResponse('Here is an OAuth Authorize link:
-           <a href="%s">Authorize</a>' % request.oauth.get_authorize_redirect())
+           return HttpResponse(
+               'Here is an OAuth Authorize link: <a href="{0}">Authorize'
+               '</a>'.format(request.oauth.get_authorize_redirect()))
 
 
 To provide a callback on authorization being completed, use the
@@ -157,26 +177,78 @@ oauth2_authorized signal:
    from oauth2client.contrib.django_util.signals import oauth2_authorized
 
    def test_callback(sender, request, credentials, **kwargs):
-       print "Authorization Signal Received %s" % credentials.id_token['email']
+       print("Authorization Signal Received {0}".format(
+               credentials.id_token['email']))
 
    oauth2_authorized.connect(test_callback)
 
+Adding Credentials To An Existing Django User System
+=====================================================
+
+As an alternative to storing the credentials in the session, the helper
+can be configured to store the fields on a Django model. This might be useful
+if you need to use the credentials outside the context of a user request. It
+also prevents the need for a logged in user to repeat the OAuth flow when
+starting a new session.
+
+To use, change ``settings.py``
+
+.. code-block:: python
+   :caption:  settings.py
+   :name: storage_model_config
+
+   GOOGLE_OAUTH2_STORAGE_MODEL = {
+       'model': 'path.to.model.MyModel',
+       'user_property': 'user_id',
+       'credentials_property': 'credential'
+    }
+
+Where ``path.to.model`` class is the fully qualified name of a
+``django.db.model`` class containing a ``django.contrib.auth.models.User``
+field with the name specified by `user_property` and a
+:class:`oauth2client.contrib.django_util.models.CredentialsField` with the name
+specified by `credentials_property`. For the sample configuration given,
+our model would look like
+
+.. code-block:: python
+   :caption: models.py
+   :name: storage_model_model
+
+   from django.contrib.auth.models import User
+   from oauth2client.contrib.django_util.models import CredentialsField
+
+   class MyModel(models.Model):
+       #  ... other fields here ...
+       user = models.OneToOneField(User)
+       credential = CredentialsField()
 """
 
+import importlib
+
 import django.conf
 from django.core import exceptions
 from django.core import urlresolvers
 import httplib2
-from oauth2client import clientsecrets
-from oauth2client.contrib.django_util import storage
 from six.moves.urllib import parse
 
+from oauth2client import clientsecrets
+from oauth2client.contrib import dictionary_storage
+from oauth2client.contrib.django_util import storage
+
 GOOGLE_OAUTH2_DEFAULT_SCOPES = ('email',)
 GOOGLE_OAUTH2_REQUEST_ATTRIBUTE = 'oauth'
 
 
 def _load_client_secrets(filename):
-    """Loads client secrets from the given filename."""
+    """Loads client secrets from the given filename.
+
+    Args:
+        filename: The name of the file containing the JSON secret key.
+
+    Returns:
+        A 2-tuple, the first item containing the client id, and the second
+        item containing a client secret.
+    """
     client_type, client_info = clientsecrets.loadfile(filename)
 
     if client_type != clientsecrets.TYPE_WEB:
@@ -187,8 +259,16 @@ def _load_client_secrets(filename):
 
 
 def _get_oauth2_client_id_and_secret(settings_instance):
-    """Initializes client id and client secret based on the settings"""
-    secret_json = getattr(django.conf.settings,
+    """Initializes client id and client secret based on the settings.
+
+    Args:
+        settings_instance: An instance of ``django.conf.settings``.
+
+    Returns:
+        A 2-tuple, the first item is the client id and the second
+         item is the client secret.
+    """
+    secret_json = getattr(settings_instance,
                           'GOOGLE_OAUTH2_CLIENT_SECRETS_JSON', None)
     if secret_json is not None:
         return _load_client_secrets(secret_json)
@@ -201,9 +281,36 @@ def _get_oauth2_client_id_and_secret(settings_instance):
             return client_id, client_secret
         else:
             raise exceptions.ImproperlyConfigured(
-                "Must specify either GOOGLE_OAUTH2_CLIENT_SECRETS_JSON, or  "
-                " both GOOGLE_OAUTH2_CLIENT_ID and GOOGLE_OAUTH2_CLIENT_SECRET "
-                "in settings.py")
+                "Must specify either GOOGLE_OAUTH2_CLIENT_SECRETS_JSON, or "
+                "both GOOGLE_OAUTH2_CLIENT_ID and "
+                "GOOGLE_OAUTH2_CLIENT_SECRET in settings.py")
+
+
+def _get_storage_model():
+    """This configures whether the credentials will be stored in the session
+    or the Django ORM based on the settings. By default, the credentials
+    will be stored in the session, unless `GOOGLE_OAUTH2_STORAGE_MODEL`
+    is found in the settings. Usually, the ORM storage is used to integrate
+    credentials into an existing Django user system.
+
+    Returns:
+        A tuple containing three strings, or None. If
+        ``GOOGLE_OAUTH2_STORAGE_MODEL`` is configured, the tuple
+        will contain the fully qualifed path of the `django.db.model`,
+        the name of the ``django.contrib.auth.models.User`` field on the
+        model, and the name of the
+        :class:`oauth2client.contrib.django_util.models.CredentialsField`
+        field on the model. If Django ORM storage is not configured,
+        this function returns None.
+    """
+    storage_model_settings = getattr(django.conf.settings,
+                                     'GOOGLE_OAUTH2_STORAGE_MODEL', None)
+    if storage_model_settings is not None:
+        return (storage_model_settings['model'],
+                storage_model_settings['user_property'],
+                storage_model_settings['credentials_property'])
+    else:
+        return None, None, None
 
 
 class OAuth2Settings(object):
@@ -215,11 +322,11 @@ class OAuth2Settings(object):
 
     Attributes:
       scopes: A list of OAuth2 scopes that the decorators and views will use
-              as defaults
+              as defaults.
       request_prefix: The name of the attribute that the decorators use to
                     attach the UserOAuth2 object to the Django request object.
-      client_id: The OAuth2 Client ID
-      client_secret: The OAuth2 Client Secret
+      client_id: The OAuth2 Client ID.
+      client_secret: The OAuth2 Client Secret.
     """
 
     def __init__(self, settings_instance):
@@ -232,75 +339,139 @@ class OAuth2Settings(object):
             _get_oauth2_client_id_and_secret(settings_instance)
 
         if ('django.contrib.sessions.middleware.SessionMiddleware'
-                not in settings_instance.MIDDLEWARE_CLASSES):
+           not in settings_instance.MIDDLEWARE_CLASSES):
             raise exceptions.ImproperlyConfigured(
-                "The Google OAuth2 Helper requires session middleware to "
-                "be installed. Edit your MIDDLEWARE_CLASSES setting"
-                " to include 'django.contrib.sessions.middleware."
-                "SessionMiddleware'.")
+                  'The Google OAuth2 Helper requires session middleware to '
+                  'be installed. Edit your MIDDLEWARE_CLASSES setting'
+                  ' to include \'django.contrib.sessions.middleware.'
+                  'SessionMiddleware\'.')
+        (self.storage_model, self.storage_model_user_property,
+         self.storage_model_credentials_property) = _get_storage_model()
 
 
 oauth2_settings = OAuth2Settings(django.conf.settings)
 
+_CREDENTIALS_KEY = 'google_oauth2_credentials'
+
+
+def get_storage(request):
+    """ Gets a Credentials storage object provided by the Django OAuth2 Helper
+    object.
+
+    Args:
+        request: Reference to the current request object.
+
+    Returns:
+       An :class:`oauth2.client.Storage` object.
+    """
+    storage_model = oauth2_settings.storage_model
+    user_property = oauth2_settings.storage_model_user_property
+    credentials_property = oauth2_settings.storage_model_credentials_property
+
+    if storage_model:
+        module_name, class_name = storage_model.rsplit('.', 1)
+        module = importlib.import_module(module_name)
+        storage_model_class = getattr(module, class_name)
+        return storage.DjangoORMStorage(storage_model_class,
+                                        user_property,
+                                        request.user,
+                                        credentials_property)
+    else:
+        # use session
+        return dictionary_storage.DictionaryStorage(
+            request.session, key=_CREDENTIALS_KEY)
+
 
 def _redirect_with_params(url_name, *args, **kwargs):
-    """Helper method to create a redirect response that uses GET URL
-    parameters."""
+    """Helper method to create a redirect response with URL params.
 
+    This builds a redirect string that converts kwargs into a
+    query string.
+
+    Args:
+        url_name: The name of the url to redirect to.
+        kwargs: the query string param and their values to build.
+
+    Returns:
+        A properly formatted redirect string.
+    """
     url = urlresolvers.reverse(url_name, args=args)
     params = parse.urlencode(kwargs, True)
     return "{0}?{1}".format(url, params)
 
 
+def _credentials_from_request(request):
+    """Gets the authorized credentials for this flow, if they exist."""
+    # ORM storage requires a logged in user
+    if (oauth2_settings.storage_model is None or
+            request.user.is_authenticated()):
+        return get_storage(request).get()
+    else:
+        return None
+
+
 class UserOAuth2(object):
     """Class to create oauth2 objects on Django request objects containing
     credentials and helper methods.
     """
 
     def __init__(self, request, scopes=None, return_url=None):
-        """Initialize the Oauth2 Object
-        :param request: Django request object
-        :param scopes: Scopes desired for this OAuth2 flow
-        :param return_url: URL to return to after authorization is complete
-        :return:
+        """Initialize the Oauth2 Object.
+
+        Args:
+            request: Django request object.
+            scopes: Scopes desired for this OAuth2 flow.
+            return_url: The url to return to after the OAuth flow is complete,
+                 defaults to the request's current URL path.
         """
         self.request = request
         self.return_url = return_url or request.get_full_path()
-        self.scopes = set(oauth2_settings.scopes)
         if scopes:
-            self.scopes |= set(scopes)
-
-        # make sure previously requested custom scopes are maintained
-        # in future authorizations
-        credentials = storage.get_storage(self.request).get()
-        if credentials:
-            self.scopes |= credentials.scopes
+            self._scopes = set(oauth2_settings.scopes) | set(scopes)
+        else:
+            self._scopes = set(oauth2_settings.scopes)
 
     def get_authorize_redirect(self):
-        """Creates a URl to start the OAuth2 authorization flow"""
+        """Creates a URl to start the OAuth2 authorization flow."""
         get_params = {
             'return_url': self.return_url,
-            'scopes': self.scopes
+            'scopes': self._get_scopes()
         }
 
-        return _redirect_with_params('google_oauth:authorize',
-                                     **get_params)
+        return _redirect_with_params('google_oauth:authorize', **get_params)
 
     def has_credentials(self):
         """Returns True if there are valid credentials for the current user
         and required scopes."""
-        return (self.credentials and not self.credentials.invalid
-                and self.credentials.has_scopes(self.scopes))
+        credentials = _credentials_from_request(self.request)
+        return (credentials and not credentials.invalid and
+                credentials.has_scopes(self._get_scopes()))
+
+    def _get_scopes(self):
+        """Returns the scopes associated with this object, kept up to
+         date for incremental auth."""
+        if _credentials_from_request(self.request):
+            return (self._scopes |
+                    _credentials_from_request(self.request).scopes)
+        else:
+            return self._scopes
+
+    @property
+    def scopes(self):
+        """Returns the scopes associated with this OAuth2 object."""
+        # make sure previously requested custom scopes are maintained
+        # in future authorizations
+        return self._get_scopes()
 
     @property
     def credentials(self):
-        """Gets the authorized credentials for this flow, if they exist"""
-        return storage.get_storage(self.request).get()
+        """Gets the authorized credentials for this flow, if they exist."""
+        return _credentials_from_request(self.request)
 
     @property
     def http(self):
         """Helper method to create an HTTP client authorized with OAuth2
-        credentials"""
+        credentials."""
         if self.has_credentials():
             return self.credentials.authorize(httplib2.Http())
         return None

src/oauth2client/contrib/django_util/apps.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Application Config For Django OAuth2 Helper
+"""Application Config For Django OAuth2 Helper.
 
 Django 1.7+ provides an
 [applications](https://docs.djangoproject.com/en/1.8/ref/applications/)

src/oauth2client/contrib/django_util/decorators.py

@@ -12,13 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Decorators for Django OAuth2 Flow.
+
+Contains two decorators, ``oauth_required`` and ``oauth_enabled``.
+
+``oauth_required`` will ensure that a user has an oauth object containing
+credentials associated with the request, and if not, redirect to the
+authorization flow.
+
+``oauth_enabled`` will attach the oauth2 object containing credentials if it
+exists. If it doesn't, the view will still render, but helper methods will be
+attached to start the oauth2 flow.
+"""
+
 from django import shortcuts
-from oauth2client.contrib import django_util
+import django.conf
 from six import wraps
+from six.moves.urllib import parse
+
+from oauth2client.contrib import django_util
 
 
 def oauth_required(decorated_function=None, scopes=None, **decorator_kwargs):
-    """ Decorator to require OAuth2 credentials for a view
+    """ Decorator to require OAuth2 credentials for a view.
 
 
     .. code-block:: python
@@ -36,21 +52,31 @@ def oauth_required(decorated_function=None, scopes=None, **decorator_kwargs):
                        developerKey=API_KEY)
           events = service.events().list(
                                     calendarId='primary').execute()['items']
-          return HttpResponse("email: %s , calendar: %s" % (email, str(events)))
+          return HttpResponse(
+              "email: {0}, calendar: {1}".format(email, str(events)))
 
-    :param decorated_function: View function to decorate, must have the Django
-           request object as the first argument
-    :param scopes: Scopes to require, will default
-    :param decorator_kwargs: Can include ``return_url`` to specify the URL to
-           return to after OAuth2 authorization is complete
-    :return: An OAuth2 Authorize view if credentials are not found or if the
-             credentials are missing the required scopes. Otherwise,
-             the decorated view.
+    Args:
+        decorated_function: View function to decorate, must have the Django
+           request object as the first argument.
+        scopes: Scopes to require, will default.
+        decorator_kwargs: Can include ``return_url`` to specify the URL to
+           return to after OAuth2 authorization is complete.
+
+    Returns:
+        An OAuth2 Authorize view if credentials are not found or if the
+        credentials are missing the required scopes. Otherwise,
+        the decorated view.
     """
-
     def curry_wrapper(wrapped_function):
         @wraps(wrapped_function)
         def required_wrapper(request, *args, **kwargs):
+            if not (django_util.oauth2_settings.storage_model is None or
+                    request.user.is_authenticated()):
+                redirect_str = '{0}?next={1}'.format(
+                    django.conf.settings.LOGIN_URL,
+                    parse.quote(request.path))
+                return shortcuts.redirect(redirect_str)
+
             return_url = decorator_kwargs.pop('return_url',
                                               request.get_full_path())
             user_oauth = django_util.UserOAuth2(request, scopes, return_url)
@@ -84,21 +110,23 @@ def oauth_enabled(decorated_function=None, scopes=None, **decorator_kwargs):
            if request.oauth.has_credentials():
                # this could be passed into a view
                # request.oauth.http is also initialized
-               return HttpResponse("User email: %s" %
+               return HttpResponse("User email: {0}".format(
                                    request.oauth.credentials.id_token['email'])
            else:
                return HttpResponse('Here is an OAuth Authorize link:
-               <a href="%s">Authorize</a>' %
-               request.oauth.get_authorize_redirect())
+               <a href="{0}">Authorize</a>'.format(
+                   request.oauth.get_authorize_redirect()))
 
 
-    :param decorated_function: View function to decorate
-    :param scopes: Scopes to require, will default
-    :param decorator_kwargs: Can include ``return_url`` to specify the URL to
-           return to after OAuth2 authorization is complete
-    :return: The decorated view function
+    Args:
+        decorated_function: View function to decorate.
+        scopes: Scopes to require, will default.
+        decorator_kwargs: Can include ``return_url`` to specify the URL to
+           return to after OAuth2 authorization is complete.
+
+    Returns:
+         The decorated view function.
     """
-
     def curry_wrapper(wrapped_function):
         @wraps(wrapped_function)
         def enabled_wrapper(request, *args, **kwargs):

src/oauth2client/contrib/django_util/models.py

@@ -0,0 +1,75 @@
+# Copyright 2016 Google Inc.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains classes used for the Django ORM storage."""
+
+import base64
+import pickle
+
+from django.db import models
+from django.utils import encoding
+
+import oauth2client
+
+
+class CredentialsField(models.Field):
+    """Django ORM field for storing OAuth2 Credentials."""
+
+    def __init__(self, *args, **kwargs):
+        if 'null' not in kwargs:
+            kwargs['null'] = True
+        super(CredentialsField, self).__init__(*args, **kwargs)
+
+    def get_internal_type(self):
+        return 'BinaryField'
+
+    def from_db_value(self, value, expression, connection, context):
+        """Overrides ``models.Field`` method. This converts the value
+        returned from the database to an instance of this class.
+        """
+        return self.to_python(value)
+
+    def to_python(self, value):
+        """Overrides ``models.Field`` method. This is used to convert
+        bytes (from serialization etc) to an instance of this class"""
+        if value is None:
+            return None
+        elif isinstance(value, oauth2client.client.Credentials):
+            return value
+        else:
+            return pickle.loads(base64.b64decode(encoding.smart_bytes(value)))
+
+    def get_prep_value(self, value):
+        """Overrides ``models.Field`` method. This is used to convert
+        the value from an instances of this class to bytes that can be
+        inserted into the database.
+        """
+        if value is None:
+            return None
+        else:
+            return encoding.smart_text(base64.b64encode(pickle.dumps(value)))
+
+    def value_to_string(self, obj):
+        """Convert the field value from the provided model to a string.
+
+        Used during model serialization.
+
+        Args:
+            obj: db.Model, model object
+
+        Returns:
+            string, the serialized field value
+        """
+        value = self._get_val_from_obj(obj)
+        return self.get_prep_value(value)

src/oauth2client/contrib/django_util/signals.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-""" Signals for Google OAuth2 Helper
+"""Signals for Google OAuth2 Helper.
 
 This module contains signals for Google OAuth2 Helper. Currently it only
 contains one, which fires when an OAuth2 authorization flow has completed.

src/oauth2client/contrib/django_util/site.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Contains Django URL patterns used for OAuth2 flow."""
+
 from django.conf import urls
+
 from oauth2client.contrib.django_util import views
 
 urlpatterns = [

src/oauth2client/contrib/django_util/storage.py

@@ -12,16 +12,70 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from oauth2client.contrib.dictionary_storage import DictionaryStorage
+"""Contains a storage module that stores credentials using the Django ORM."""
 
-_CREDENTIALS_KEY = 'google_oauth2_credentials'
+from oauth2client import client
 
 
-def get_storage(request):
-    # TODO(issue 319): Make this pluggable with different storage providers
-    # https://github.com/google/oauth2client/issues/319
-    """ Gets a Credentials storage object for the Django OAuth2 Helper object
-    :param request: Reference to the current request object
-    :return: A OAuth2Client Storage implementation based on sessions
+class DjangoORMStorage(client.Storage):
+    """Store and retrieve a single credential to and from the Django datastore.
+
+    This Storage helper presumes the Credentials
+    have been stored as a CredentialsField
+    on a db model class.
     """
-    return DictionaryStorage(request.session, key=_CREDENTIALS_KEY)
+
+    def __init__(self, model_class, key_name, key_value, property_name):
+        """Constructor for Storage.
+
+        Args:
+            model: string, fully qualified name of db.Model model class.
+            key_name: string, key name for the entity that has the credentials
+            key_value: string, key value for the entity that has the
+               credentials.
+            property_name: string, name of the property that is an
+                           CredentialsProperty.
+        """
+        super(DjangoORMStorage, self).__init__()
+        self.model_class = model_class
+        self.key_name = key_name
+        self.key_value = key_value
+        self.property_name = property_name
+
+    def locked_get(self):
+        """Retrieve stored credential from the Django ORM.
+
+        Returns:
+            oauth2client.Credentials retrieved from the Django ORM, associated
+             with the ``model``, ``key_value``->``key_name`` pair used to query
+             for the model, and ``property_name`` identifying the
+             ``CredentialsProperty`` field, all of which are defined in the
+             constructor for this Storage object.
+
+        """
+        query = {self.key_name: self.key_value}
+        entities = self.model_class.objects.filter(**query)
+        if len(entities) > 0:
+            credential = getattr(entities[0], self.property_name)
+            if getattr(credential, 'set_store', None) is not None:
+                credential.set_store(self)
+            return credential
+        else:
+            return None
+
+    def locked_put(self, credentials):
+        """Write a Credentials to the Django datastore.
+
+        Args:
+            credentials: Credentials, the credentials to store.
+        """
+        entity, _ = self.model_class.objects.get_or_create(
+            **{self.key_name: self.key_value})
+
+        setattr(entity, self.property_name, credentials)
+        entity.save()
+
+    def locked_delete(self):
+        """Delete Credentials from the datastore."""
+        query = {self.key_name: self.key_value}
+        self.model_class.objects.filter(**query).delete()

src/oauth2client/contrib/django_util/views.py

@@ -12,24 +12,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""This module contains the views used by the OAuth2 flows.
+
+Their are two views used by the OAuth2 flow, the authorize and the callback
+view. The authorize view kicks off the three-legged OAuth flow, and the
+callback view validates the flow and if successful stores the credentials
+in the configured storage."""
+
 import hashlib
 import json
 import os
 import pickle
+
 from django import http
-from django.core import urlresolvers
 from django import shortcuts
+from django.conf import settings
+from django.core import urlresolvers
+from django.shortcuts import redirect
+from six.moves.urllib import parse
+
 from oauth2client import client
 from oauth2client.contrib import django_util
+from oauth2client.contrib.django_util import get_storage
 from oauth2client.contrib.django_util import signals
-from oauth2client.contrib.django_util import storage
 
 _CSRF_KEY = 'google_oauth2_csrf_token'
 _FLOW_KEY = 'google_oauth2_flow_{0}'
 
 
 def _make_flow(request, scopes, return_url=None):
-    """Creates a Web Server Flow"""
+    """Creates a Web Server Flow
+
+    Args:
+        request: A Django request object.
+        scopes: the request oauth2 scopes.
+        return_url: The URL to return to after the flow is complete. Defaults
+            to the path of the current request.
+
+    Returns:
+        An OAuth2 flow object that has been stored in the session.
+    """
     # Generate a CSRF token to prevent malicious requests.
     csrf_token = hashlib.sha256(os.urandom(1024)).hexdigest()
 
@@ -55,7 +77,17 @@ def _make_flow(request, scopes, return_url=None):
 
 def _get_flow_for_token(csrf_token, request):
     """ Looks up the flow in session to recover information about requested
-    scopes."""
+    scopes.
+
+    Args:
+        csrf_token: The token passed in the callback request that should
+            match the one previously generated and stored in the request on the
+            initial authorization view.
+
+    Returns:
+        The OAuth2 Flow object associated with this flow based on the
+        CSRF token.
+    """
     flow_pickle = request.session.get(_FLOW_KEY.format(csrf_token), None)
     return None if flow_pickle is None else pickle.loads(flow_pickle)
 
@@ -68,26 +100,30 @@ def oauth2_callback(request):
     and redirects to the return_url specified in the authorize view and
     stored in the session.
 
-    :param request: Django request
-    :return: A redirect response back to the return_url
+    Args:
+        request: Django request.
+
+    Returns:
+         A redirect response back to the return_url.
     """
     if 'error' in request.GET:
         reason = request.GET.get(
             'error_description', request.GET.get('error', ''))
         return http.HttpResponseBadRequest(
-            'Authorization failed %s' % reason)
+            'Authorization failed {0}'.format(reason))
 
     try:
         encoded_state = request.GET['state']
         code = request.GET['code']
     except KeyError:
         return http.HttpResponseBadRequest(
-            "Request missing state or authorization code")
+            'Request missing state or authorization code')
 
     try:
         server_csrf = request.session[_CSRF_KEY]
     except KeyError:
-        return http.HttpResponseBadRequest("No existing session for this flow.")
+        return http.HttpResponseBadRequest(
+            'No existing session for this flow.')
 
     try:
         state = json.loads(encoded_state)
@@ -102,23 +138,24 @@ def oauth2_callback(request):
     flow = _get_flow_for_token(client_csrf, request)
 
     if not flow:
-        return http.HttpResponseBadRequest("Missing Oauth2 flow.")
+        return http.HttpResponseBadRequest('Missing Oauth2 flow.')
 
     try:
         credentials = flow.step2_exchange(code)
     except client.FlowExchangeError as exchange_error:
         return http.HttpResponseBadRequest(
-            "An error has occurred: {0}".format(exchange_error))
+            'An error has occurred: {0}'.format(exchange_error))
 
-    storage.get_storage(request).put(credentials)
+    get_storage(request).put(credentials)
 
     signals.oauth2_authorized.send(sender=signals.oauth2_authorized,
                                    request=request, credentials=credentials)
+
     return shortcuts.redirect(return_url)
 
 
 def oauth2_authorize(request):
-    """ View to start the OAuth2 Authorization flow
+    """ View to start the OAuth2 Authorization flow.
 
      This view starts the OAuth2 authorization flow. If scopes is passed in
      as a  GET URL parameter, it will authorize those scopes, otherwise the
@@ -126,12 +163,26 @@ def oauth2_authorize(request):
      specified as a GET parameter, otherwise the referer header will be
      checked, and if that isn't found it will return to the root path.
 
-    :param request: The Django request object
-    :return: A redirect to Google OAuth2 Authorization
+    Args:
+       request: The Django request object.
+
+    Returns:
+         A redirect to Google OAuth2 Authorization.
     """
-    scopes = request.GET.getlist('scopes', django_util.oauth2_settings.scopes)
     return_url = request.GET.get('return_url', None)
 
+    # Model storage (but not session storage) requires a logged in user
+    if django_util.oauth2_settings.storage_model:
+        if not request.user.is_authenticated():
+            return redirect('{0}?next={1}'.format(
+                settings.LOGIN_URL, parse.quote(request.get_full_path())))
+        # This checks for the case where we ended up here because of a logged
+        # out user but we had credentials for it in the first place
+        elif get_storage(request).get() is not None:
+            return redirect(return_url)
+
+    scopes = request.GET.getlist('scopes', django_util.oauth2_settings.scopes)
+
     if not return_url:
         return_url = request.META.get('HTTP_REFERER', '/')
     flow = _make_flow(request=request, scopes=scopes, return_url=return_url)

src/oauth2client/contrib/flask_util.py

@@ -35,7 +35,7 @@ apiui/credential>`__.
 
     app.config['SECRET_KEY'] = 'your-secret-key'
 
-    app.config['GOOGLE_OAUTH2_CLIENT_SECRETS_JSON'] = 'client_secrets.json'
+    app.config['GOOGLE_OAUTH2_CLIENT_SECRETS_FILE'] = 'client_secrets.json'
 
     # or, specify the client id and secret separately
     app.config['GOOGLE_OAUTH2_CLIENT_ID'] = 'your-client-id'
@@ -162,14 +162,11 @@ available outside of a request context, you will need to implement your own
 :class:`oauth2client.Storage`.
 """
 
+from functools import wraps
 import hashlib
 import json
 import os
 import pickle
-from functools import wraps
-
-import six.moves.http_client as httplib
-import httplib2
 
 try:
     from flask import Blueprint
@@ -182,10 +179,12 @@ try:
 except ImportError:  # pragma: NO COVER
     raise ImportError('The flask utilities require flask 0.9 or newer.')
 
-from oauth2client.client import FlowExchangeError
-from oauth2client.client import OAuth2WebServerFlow
-from oauth2client.contrib.dictionary_storage import DictionaryStorage
+import httplib2
+import six.moves.http_client as httplib
+
+from oauth2client import client
 from oauth2client import clientsecrets
+from oauth2client.contrib import dictionary_storage
 
 
 __author__ = 'jonwayne@google.com (Jon Wayne Parrott)'
@@ -199,7 +198,7 @@ _CSRF_KEY = 'google_oauth2_csrf_token'
 def _get_flow_for_token(csrf_token):
     """Retrieves the flow instance associated with a given CSRF token from
     the Flask session."""
-    flow_pickle = session.get(
+    flow_pickle = session.pop(
         _FLOW_KEY.format(csrf_token), None)
 
     if flow_pickle is None:
@@ -213,14 +212,14 @@ class UserOAuth2(object):
 
     Configuration values:
 
-        * ``GOOGLE_OAUTH2_CLIENT_SECRETS_JSON`` path to a client secrets json
+        * ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` path to a client secrets json
           file, obtained from the credentials screen in the Google Developers
           console.
         * ``GOOGLE_OAUTH2_CLIENT_ID`` the oauth2 credentials' client ID. This
-          is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_JSON`` is not
+          is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE`` is not
           specified.
         * ``GOOGLE_OAUTH2_CLIENT_SECRET`` the oauth2 credentials' client
-          secret. This is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_JSON``
+          secret. This is only needed if ``GOOGLE_OAUTH2_CLIENT_SECRETS_FILE``
           is not specified.
 
     If app is specified, all arguments will be passed along to init_app.
@@ -243,7 +242,7 @@ class UserOAuth2(object):
             app: A Flask application.
             scopes: Optional list of scopes to authorize.
             client_secrets_file: Path to a file containing client secrets. You
-                can also specify the GOOGLE_OAUTH2_CLIENT_SECRETS_JSON config
+                can also specify the GOOGLE_OAUTH2_CLIENT_SECRETS_FILE config
                 value.
             client_id: If not specifying a client secrets file, specify the
                 OAuth2 client id. You can also specify the
@@ -263,7 +262,8 @@ class UserOAuth2(object):
         self.flow_kwargs = kwargs
 
         if storage is None:
-            storage = DictionaryStorage(session, key=_CREDENTIALS_KEY)
+            storage = dictionary_storage.DictionaryStorage(
+                session, key=_CREDENTIALS_KEY)
         self.storage = storage
 
         if scopes is None:
@@ -307,8 +307,8 @@ class UserOAuth2(object):
         except KeyError:
             raise ValueError(
                 'OAuth2 configuration could not be found. Either specify the '
-                'client_secrets_file or client_id and client_secret or set the'
-                'app configuration variables '
+                'client_secrets_file or client_id and client_secret or set '
+                'the app configuration variables '
                 'GOOGLE_OAUTH2_CLIENT_SECRETS_FILE or '
                 'GOOGLE_OAUTH2_CLIENT_ID and GOOGLE_OAUTH2_CLIENT_SECRET.')
 
@@ -341,7 +341,7 @@ class UserOAuth2(object):
         extra_scopes = kw.pop('scopes', [])
         scopes = set(self.scopes).union(set(extra_scopes))
 
-        flow = OAuth2WebServerFlow(
+        flow = client.OAuth2WebServerFlow(
             client_id=self.client_id,
             client_secret=self.client_secret,
             scope=scopes,
@@ -418,7 +418,7 @@ class UserOAuth2(object):
         # Exchange the auth code for credentials.
         try:
             credentials = flow.step2_exchange(code)
-        except FlowExchangeError as exchange_error:
+        except client.FlowExchangeError as exchange_error:
             current_app.logger.exception(exchange_error)
             content = 'An error occurred: {0}'.format(exchange_error)
             return content, httplib.BAD_REQUEST
@@ -443,7 +443,14 @@ class UserOAuth2(object):
 
     def has_credentials(self):
         """Returns True if there are valid credentials for the current user."""
-        return self.credentials and not self.credentials.invalid
+        if not self.credentials:
+            return False
+        # Is the access token expired? If so, do we have an refresh token?
+        elif (self.credentials.access_token_expired and
+                not self.credentials.refresh_token):
+            return False
+        else:
+            return True
 
     @property
     def email(self):

src/oauth2client/contrib/gce.py

@@ -17,29 +17,19 @@
 Utilities for making it easier to use OAuth 2.0 on Google Compute Engine.
 """
 
-import json
 import logging
 import warnings
 
 import httplib2
-from six.moves import http_client
-from six.moves import urllib
 
-from oauth2client._helpers import _from_bytes
-from oauth2client import util
-from oauth2client.client import HttpAccessTokenRefreshError
-from oauth2client.client import AssertionCredentials
+from oauth2client import client
+from oauth2client.contrib import _metadata
 
 
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
 
 logger = logging.getLogger(__name__)
 
-# URI Template for the endpoint that returns access_tokens.
-_METADATA_ROOT = ('http://metadata.google.internal/computeMetadata/v1/'
-                  'instance/service-accounts/default/')
-META = _METADATA_ROOT + 'token'
-_DEFAULT_EMAIL_METADATA = _METADATA_ROOT + 'email'
 _SCOPES_WARNING = """\
 You have requested explicit scopes to be used with a GCE service account.
 Using this argument will have no effect on the actual scopes for tokens
@@ -48,31 +38,7 @@ can't be overridden in the request.
 """
 
 
-def _get_service_account_email(http_request=None):
-    """Get the GCE service account email from the current environment.
-
-    Args:
-        http_request: callable, (Optional) a callable that matches the method
-                      signature of httplib2.Http.request, used to make
-                      the request to the metadata service.
-
-    Returns:
-        tuple, A pair where the first entry is an optional response (from a
-        failed request) and the second is service account email found (as
-        a string).
-    """
-    if http_request is None:
-        http_request = httplib2.Http().request
-    response, content = http_request(
-        _DEFAULT_EMAIL_METADATA, headers={'Metadata-Flavor': 'Google'})
-    if response.status == http_client.OK:
-        content = _from_bytes(content)
-        return None, content
-    else:
-        return response, content
-
-
-class AppAssertionCredentials(AssertionCredentials):
+class AppAssertionCredentials(client.AssertionCredentials):
     """Credentials object for Compute Engine Assertion Grants
 
     This object will allow a Compute Engine instance to identify itself to
@@ -83,34 +49,73 @@ class AppAssertionCredentials(AssertionCredentials):
     This credential does not require a flow to instantiate because it
     represents a two legged flow, and therefore has all of the required
     information to generate and refresh its own access tokens.
+
+    Note that :attr:`service_account_email` and :attr:`scopes`
+    will both return None until the credentials have been refreshed.
+    To check whether credentials have previously been refreshed use
+    :attr:`invalid`.
     """
 
-    @util.positional(2)
-    def __init__(self, scope='', **kwargs):
+    def __init__(self, email=None, *args, **kwargs):
         """Constructor for AppAssertionCredentials
 
         Args:
-            scope: string or iterable of strings, scope(s) of the credentials
-                   being requested. Using this argument will have no effect on
-                   the actual scopes for tokens requested. These scopes are
-                   set at VM instance creation time and won't change.
+            email: an email that specifies the service account to use.
+                   Only necessary if using custom service accounts
+                   (see https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#createdefaultserviceaccount).
         """
-        if scope:
+        if 'scopes' in kwargs:
             warnings.warn(_SCOPES_WARNING)
-        # This is just provided for backwards compatibility, but is not
-        # used by this class.
-        self.scope = util.scopes_to_string(scope)
-        self.kwargs = kwargs
+            kwargs['scopes'] = None
 
         # Assertion type is no longer used, but still in the
         # parent class signature.
-        super(AppAssertionCredentials, self).__init__(None)
-        self._service_account_email = None
+        super(AppAssertionCredentials, self).__init__(None, *args, **kwargs)
+
+        self.service_account_email = email
+        self.scopes = None
+        self.invalid = True
 
     @classmethod
     def from_json(cls, json_data):
-        data = json.loads(_from_bytes(json_data))
-        return AppAssertionCredentials(data['scope'])
+        raise NotImplementedError(
+            'Cannot serialize credentials for GCE service accounts.')
+
+    def to_json(self):
+        raise NotImplementedError(
+            'Cannot serialize credentials for GCE service accounts.')
+
+    def retrieve_scopes(self, http):
+        """Retrieves the canonical list of scopes for this access token.
+
+        Overrides client.Credentials.retrieve_scopes. Fetches scopes info
+        from the metadata server.
+
+        Args:
+            http: httplib2.Http, an http object to be used to make the refresh
+                  request.
+
+        Returns:
+            A set of strings containing the canonical list of scopes.
+        """
+        self._retrieve_info(http.request)
+        return self.scopes
+
+    def _retrieve_info(self, http_request):
+        """Validates invalid service accounts by retrieving service account info.
+
+        Args:
+            http_request: callable, a callable that matches the method
+                          signature of httplib2.Http.request, used to make the
+                          request to the metadata server
+        """
+        if self.invalid:
+            info = _metadata.get_service_account_info(
+                http_request,
+                service_account=self.service_account_email or 'default')
+            self.invalid = False
+            self.service_account_email = info['email']
+            self.scopes = info['scopes']
 
     def _refresh(self, http_request):
         """Refreshes the access_token.
@@ -125,21 +130,12 @@ class AppAssertionCredentials(AssertionCredentials):
         Raises:
             HttpAccessTokenRefreshError: When the refresh fails.
         """
-        response, content = http_request(
-            META, headers={'Metadata-Flavor': 'Google'})
-        content = _from_bytes(content)
-        if response.status == http_client.OK:
-            try:
-                token_content = json.loads(content)
-            except Exception as e:
-                raise HttpAccessTokenRefreshError(str(e),
-                                                  status=response.status)
-            self.access_token = token_content['access_token']
-        else:
-            if response.status == http_client.NOT_FOUND:
-                content += (' This can occur if a VM was created'
-                            ' with no service account or scopes.')
-            raise HttpAccessTokenRefreshError(content, status=response.status)
+        try:
+            self._retrieve_info(http_request)
+            self.access_token, self.token_expiry = _metadata.get_token(
+                http_request, service_account=self.service_account_email)
+        except httplib2.HttpLib2Error as e:
+            raise client.HttpAccessTokenRefreshError(str(e))
 
     @property
     def serialization_data(self):
@@ -149,9 +145,6 @@ class AppAssertionCredentials(AssertionCredentials):
     def create_scoped_required(self):
         return False
 
-    def create_scoped(self, scopes):
-        return AppAssertionCredentials(scopes, **self.kwargs)
-
     def sign_blob(self, blob):
         """Cryptographically sign a blob (of bytes).
 
@@ -167,28 +160,3 @@ class AppAssertionCredentials(AssertionCredentials):
         """
         raise NotImplementedError(
             'Compute Engine service accounts cannot sign blobs')
-
-    @property
-    def service_account_email(self):
-        """Get the email for the current service account.
-
-        Uses the Google Compute Engine metadata service to retrieve the email
-        of the default service account.
-
-        Returns:
-            string, The email associated with the Google Compute Engine
-            service account.
-
-        Raises:
-            AttributeError, if the email can not be retrieved from the Google
-            Compute Engine metadata service.
-        """
-        if self._service_account_email is None:
-            failure, email = _get_service_account_email()
-            if failure is None:
-                self._service_account_email = email
-            else:
-                raise AttributeError('Failed to retrieve the email from the '
-                                     'Google Compute Engine metadata service',
-                                     failure, email)
-        return self._service_account_email

src/oauth2client/contrib/keyring_storage.py

@@ -21,14 +21,13 @@ import threading
 
 import keyring
 
-from oauth2client.client import Credentials
-from oauth2client.client import Storage as BaseStorage
+from oauth2client import client
 
 
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
 
 
-class Storage(BaseStorage):
+class Storage(client.Storage):
     """Store and retrieve a single credential to and from the keyring.
 
     To use this module you must have the keyring module installed. See
@@ -44,9 +43,9 @@ class Storage(BaseStorage):
 
     Usage::
 
-        from oauth2client.keyring_storage import Storage
+        from oauth2client import keyring_storage
 
-        s = Storage('name_of_application', 'user1')
+        s = keyring_storage.Storage('name_of_application', 'user1')
         credentials = s.get()
 
     """
@@ -74,7 +73,7 @@ class Storage(BaseStorage):
 
         if content is not None:
             try:
-                credentials = Credentials.new_from_json(content)
+                credentials = client.Credentials.new_from_json(content)
                 credentials.set_store(self)
             except ValueError:
                 pass

src/oauth2client/contrib/locked_file.py

@@ -57,7 +57,7 @@ class AlreadyLockedException(Exception):
 def validate_file(filename):
     if os.path.islink(filename):
         raise CredentialsFileSymbolicLinkError(
-            'File: %s is a symbolic link.' % filename)
+            'File: {0} is a symbolic link.'.format(filename))
 
 
 class _Opener(object):
@@ -122,8 +122,8 @@ class _PosixOpener(_Opener):
             CredentialsFileSymbolicLinkError if the file is a symbolic link.
         """
         if self._locked:
-            raise AlreadyLockedException('File %s is already locked' %
-                                         self._filename)
+            raise AlreadyLockedException(
+                'File {0} is already locked'.format(self._filename))
         self._locked = False
 
         validate_file(self._filename)
@@ -170,165 +170,7 @@ class _PosixOpener(_Opener):
 
     def _posix_lockfile(self, filename):
         """The name of the lock file to use for posix locking."""
-        return '%s.lock' % filename
-
-
-try:
-    import fcntl
-
-    class _FcntlOpener(_Opener):
-        """Open, lock, and unlock a file using fcntl.lockf."""
-
-        def open_and_lock(self, timeout, delay):
-            """Open the file and lock it.
-
-            Args:
-                timeout: float, How long to try to lock for.
-                delay: float, How long to wait between retries
-
-            Raises:
-                AlreadyLockedException: if the lock is already acquired.
-                IOError: if the open fails.
-                CredentialsFileSymbolicLinkError: if the file is a symbolic
-                                                  link.
-            """
-            if self._locked:
-                raise AlreadyLockedException('File %s is already locked' %
-                                             self._filename)
-            start_time = time.time()
-
-            validate_file(self._filename)
-            try:
-                self._fh = open(self._filename, self._mode)
-            except IOError as e:
-                # If we can't access with _mode, try _fallback_mode and
-                # don't lock.
-                if e.errno in (errno.EPERM, errno.EACCES):
-                    self._fh = open(self._filename, self._fallback_mode)
-                    return
-
-            # We opened in _mode, try to lock the file.
-            while True:
-                try:
-                    fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
-                    self._locked = True
-                    return
-                except IOError as e:
-                    # If not retrying, then just pass on the error.
-                    if timeout == 0:
-                        raise
-                    if e.errno != errno.EACCES:
-                        raise
-                    # We could not acquire the lock. Try again.
-                    if (time.time() - start_time) >= timeout:
-                        logger.warn('Could not lock %s in %s seconds',
-                                    self._filename, timeout)
-                        if self._fh:
-                            self._fh.close()
-                        self._fh = open(self._filename, self._fallback_mode)
-                        return
-                    time.sleep(delay)
-
-        def unlock_and_close(self):
-            """Close and unlock the file using the fcntl.lockf primitive."""
-            if self._locked:
-                fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
-            self._locked = False
-            if self._fh:
-                self._fh.close()
-except ImportError:
-    _FcntlOpener = None
-
-
-try:
-    import pywintypes
-    import win32con
-    import win32file
-
-    class _Win32Opener(_Opener):
-        """Open, lock, and unlock a file using windows primitives."""
-
-        # Error #33:
-        #  'The process cannot access the file because another process'
-        FILE_IN_USE_ERROR = 33
-
-        # Error #158:
-        #  'The segment is already unlocked.'
-        FILE_ALREADY_UNLOCKED_ERROR = 158
-
-        def open_and_lock(self, timeout, delay):
-            """Open the file and lock it.
-
-            Args:
-                timeout: float, How long to try to lock for.
-                delay: float, How long to wait between retries
-
-            Raises:
-                AlreadyLockedException: if the lock is already acquired.
-                IOError: if the open fails.
-                CredentialsFileSymbolicLinkError: if the file is a symbolic
-                                                  link.
-            """
-            if self._locked:
-                raise AlreadyLockedException('File %s is already locked' %
-                                             self._filename)
-            start_time = time.time()
-
-            validate_file(self._filename)
-            try:
-                self._fh = open(self._filename, self._mode)
-            except IOError as e:
-                # If we can't access with _mode, try _fallback_mode
-                # and don't lock.
-                if e.errno == errno.EACCES:
-                    self._fh = open(self._filename, self._fallback_mode)
-                    return
-
-            # We opened in _mode, try to lock the file.
-            while True:
-                try:
-                    hfile = win32file._get_osfhandle(self._fh.fileno())
-                    win32file.LockFileEx(
-                        hfile,
-                        (win32con.LOCKFILE_FAIL_IMMEDIATELY |
-                         win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
-                        pywintypes.OVERLAPPED())
-                    self._locked = True
-                    return
-                except pywintypes.error as e:
-                    if timeout == 0:
-                        raise
-
-                    # If the error is not that the file is already
-                    # in use, raise.
-                    if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
-                        raise
-
-                    # We could not acquire the lock. Try again.
-                    if (time.time() - start_time) >= timeout:
-                        logger.warn('Could not lock %s in %s seconds' % (
-                            self._filename, timeout))
-                        if self._fh:
-                            self._fh.close()
-                        self._fh = open(self._filename, self._fallback_mode)
-                        return
-                    time.sleep(delay)
-
-        def unlock_and_close(self):
-            """Close and unlock the file using the win32 primitive."""
-            if self._locked:
-                try:
-                    hfile = win32file._get_osfhandle(self._fh.fileno())
-                    win32file.UnlockFileEx(hfile, 0, -0x10000,
-                                           pywintypes.OVERLAPPED())
-                except pywintypes.error as e:
-                    if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
-                        raise
-            self._locked = False
-            if self._fh:
-                self._fh.close()
-except ImportError:
-    _Win32Opener = None
+        return '{0}.lock'.format(filename)
 
 
 class LockedFile(object):
@@ -347,10 +189,15 @@ class LockedFile(object):
         """
         opener = None
         if not opener and use_native_locking:
-            if _Win32Opener:
+            try:
+                from oauth2client.contrib._win32_opener import _Win32Opener
                 opener = _Win32Opener(filename, mode, fallback_mode)
-            if _FcntlOpener:
-                opener = _FcntlOpener(filename, mode, fallback_mode)
+            except ImportError:
+                try:
+                    from oauth2client.contrib._fcntl_opener import _FcntlOpener
+                    opener = _FcntlOpener(filename, mode, fallback_mode)
+                except ImportError:
+                    pass
 
         if not opener:
             opener = _PosixOpener(filename, mode, fallback_mode)

src/oauth2client/contrib/multiprocess_file_storage.py

@@ -0,0 +1,355 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Multiprocess file credential storage.
+
+This module provides file-based storage that supports multiple credentials and
+cross-thread and process access.
+
+This module supersedes the functionality previously found in `multistore_file`.
+
+This module provides :class:`MultiprocessFileStorage` which:
+    * Is tied to a single credential via a user-specified key. This key can be
+      used to distinguish between multiple users, client ids, and/or scopes.
+    * Can be safely accessed and refreshed across threads and processes.
+
+Process & thread safety guarantees the following behavior:
+    * If one thread or process refreshes a credential, subsequent refreshes
+      from other processes will re-fetch the credentials from the file instead
+      of performing an http request.
+    * If two processes or threads attempt to refresh concurrently, only one
+      will be able to acquire the lock and refresh, with the deadlock caveat
+      below.
+    * The interprocess lock will not deadlock, instead, the if a process can
+      not acquire the interprocess lock within ``INTERPROCESS_LOCK_DEADLINE``
+      it will allow refreshing the credential but will not write the updated
+      credential to disk, This logic happens during every lock cycle - if the
+      credentials are refreshed again it will retry locking and writing as
+      normal.
+
+Usage
+=====
+
+Before using the storage, you need to decide how you want to key the
+credentials. A few common strategies include:
+
+    * If you're storing credentials for multiple users in a single file, use
+      a unique identifier for each user as the key.
+    * If you're storing credentials for multiple client IDs in a single file,
+      use the client ID as the key.
+    * If you're storing multiple credentials for one user, use the scopes as
+      the key.
+    * If you have a complicated setup, use a compound key. For example, you
+      can use a combination of the client ID and scopes as the key.
+
+Create an instance of :class:`MultiprocessFileStorage` for each credential you
+want to store, for example::
+
+    filename = 'credentials'
+    key = '{}-{}'.format(client_id, user_id)
+    storage = MultiprocessFileStorage(filename, key)
+
+To store the credentials::
+
+    storage.put(credentials)
+
+If you're going to continue to use the credentials after storing them, be sure
+to call :func:`set_store`::
+
+    credentials.set_store(storage)
+
+To retrieve the credentials::
+
+    storage.get(credentials)
+
+"""
+
+import base64
+import json
+import logging
+import os
+import threading
+
+import fasteners
+from six import iteritems
+
+from oauth2client import _helpers
+from oauth2client import client
+
+
+#: The maximum amount of time, in seconds, to wait when acquire the
+#: interprocess lock before falling back to read-only mode.
+INTERPROCESS_LOCK_DEADLINE = 1
+
+logger = logging.getLogger(__name__)
+_backends = {}
+_backends_lock = threading.Lock()
+
+
+def _create_file_if_needed(filename):
+    """Creates the an empty file if it does not already exist.
+
+    Returns:
+        True if the file was created, False otherwise.
+    """
+    if os.path.exists(filename):
+        return False
+    else:
+        # Equivalent to "touch".
+        open(filename, 'a+b').close()
+        logger.info('Credential file {0} created'.format(filename))
+        return True
+
+
+def _load_credentials_file(credentials_file):
+    """Load credentials from the given file handle.
+
+    The file is expected to be in this format:
+
+        {
+            "file_version": 2,
+            "credentials": {
+                "key": "base64 encoded json representation of credentials."
+            }
+        }
+
+    This function will warn and return empty credentials instead of raising
+    exceptions.
+
+    Args:
+        credentials_file: An open file handle.
+
+    Returns:
+        A dictionary mapping user-defined keys to an instance of
+        :class:`oauth2client.client.Credentials`.
+    """
+    try:
+        credentials_file.seek(0)
+        data = json.load(credentials_file)
+    except Exception:
+        logger.warning(
+            'Credentials file could not be loaded, will ignore and '
+            'overwrite.')
+        return {}
+
+    if data.get('file_version') != 2:
+        logger.warning(
+            'Credentials file is not version 2, will ignore and '
+            'overwrite.')
+        return {}
+
+    credentials = {}
+
+    for key, encoded_credential in iteritems(data.get('credentials', {})):
+        try:
+            credential_json = base64.b64decode(encoded_credential)
+            credential = client.Credentials.new_from_json(credential_json)
+            credentials[key] = credential
+        except:
+            logger.warning(
+                'Invalid credential {0} in file, ignoring.'.format(key))
+
+    return credentials
+
+
+def _write_credentials_file(credentials_file, credentials):
+    """Writes credentials to a file.
+
+    Refer to :func:`_load_credentials_file` for the format.
+
+    Args:
+        credentials_file: An open file handle, must be read/write.
+        credentials: A dictionary mapping user-defined keys to an instance of
+            :class:`oauth2client.client.Credentials`.
+    """
+    data = {'file_version': 2, 'credentials': {}}
+
+    for key, credential in iteritems(credentials):
+        credential_json = credential.to_json()
+        encoded_credential = _helpers._from_bytes(base64.b64encode(
+            _helpers._to_bytes(credential_json)))
+        data['credentials'][key] = encoded_credential
+
+    credentials_file.seek(0)
+    json.dump(data, credentials_file)
+    credentials_file.truncate()
+
+
+class _MultiprocessStorageBackend(object):
+    """Thread-local backend for multiprocess storage.
+
+    Each process has only one instance of this backend per file. All threads
+    share a single instance of this backend. This ensures that all threads
+    use the same thread lock and process lock when accessing the file.
+    """
+
+    def __init__(self, filename):
+        self._file = None
+        self._filename = filename
+        self._process_lock = fasteners.InterProcessLock(
+            '{0}.lock'.format(filename))
+        self._thread_lock = threading.Lock()
+        self._read_only = False
+        self._credentials = {}
+
+    def _load_credentials(self):
+        """(Re-)loads the credentials from the file."""
+        if not self._file:
+            return
+
+        loaded_credentials = _load_credentials_file(self._file)
+        self._credentials.update(loaded_credentials)
+
+        logger.debug('Read credential file')
+
+    def _write_credentials(self):
+        if self._read_only:
+            logger.debug('In read-only mode, not writing credentials.')
+            return
+
+        _write_credentials_file(self._file, self._credentials)
+        logger.debug('Wrote credential file {0}.'.format(self._filename))
+
+    def acquire_lock(self):
+        self._thread_lock.acquire()
+        locked = self._process_lock.acquire(timeout=INTERPROCESS_LOCK_DEADLINE)
+
+        if locked:
+            _create_file_if_needed(self._filename)
+            self._file = open(self._filename, 'r+')
+            self._read_only = False
+
+        else:
+            logger.warn(
+                'Failed to obtain interprocess lock for credentials. '
+                'If a credential is being refreshed, other processes may '
+                'not see the updated access token and refresh as well.')
+            if os.path.exists(self._filename):
+                self._file = open(self._filename, 'r')
+            else:
+                self._file = None
+            self._read_only = True
+
+        self._load_credentials()
+
+    def release_lock(self):
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+
+        if not self._read_only:
+            self._process_lock.release()
+
+        self._thread_lock.release()
+
+    def _refresh_predicate(self, credentials):
+        if credentials is None:
+            return True
+        elif credentials.invalid:
+            return True
+        elif credentials.access_token_expired:
+            return True
+        else:
+            return False
+
+    def locked_get(self, key):
+        # Check if the credential is already in memory.
+        credentials = self._credentials.get(key, None)
+
+        # Use the refresh predicate to determine if the entire store should be
+        # reloaded. This basically checks if the credentials are invalid
+        # or expired. This covers the situation where another process has
+        # refreshed the credentials and this process doesn't know about it yet.
+        # In that case, this process won't needlessly refresh the credentials.
+        if self._refresh_predicate(credentials):
+            self._load_credentials()
+            credentials = self._credentials.get(key, None)
+
+        return credentials
+
+    def locked_put(self, key, credentials):
+        self._load_credentials()
+        self._credentials[key] = credentials
+        self._write_credentials()
+
+    def locked_delete(self, key):
+        self._load_credentials()
+        self._credentials.pop(key, None)
+        self._write_credentials()
+
+
+def _get_backend(filename):
+    """A helper method to get or create a backend with thread locking.
+
+    This ensures that only one backend is used per-file per-process, so that
+    thread and process locks are appropriately shared.
+
+    Args:
+        filename: The full path to the credential storage file.
+
+    Returns:
+        An instance of :class:`_MultiprocessStorageBackend`.
+    """
+    filename = os.path.abspath(filename)
+
+    with _backends_lock:
+        if filename not in _backends:
+            _backends[filename] = _MultiprocessStorageBackend(filename)
+        return _backends[filename]
+
+
+class MultiprocessFileStorage(client.Storage):
+    """Multiprocess file credential storage.
+
+    Args:
+      filename: The path to the file where credentials will be stored.
+      key: An arbitrary string used to uniquely identify this set of
+          credentials. For example, you may use the user's ID as the key or
+          a combination of the client ID and user ID.
+    """
+    def __init__(self, filename, key):
+        self._key = key
+        self._backend = _get_backend(filename)
+
+    def acquire_lock(self):
+        self._backend.acquire_lock()
+
+    def release_lock(self):
+        self._backend.release_lock()
+
+    def locked_get(self):
+        """Retrieves the current credentials from the store.
+
+        Returns:
+            An instance of :class:`oauth2client.client.Credentials` or `None`.
+        """
+        credential = self._backend.locked_get(self._key)
+
+        if credential is not None:
+            credential.set_store(self)
+
+        return credential
+
+    def locked_put(self, credentials):
+        """Writes the given credentials to the store.
+
+        Args:
+            credentials: an instance of
+                :class:`oauth2client.client.Credentials`.
+        """
+        return self._backend.locked_put(self._key, credentials)
+
+    def locked_delete(self):
+        """Deletes the current credentials from the store."""
+        return self._backend.locked_delete(self._key)

src/oauth2client/contrib/multistore_file.py

@@ -50,16 +50,19 @@ import logging
 import os
 import threading
 
-from oauth2client.client import Credentials
-from oauth2client.client import Storage as BaseStorage
+from oauth2client import client
 from oauth2client import util
-from oauth2client.contrib.locked_file import LockedFile
-
+from oauth2client.contrib import locked_file
 
 __author__ = 'jbeda@google.com (Joe Beda)'
 
 logger = logging.getLogger(__name__)
 
+logger.warning(
+    'The oauth2client.contrib.multistore_file module has been deprecated and '
+    'will be removed in the next release of oauth2client. Please migrate to '
+    'multiprocess_file_storage.')
+
 # A dict from 'filename'->_MultiStore instances
 _multistores = {}
 _multistores_lock = threading.Lock()
@@ -108,7 +111,7 @@ def get_credential_storage(filename, client_id, user_agent, scope,
     key = {'clientId': client_id, 'userAgent': user_agent,
            'scope': util.scopes_to_string(scope)}
     return get_credential_storage_custom_key(
-      filename, key, warn_on_readonly=warn_on_readonly)
+        filename, key, warn_on_readonly=warn_on_readonly)
 
 
 @util.positional(2)
@@ -131,7 +134,7 @@ def get_credential_storage_custom_string_key(filename, key_string,
     # Create a key dictionary that can be used
     key_dict = {'key': key_string}
     return get_credential_storage_custom_key(
-      filename, key_dict, warn_on_readonly=warn_on_readonly)
+        filename, key_dict, warn_on_readonly=warn_on_readonly)
 
 
 @util.positional(2)
@@ -209,7 +212,7 @@ class _MultiStore(object):
 
         This will create the file if necessary.
         """
-        self._file = LockedFile(filename, 'r+', 'r')
+        self._file = locked_file.LockedFile(filename, 'r+', 'r')
         self._thread_lock = threading.Lock()
         self._read_only = False
         self._warn_on_readonly = warn_on_readonly
@@ -225,7 +228,7 @@ class _MultiStore(object):
         # If this is None, then the store hasn't been read yet.
         self._data = None
 
-    class _Storage(BaseStorage):
+    class _Storage(client.Storage):
         """A Storage object that can read/write a single credential."""
 
         def __init__(self, multistore, key):
@@ -298,7 +301,7 @@ class _MultiStore(object):
         self._thread_lock.acquire()
         try:
             self._file.open_and_lock()
-        except IOError as e:
+        except (IOError, OSError) as e:
             if e.errno == errno.ENOSYS:
                 logger.warn('File system does not support locking the '
                             'credentials file.')
@@ -319,6 +322,7 @@ class _MultiStore(object):
                             'Opening in read-only mode. Any refreshed '
                             'credentials will only be '
                             'valid for this run.', self._file.filename())
+
         if os.path.getsize(self._file.filename()) == 0:
             logger.debug('Initializing empty multistore file')
             # The multistore is empty so write out an empty file.
@@ -390,8 +394,8 @@ class _MultiStore(object):
                         'corrupt or an old version. Overwriting.')
         if version > 1:
             raise NewerCredentialStoreError(
-                'Credential file has file_version of %d. '
-                'Only file_version of 1 is supported.' % version)
+                'Credential file has file_version of {0}. '
+                'Only file_version of 1 is supported.'.format(version))
 
         credentials = []
         try:
@@ -421,7 +425,7 @@ class _MultiStore(object):
         raw_key = cred_entry['key']
         key = _dict_to_tuple_key(raw_key)
         credential = None
-        credential = Credentials.new_from_json(
+        credential = client.Credentials.new_from_json(
             json.dumps(cred_entry['credential']))
         return (key, credential)
 

src/oauth2client/contrib/sqlalchemy.py

@@ -0,0 +1,173 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""OAuth 2.0 utilities for SQLAlchemy.
+
+Utilities for using OAuth 2.0 in conjunction with a SQLAlchemy.
+
+Configuration
+=============
+
+In order to use this storage, you'll need to create table
+with :class:`oauth2client.contrib.sqlalchemy.CredentialsType` column.
+It's recommended to either put this column on some sort of user info
+table or put the column in a table with a belongs-to relationship to
+a user info table.
+
+Here's an example of a simple table with a :class:`CredentialsType`
+column that's related to a user table by the `user_id` key.
+
+.. code-block:: python
+
+    from sqlalchemy import Column, ForeignKey, Integer
+    from sqlalchemy.ext.declarative import declarative_base
+    from sqlalchemy.orm import relationship
+
+    from oauth2client.contrib.sqlalchemy import CredentialsType
+
+
+    Base = declarative_base()
+
+
+    class Credentials(Base):
+        __tablename__ = 'credentials'
+
+        user_id = Column(Integer, ForeignKey('user.id'))
+        credentials = Column(CredentialsType)
+
+
+    class User(Base):
+        id = Column(Integer, primary_key=True)
+        # bunch of other columns
+        credentials = relationship('Credentials')
+
+
+Usage
+=====
+
+With tables ready, you are now able to store credentials in database.
+We will reuse tables defined above.
+
+.. code-block:: python
+
+    from sqlalchemy.orm import Session
+
+    from oauth2client.client import OAuth2Credentials
+    from oauth2client.contrib.sql_alchemy import Storage
+
+    session = Session()
+    user = session.query(User).first()
+    storage = Storage(
+        session=session,
+        model_class=Credentials,
+        # This is the key column used to identify
+        # the row that stores the credentials.
+        key_name='user_id',
+        key_value=user.id,
+        property_name='credentials',
+    )
+
+    # Store
+    credentials = OAuth2Credentials(...)
+    storage.put(credentials)
+
+    # Retrieve
+    credentials = storage.get()
+
+    # Delete
+    storage.delete()
+
+"""
+
+from __future__ import absolute_import
+
+import sqlalchemy.types
+
+from oauth2client import client
+
+
+class CredentialsType(sqlalchemy.types.PickleType):
+    """Type representing credentials.
+
+    Alias for :class:`sqlalchemy.types.PickleType`.
+    """
+
+
+class Storage(client.Storage):
+    """Store and retrieve a single credential to and from SQLAlchemy.
+    This helper presumes the Credentials
+    have been stored as a Credentials column
+    on a db model class.
+    """
+
+    def __init__(self, session, model_class, key_name,
+                 key_value, property_name):
+        """Constructor for Storage.
+
+        Args:
+            session: An instance of :class:`sqlalchemy.orm.Session`.
+            model_class: SQLAlchemy declarative mapping.
+            key_name: string, key name for the entity that has the credentials
+            key_value: key value for the entity that has the credentials
+            property_name: A string indicating which property on the
+                           ``model_class`` to store the credentials.
+                           This property must be a
+                           :class:`CredentialsType` column.
+        """
+        super(Storage, self).__init__()
+
+        self.session = session
+        self.model_class = model_class
+        self.key_name = key_name
+        self.key_value = key_value
+        self.property_name = property_name
+
+    def locked_get(self):
+        """Retrieve stored credential.
+
+        Returns:
+            A :class:`oauth2client.Credentials` instance or `None`.
+        """
+        filters = {self.key_name: self.key_value}
+        query = self.session.query(self.model_class).filter_by(**filters)
+        entity = query.first()
+
+        if entity:
+            credential = getattr(entity, self.property_name)
+            if credential and hasattr(credential, 'set_store'):
+                credential.set_store(self)
+            return credential
+        else:
+            return None
+
+    def locked_put(self, credentials):
+        """Write a credentials to the SQLAlchemy datastore.
+
+        Args:
+            credentials: :class:`oauth2client.Credentials`
+        """
+        filters = {self.key_name: self.key_value}
+        query = self.session.query(self.model_class).filter_by(**filters)
+        entity = query.first()
+
+        if not entity:
+            entity = self.model_class(**filters)
+
+        setattr(entity, self.property_name, credentials)
+        self.session.add(entity)
+
+    def locked_delete(self):
+        """Delete credentials from the SQLAlchemy datastore."""
+        filters = {self.key_name: self.key_value}
+        self.session.query(self.model_class).filter_by(**filters).delete()

src/oauth2client/contrib/xsrfutil.py

@@ -19,7 +19,7 @@ import binascii
 import hmac
 import time
 
-from oauth2client._helpers import _to_bytes
+from oauth2client import _helpers
 from oauth2client import util
 
 __authors__ = [
@@ -49,12 +49,12 @@ def generate_token(key, user_id, action_id='', when=None):
     Returns:
         A string XSRF protection token.
     """
-    digester = hmac.new(_to_bytes(key, encoding='utf-8'))
-    digester.update(_to_bytes(str(user_id), encoding='utf-8'))
+    digester = hmac.new(_helpers._to_bytes(key, encoding='utf-8'))
+    digester.update(_helpers._to_bytes(str(user_id), encoding='utf-8'))
     digester.update(DELIMITER)
-    digester.update(_to_bytes(action_id, encoding='utf-8'))
+    digester.update(_helpers._to_bytes(action_id, encoding='utf-8'))
     digester.update(DELIMITER)
-    when = _to_bytes(str(when or int(time.time())), encoding='utf-8')
+    when = _helpers._to_bytes(str(when or int(time.time())), encoding='utf-8')
     digester.update(when)
     digest = digester.digest()
 

src/oauth2client/crypt.py

@@ -19,15 +19,13 @@ import json
 import logging
 import time
 
-from oauth2client._helpers import _from_bytes
-from oauth2client._helpers import _json_encode
-from oauth2client._helpers import _to_bytes
-from oauth2client._helpers import _urlsafe_b64decode
-from oauth2client._helpers import _urlsafe_b64encode
-from oauth2client._pure_python_crypt import RsaSigner
-from oauth2client._pure_python_crypt import RsaVerifier
+from oauth2client import _helpers
+from oauth2client import _pure_python_crypt
 
 
+RsaSigner = _pure_python_crypt.RsaSigner
+RsaVerifier = _pure_python_crypt.RsaVerifier
+
 CLOCK_SKEW_SECS = 300  # 5 minutes in seconds
 AUTH_TOKEN_LIFETIME_SECS = 300  # 5 minutes in seconds
 MAX_TOKEN_LIFETIME_SECS = 86400  # 1 day in seconds
@@ -44,17 +42,19 @@ def _bad_pkcs12_key_as_pem(*args, **kwargs):
 
 
 try:
-    from oauth2client._openssl_crypt import OpenSSLVerifier
-    from oauth2client._openssl_crypt import OpenSSLSigner
-    from oauth2client._openssl_crypt import pkcs12_key_as_pem
+    from oauth2client import _openssl_crypt
+    OpenSSLSigner = _openssl_crypt.OpenSSLSigner
+    OpenSSLVerifier = _openssl_crypt.OpenSSLVerifier
+    pkcs12_key_as_pem = _openssl_crypt.pkcs12_key_as_pem
 except ImportError:  # pragma: NO COVER
     OpenSSLVerifier = None
     OpenSSLSigner = None
     pkcs12_key_as_pem = _bad_pkcs12_key_as_pem
 
 try:
-    from oauth2client._pycrypto_crypt import PyCryptoVerifier
-    from oauth2client._pycrypto_crypt import PyCryptoSigner
+    from oauth2client import _pycrypto_crypt
+    PyCryptoSigner = _pycrypto_crypt.PyCryptoSigner
+    PyCryptoVerifier = _pycrypto_crypt.PyCryptoVerifier
 except ImportError:  # pragma: NO COVER
     PyCryptoVerifier = None
     PyCryptoSigner = None
@@ -89,13 +89,13 @@ def make_signed_jwt(signer, payload, key_id=None):
         header['kid'] = key_id
 
     segments = [
-        _urlsafe_b64encode(_json_encode(header)),
-        _urlsafe_b64encode(_json_encode(payload)),
+        _helpers._urlsafe_b64encode(_helpers._json_encode(header)),
+        _helpers._urlsafe_b64encode(_helpers._json_encode(payload)),
     ]
     signing_input = b'.'.join(segments)
 
     signature = signer.sign(signing_input)
-    segments.append(_urlsafe_b64encode(signature))
+    segments.append(_helpers._urlsafe_b64encode(signature))
 
     logger.debug(str(segments))
 
@@ -144,11 +144,11 @@ def _check_audience(payload_dict, audience):
 
     audience_in_payload = payload_dict.get('aud')
     if audience_in_payload is None:
-        raise AppIdentityError('No aud field in token: %s' %
-                               (payload_dict,))
+        raise AppIdentityError(
+            'No aud field in token: {0}'.format(payload_dict))
     if audience_in_payload != audience:
-        raise AppIdentityError('Wrong recipient, %s != %s: %s' %
-                               (audience_in_payload, audience, payload_dict))
+        raise AppIdentityError('Wrong recipient, {0} != {1}: {2}'.format(
+            audience_in_payload, audience, payload_dict))
 
 
 def _verify_time_range(payload_dict):
@@ -180,26 +180,28 @@ def _verify_time_range(payload_dict):
     # Make sure issued at and expiration are in the payload.
     issued_at = payload_dict.get('iat')
     if issued_at is None:
-        raise AppIdentityError('No iat field in token: %s' % (payload_dict,))
+        raise AppIdentityError(
+            'No iat field in token: {0}'.format(payload_dict))
     expiration = payload_dict.get('exp')
     if expiration is None:
-        raise AppIdentityError('No exp field in token: %s' % (payload_dict,))
+        raise AppIdentityError(
+            'No exp field in token: {0}'.format(payload_dict))
 
     # Make sure the expiration gives an acceptable token lifetime.
     if expiration >= now + MAX_TOKEN_LIFETIME_SECS:
-        raise AppIdentityError('exp field too far in future: %s' %
-                               (payload_dict,))
+        raise AppIdentityError(
+            'exp field too far in future: {0}'.format(payload_dict))
 
     # Make sure (up to clock skew) that the token wasn't issued in the future.
     earliest = issued_at - CLOCK_SKEW_SECS
     if now < earliest:
-        raise AppIdentityError('Token used too early, %d < %d: %s' %
-                               (now, earliest, payload_dict))
+        raise AppIdentityError('Token used too early, {0} < {1}: {2}'.format(
+            now, earliest, payload_dict))
     # Make sure (up to clock skew) that the token isn't already expired.
     latest = expiration + CLOCK_SKEW_SECS
     if now > latest:
-        raise AppIdentityError('Token used too late, %d > %d: %s' %
-                               (now, latest, payload_dict))
+        raise AppIdentityError('Token used too late, {0} > {1}: {2}'.format(
+            now, latest, payload_dict))
 
 
 def verify_signed_jwt_with_certs(jwt, certs, audience=None):
@@ -219,22 +221,22 @@ def verify_signed_jwt_with_certs(jwt, certs, audience=None):
     Raises:
         AppIdentityError: if any checks are failed.
     """
-    jwt = _to_bytes(jwt)
+    jwt = _helpers._to_bytes(jwt)
 
     if jwt.count(b'.') != 2:
         raise AppIdentityError(
-            'Wrong number of segments in token: %s' % (jwt,))
+            'Wrong number of segments in token: {0}'.format(jwt))
 
     header, payload, signature = jwt.split(b'.')
     message_to_sign = header + b'.' + payload
-    signature = _urlsafe_b64decode(signature)
+    signature = _helpers._urlsafe_b64decode(signature)
 
     # Parse token.
-    payload_bytes = _urlsafe_b64decode(payload)
+    payload_bytes = _helpers._urlsafe_b64decode(payload)
     try:
-        payload_dict = json.loads(_from_bytes(payload_bytes))
+        payload_dict = json.loads(_helpers._from_bytes(payload_bytes))
     except:
-        raise AppIdentityError('Can\'t parse token: %s' % (payload_bytes,))
+        raise AppIdentityError('Can\'t parse token: {0}'.format(payload_bytes))
 
     # Verify that the signature matches the message.
     _verify_signature(message_to_sign, signature, certs.values())

src/oauth2client/file.py

@@ -21,8 +21,7 @@ credentials.
 import os
 import threading
 
-from oauth2client.client import Credentials
-from oauth2client.client import Storage as BaseStorage
+from oauth2client import client
 
 
 __author__ = 'jcgregorio@google.com (Joe Gregorio)'
@@ -32,7 +31,7 @@ class CredentialsFileSymbolicLinkError(Exception):
     """Credentials files must not be symbolic links."""
 
 
-class Storage(BaseStorage):
+class Storage(client.Storage):
     """Store and retrieve a single credential to and from a file."""
 
     def __init__(self, filename):
@@ -42,7 +41,7 @@ class Storage(BaseStorage):
     def _validate_file(self):
         if os.path.islink(self._filename):
             raise CredentialsFileSymbolicLinkError(
-                'File: %s is a symbolic link.' % self._filename)
+                'File: {0} is a symbolic link.'.format(self._filename))
 
     def locked_get(self):
         """Retrieve Credential from file.
@@ -63,7 +62,7 @@ class Storage(BaseStorage):
             return credentials
 
         try:
-            credentials = Credentials.new_from_json(content)
+            credentials = client.Credentials.new_from_json(content)
             credentials.set_store(self)
         except ValueError:
             pass

src/oauth2client/service_account.py

@@ -20,16 +20,12 @@ import datetime
 import json
 import time
 
-from oauth2client import GOOGLE_REVOKE_URI
-from oauth2client import GOOGLE_TOKEN_URI
-from oauth2client._helpers import _json_encode
-from oauth2client._helpers import _from_bytes
-from oauth2client._helpers import _urlsafe_b64encode
-from oauth2client import util
-from oauth2client.client import AssertionCredentials
-from oauth2client.client import EXPIRY_FORMAT
-from oauth2client.client import SERVICE_ACCOUNT
+import oauth2client
+from oauth2client import _helpers
+from oauth2client import client
 from oauth2client import crypt
+from oauth2client import transport
+from oauth2client import util
 
 
 _PASSWORD_DEFAULT = 'notasecret'
@@ -44,7 +40,7 @@ to .pem format:
 """
 
 
-class ServiceAccountCredentials(AssertionCredentials):
+class ServiceAccountCredentials(client.AssertionCredentials):
     """Service Account credential for OAuth 2.0 signed JWT grants.
 
     Supports
@@ -73,6 +69,12 @@ class ServiceAccountCredentials(AssertionCredentials):
                    service account.
         user_agent: string, (Optional) User agent to use when sending
                     request.
+        token_uri: string, URI for token endpoint. For convenience defaults
+                   to Google's endpoints but any OAuth 2.0 provider can be
+                   used.
+        revoke_uri: string, URI for revoke endpoint.  For convenience defaults
+                   to Google's endpoints but any OAuth 2.0 provider can be
+                   used.
         kwargs: dict, Extra key-value pairs (both strings) to send in the
                 payload body when making an assertion.
     """
@@ -80,9 +82,9 @@ class ServiceAccountCredentials(AssertionCredentials):
     MAX_TOKEN_LIFETIME_SECS = 3600
     """Max lifetime of the token (one hour, in seconds)."""
 
-    NON_SERIALIZED_MEMBERS =  (
+    NON_SERIALIZED_MEMBERS = (
         frozenset(['_signer']) |
-        AssertionCredentials.NON_SERIALIZED_MEMBERS)
+        client.AssertionCredentials.NON_SERIALIZED_MEMBERS)
     """Members that aren't serialized when object is converted to JSON."""
 
     # Can be over-ridden by factory constructors. Used for
@@ -98,10 +100,13 @@ class ServiceAccountCredentials(AssertionCredentials):
                  private_key_id=None,
                  client_id=None,
                  user_agent=None,
+                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
                  **kwargs):
 
         super(ServiceAccountCredentials, self).__init__(
-            None, user_agent=user_agent)
+            None, user_agent=user_agent, token_uri=token_uri,
+            revoke_uri=revoke_uri)
 
         self._service_account_email = service_account_email
         self._signer = signer
@@ -121,8 +126,8 @@ class ServiceAccountCredentials(AssertionCredentials):
             strip: array, An array of names of members to exclude from the
                    JSON.
             to_serialize: dict, (Optional) The properties for this object
-                          that will be serialized. This allows callers to modify
-                          before serializing.
+                          that will be serialized. This allows callers to
+                          modify before serializing.
 
         Returns:
             string, a JSON representation of this instance, suitable to pass to
@@ -137,7 +142,8 @@ class ServiceAccountCredentials(AssertionCredentials):
             strip, to_serialize=to_serialize)
 
     @classmethod
-    def _from_parsed_json_keyfile(cls, keyfile_dict, scopes):
+    def _from_parsed_json_keyfile(cls, keyfile_dict, scopes,
+                                  token_uri=None, revoke_uri=None):
         """Helper for factory constructors from JSON keyfile.
 
         Args:
@@ -145,6 +151,12 @@ class ServiceAccountCredentials(AssertionCredentials):
                           containing the contents of the JSON keyfile.
             scopes: List or string, Scopes to use when acquiring an
                     access token.
+            token_uri: string, URI for OAuth 2.0 provider token endpoint.
+                       If unset and not present in keyfile_dict, defaults
+                       to Google's endpoints.
+            revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint.
+                       If unset and not present in keyfile_dict, defaults
+                       to Google's endpoints.
 
         Returns:
             ServiceAccountCredentials, a credentials object created from
@@ -156,30 +168,45 @@ class ServiceAccountCredentials(AssertionCredentials):
                 the keyfile.
         """
         creds_type = keyfile_dict.get('type')
-        if creds_type != SERVICE_ACCOUNT:
+        if creds_type != client.SERVICE_ACCOUNT:
             raise ValueError('Unexpected credentials type', creds_type,
-                             'Expected', SERVICE_ACCOUNT)
+                             'Expected', client.SERVICE_ACCOUNT)
 
         service_account_email = keyfile_dict['client_email']
         private_key_pkcs8_pem = keyfile_dict['private_key']
         private_key_id = keyfile_dict['private_key_id']
         client_id = keyfile_dict['client_id']
+        if not token_uri:
+            token_uri = keyfile_dict.get('token_uri',
+                                         oauth2client.GOOGLE_TOKEN_URI)
+        if not revoke_uri:
+            revoke_uri = keyfile_dict.get('revoke_uri',
+                                          oauth2client.GOOGLE_REVOKE_URI)
 
         signer = crypt.Signer.from_string(private_key_pkcs8_pem)
         credentials = cls(service_account_email, signer, scopes=scopes,
                           private_key_id=private_key_id,
-                          client_id=client_id)
+                          client_id=client_id, token_uri=token_uri,
+                          revoke_uri=revoke_uri)
         credentials._private_key_pkcs8_pem = private_key_pkcs8_pem
         return credentials
 
     @classmethod
-    def from_json_keyfile_name(cls, filename, scopes=''):
+    def from_json_keyfile_name(cls, filename, scopes='',
+                               token_uri=None, revoke_uri=None):
+
         """Factory constructor from JSON keyfile by name.
 
         Args:
             filename: string, The location of the keyfile.
             scopes: List or string, (Optional) Scopes to use when acquiring an
                     access token.
+            token_uri: string, URI for OAuth 2.0 provider token endpoint.
+                       If unset and not present in the key file, defaults
+                       to Google's endpoints.
+            revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint.
+                       If unset and not present in the key file, defaults
+                       to Google's endpoints.
 
         Returns:
             ServiceAccountCredentials, a credentials object created from
@@ -192,10 +219,13 @@ class ServiceAccountCredentials(AssertionCredentials):
         """
         with open(filename, 'r') as file_obj:
             client_credentials = json.load(file_obj)
-        return cls._from_parsed_json_keyfile(client_credentials, scopes)
+        return cls._from_parsed_json_keyfile(client_credentials, scopes,
+                                             token_uri=token_uri,
+                                             revoke_uri=revoke_uri)
 
     @classmethod
-    def from_json_keyfile_dict(cls, keyfile_dict, scopes=''):
+    def from_json_keyfile_dict(cls, keyfile_dict, scopes='',
+                               token_uri=None, revoke_uri=None):
         """Factory constructor from parsed JSON keyfile.
 
         Args:
@@ -203,6 +233,12 @@ class ServiceAccountCredentials(AssertionCredentials):
                           containing the contents of the JSON keyfile.
             scopes: List or string, (Optional) Scopes to use when acquiring an
                     access token.
+            token_uri: string, URI for OAuth 2.0 provider token endpoint.
+                       If unset and not present in keyfile_dict, defaults
+                       to Google's endpoints.
+            revoke_uri: string, URI for OAuth 2.0 provider revoke endpoint.
+                       If unset and not present in keyfile_dict, defaults
+                       to Google's endpoints.
 
         Returns:
             ServiceAccountCredentials, a credentials object created from
@@ -213,12 +249,16 @@ class ServiceAccountCredentials(AssertionCredentials):
             KeyError, if one of the expected keys is not present in
                 the keyfile.
         """
-        return cls._from_parsed_json_keyfile(keyfile_dict, scopes)
+        return cls._from_parsed_json_keyfile(keyfile_dict, scopes,
+                                             token_uri=token_uri,
+                                             revoke_uri=revoke_uri)
 
     @classmethod
     def _from_p12_keyfile_contents(cls, service_account_email,
                                    private_key_pkcs12,
-                                   private_key_password=None, scopes=''):
+                                   private_key_password=None, scopes='',
+                                   token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                                   revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
         """Factory constructor from JSON keyfile.
 
         Args:
@@ -229,6 +269,12 @@ class ServiceAccountCredentials(AssertionCredentials):
                                   private key. Defaults to ``notasecret``.
             scopes: List or string, (Optional) Scopes to use when acquiring an
                     access token.
+            token_uri: string, URI for token endpoint. For convenience defaults
+                       to Google's endpoints but any OAuth 2.0 provider can be
+                       used.
+            revoke_uri: string, URI for revoke endpoint. For convenience
+                        defaults to Google's endpoints but any OAuth 2.0
+                        provider can be used.
 
         Returns:
             ServiceAccountCredentials, a credentials object created from
@@ -244,14 +290,18 @@ class ServiceAccountCredentials(AssertionCredentials):
             raise NotImplementedError(_PKCS12_ERROR)
         signer = crypt.Signer.from_string(private_key_pkcs12,
                                           private_key_password)
-        credentials = cls(service_account_email, signer, scopes=scopes)
+        credentials = cls(service_account_email, signer, scopes=scopes,
+                          token_uri=token_uri, revoke_uri=revoke_uri)
         credentials._private_key_pkcs12 = private_key_pkcs12
         credentials._private_key_password = private_key_password
         return credentials
 
     @classmethod
     def from_p12_keyfile(cls, service_account_email, filename,
-                         private_key_password=None, scopes=''):
+                         private_key_password=None, scopes='',
+                         token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                         revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
+
         """Factory constructor from JSON keyfile.
 
         Args:
@@ -262,6 +312,12 @@ class ServiceAccountCredentials(AssertionCredentials):
                                   private key. Defaults to ``notasecret``.
             scopes: List or string, (Optional) Scopes to use when acquiring an
                     access token.
+            token_uri: string, URI for token endpoint. For convenience defaults
+                       to Google's endpoints but any OAuth 2.0 provider can be
+                       used.
+            revoke_uri: string, URI for revoke endpoint. For convenience
+                        defaults to Google's endpoints but any OAuth 2.0
+                        provider can be used.
 
         Returns:
             ServiceAccountCredentials, a credentials object created from
@@ -275,11 +331,14 @@ class ServiceAccountCredentials(AssertionCredentials):
             private_key_pkcs12 = file_obj.read()
         return cls._from_p12_keyfile_contents(
             service_account_email, private_key_pkcs12,
-            private_key_password=private_key_password, scopes=scopes)
+            private_key_password=private_key_password, scopes=scopes,
+            token_uri=token_uri, revoke_uri=revoke_uri)
 
     @classmethod
     def from_p12_keyfile_buffer(cls, service_account_email, file_buffer,
-                                private_key_password=None, scopes=''):
+                                private_key_password=None, scopes='',
+                                token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                                revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
         """Factory constructor from JSON keyfile.
 
         Args:
@@ -291,6 +350,12 @@ class ServiceAccountCredentials(AssertionCredentials):
                                   private key. Defaults to ``notasecret``.
             scopes: List or string, (Optional) Scopes to use when acquiring an
                     access token.
+            token_uri: string, URI for token endpoint. For convenience defaults
+                       to Google's endpoints but any OAuth 2.0 provider can be
+                       used.
+            revoke_uri: string, URI for revoke endpoint. For convenience
+                        defaults to Google's endpoints but any OAuth 2.0
+                        provider can be used.
 
         Returns:
             ServiceAccountCredentials, a credentials object created from
@@ -303,7 +368,8 @@ class ServiceAccountCredentials(AssertionCredentials):
         private_key_pkcs12 = file_buffer.read()
         return cls._from_p12_keyfile_contents(
             service_account_email, private_key_pkcs12,
-            private_key_password=private_key_password, scopes=scopes)
+            private_key_password=private_key_password, scopes=scopes,
+            token_uri=token_uri, revoke_uri=revoke_uri)
 
     def _generate_assertion(self):
         """Generate the assertion that will be used in the request."""
@@ -368,7 +434,7 @@ class ServiceAccountCredentials(AssertionCredentials):
             ServiceAccountCredentials from the serialized data.
         """
         if not isinstance(json_data, dict):
-            json_data = json.loads(_from_bytes(json_data))
+            json_data = json.loads(_helpers._from_bytes(json_data))
 
         private_key_pkcs8_pem = None
         pkcs12_val = json_data.get(_PKCS12_KEY)
@@ -406,7 +472,7 @@ class ServiceAccountCredentials(AssertionCredentials):
         token_expiry = json_data.get('token_expiry', None)
         if token_expiry is not None:
             credentials.token_expiry = datetime.datetime.strptime(
-                token_expiry, EXPIRY_FORMAT)
+                token_expiry, client.EXPIRY_FORMAT)
         return credentials
 
     def create_scoped_required(self):
@@ -427,6 +493,33 @@ class ServiceAccountCredentials(AssertionCredentials):
         result._private_key_password = self._private_key_password
         return result
 
+    def create_with_claims(self, claims):
+        """Create credentials that specify additional claims.
+
+        Args:
+            claims: dict, key-value pairs for claims.
+
+        Returns:
+            ServiceAccountCredentials, a copy of the current service account
+            credentials with updated claims to use when obtaining access
+            tokens.
+        """
+        new_kwargs = dict(self._kwargs)
+        new_kwargs.update(claims)
+        result = self.__class__(self._service_account_email,
+                                self._signer,
+                                scopes=self._scopes,
+                                private_key_id=self._private_key_id,
+                                client_id=self.client_id,
+                                user_agent=self._user_agent,
+                                **new_kwargs)
+        result.token_uri = self.token_uri
+        result.revoke_uri = self.revoke_uri
+        result._private_key_pkcs8_pem = self._private_key_pkcs8_pem
+        result._private_key_pkcs12 = self._private_key_pkcs12
+        result._private_key_password = self._private_key_password
+        return result
+
     def create_delegated(self, sub):
         """Create credentials that act as domain-wide delegation of authority.
 
@@ -446,18 +539,135 @@ class ServiceAccountCredentials(AssertionCredentials):
             ServiceAccountCredentials, a copy of the current service account
             updated to act on behalf of ``sub``.
         """
-        new_kwargs = dict(self._kwargs)
-        new_kwargs['sub'] = sub
-        result = self.__class__(self._service_account_email,
-                                self._signer,
-                                scopes=self._scopes,
-                                private_key_id=self._private_key_id,
-                                client_id=self.client_id,
-                                user_agent=self._user_agent,
-                                **new_kwargs)
-        result.token_uri = self.token_uri
-        result.revoke_uri = self.revoke_uri
-        result._private_key_pkcs8_pem = self._private_key_pkcs8_pem
-        result._private_key_pkcs12 = self._private_key_pkcs12
-        result._private_key_password = self._private_key_password
+        return self.create_with_claims({'sub': sub})
+
+
+def _datetime_to_secs(utc_time):
+    # TODO(issue 298): use time_delta.total_seconds()
+    # time_delta.total_seconds() not supported in Python 2.6
+    epoch = datetime.datetime(1970, 1, 1)
+    time_delta = utc_time - epoch
+    return time_delta.days * 86400 + time_delta.seconds
+
+
+class _JWTAccessCredentials(ServiceAccountCredentials):
+    """Self signed JWT credentials.
+
+    Makes an assertion to server using a self signed JWT from service account
+    credentials.  These credentials do NOT use OAuth 2.0 and instead
+    authenticate directly.
+    """
+    _MAX_TOKEN_LIFETIME_SECS = 3600
+    """Max lifetime of the token (one hour, in seconds)."""
+
+    def __init__(self,
+                 service_account_email,
+                 signer,
+                 scopes=None,
+                 private_key_id=None,
+                 client_id=None,
+                 user_agent=None,
+                 token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                 revoke_uri=oauth2client.GOOGLE_REVOKE_URI,
+                 additional_claims=None):
+        if additional_claims is None:
+            additional_claims = {}
+        super(_JWTAccessCredentials, self).__init__(
+            service_account_email,
+            signer,
+            private_key_id=private_key_id,
+            client_id=client_id,
+            user_agent=user_agent,
+            token_uri=token_uri,
+            revoke_uri=revoke_uri,
+            **additional_claims)
+
+    def authorize(self, http):
+        """Authorize an httplib2.Http instance with a JWT assertion.
+
+        Unless specified, the 'aud' of the assertion will be the base
+        uri of the request.
+
+        Args:
+            http: An instance of ``httplib2.Http`` or something that acts
+                  like it.
+        Returns:
+            A modified instance of http that was passed in.
+        Example::
+            h = httplib2.Http()
+            h = credentials.authorize(h)
+        """
+        transport.wrap_http_for_jwt_access(self, http)
+        return http
+
+    def get_access_token(self, http=None, additional_claims=None):
+        """Create a signed jwt.
+
+        Args:
+            http: unused
+            additional_claims: dict, additional claims to add to
+                the payload of the JWT.
+        Returns:
+            An AccessTokenInfo with the signed jwt
+        """
+        if additional_claims is None:
+            if self.access_token is None or self.access_token_expired:
+                self.refresh(None)
+            return client.AccessTokenInfo(
+              access_token=self.access_token, expires_in=self._expires_in())
+        else:
+            # Create a 1 time token
+            token, unused_expiry = self._create_token(additional_claims)
+            return client.AccessTokenInfo(
+              access_token=token, expires_in=self._MAX_TOKEN_LIFETIME_SECS)
+
+    def revoke(self, http):
+        """Cannot revoke JWTAccessCredentials tokens."""
+        pass
+
+    def create_scoped_required(self):
+        # JWTAccessCredentials are unscoped by definition
+        return True
+
+    def create_scoped(self, scopes, token_uri=oauth2client.GOOGLE_TOKEN_URI,
+                      revoke_uri=oauth2client.GOOGLE_REVOKE_URI):
+        # Returns an OAuth2 credentials with the given scope
+        result = ServiceAccountCredentials(self._service_account_email,
+                                           self._signer,
+                                           scopes=scopes,
+                                           private_key_id=self._private_key_id,
+                                           client_id=self.client_id,
+                                           user_agent=self._user_agent,
+                                           token_uri=token_uri,
+                                           revoke_uri=revoke_uri,
+                                           **self._kwargs)
+        if self._private_key_pkcs8_pem is not None:
+            result._private_key_pkcs8_pem = self._private_key_pkcs8_pem
+        if self._private_key_pkcs12 is not None:
+            result._private_key_pkcs12 = self._private_key_pkcs12
+        if self._private_key_password is not None:
+            result._private_key_password = self._private_key_password
         return result
+
+    def refresh(self, http):
+        self._refresh(None)
+
+    def _refresh(self, http_request):
+        self.access_token, self.token_expiry = self._create_token()
+
+    def _create_token(self, additional_claims=None):
+        now = client._UTCNOW()
+        lifetime = datetime.timedelta(seconds=self._MAX_TOKEN_LIFETIME_SECS)
+        expiry = now + lifetime
+        payload = {
+            'iat': _datetime_to_secs(now),
+            'exp': _datetime_to_secs(expiry),
+            'iss': self._service_account_email,
+            'sub': self._service_account_email
+        }
+        payload.update(self._kwargs)
+        if additional_claims is not None:
+            payload.update(additional_claims)
+        jwt = crypt.make_signed_jwt(self._signer, payload,
+                                    key_id=self._private_key_id)
+        return jwt.decode('ascii'), expiry

src/oauth2client/tools.py

@@ -27,8 +27,8 @@ import sys
 
 from six.moves import BaseHTTPServer
 from six.moves import http_client
-from six.moves import urllib
 from six.moves import input
+from six.moves import urllib
 
 from oauth2client import client
 from oauth2client import util
@@ -42,17 +42,43 @@ _CLIENT_SECRETS_MESSAGE = """WARNING: Please configure OAuth 2.0
 To make this sample run you will need to populate the client_secrets.json file
 found at:
 
-   %s
+   {file_path}
 
 with information from the APIs Console <https://code.google.com/apis/console>.
 
 """
 
+_FAILED_START_MESSAGE = """
+Failed to start a local webserver listening on either port 8080
+or port 8090. Please check your firewall settings and locally
+running programs that may be blocking or using those ports.
+
+Falling back to --noauth_local_webserver and continuing with
+authorization.
+"""
+
+_BROWSER_OPENED_MESSAGE = """
+Your browser has been opened to visit:
+
+    {address}
+
+If your browser is on a different machine then exit and re-run this
+application with the command-line parameter
+
+  --noauth_local_webserver
+"""
+
+_GO_TO_LINK_MESSAGE = """
+Go to the following link in your browser:
+
+    {address}
+"""
+
 
 def _CreateArgumentParser():
     try:
         import argparse
-    except ImportError:
+    except ImportError:  # pragma: NO COVER
         return None
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument('--auth_host_name', default='localhost',
@@ -182,17 +208,11 @@ def run_flow(flow, storage, flags=None, http=None):
                 break
         flags.noauth_local_webserver = not success
         if not success:
-            print('Failed to start a local webserver listening '
-                  'on either port 8080')
-            print('or port 8090. Please check your firewall settings and locally')
-            print('running programs that may be blocking or using those ports.')
-            print()
-            print('Falling back to --noauth_local_webserver and continuing with')
-            print('authorization.')
-            print()
+            print(_FAILED_START_MESSAGE)
 
     if not flags.noauth_local_webserver:
-        oauth_callback = 'http://%s:%s/' % (flags.auth_host_name, port_number)
+        oauth_callback = 'http://{host}:{port}/'.format(
+            host=flags.auth_host_name, port=port_number)
     else:
         oauth_callback = client.OOB_CALLBACK_URN
     flow.redirect_uri = oauth_callback
@@ -211,21 +231,9 @@ def run_flow(flow, storage, flags=None, http=None):
     if not flags.noauth_local_webserver:
         import webbrowser
         webbrowser.open(authorize_url, new=1, autoraise=True)
-        print('Your browser has been opened to visit:')
-        print()
-        print('    ' + authorize_url)
-        print()
-        print('If your browser is on a different machine then '
-              'exit and re-run this')
-        print('application with the command-line parameter ')
-        print()
-        print('  --noauth_local_webserver')
-        print()
+        print(_BROWSER_OPENED_MESSAGE.format(address=authorize_url))
     else:
-        print('Go to the following link in your browser:')
-        print()
-        print('    ' + authorize_url)
-        print()
+        print(_GO_TO_LINK_MESSAGE.format(address=authorize_url))
 
     code = None
     if not flags.noauth_local_webserver:
@@ -244,7 +252,7 @@ def run_flow(flow, storage, flags=None, http=None):
     try:
         credential = flow.step2_exchange(code, http=http)
     except client.FlowExchangeError as e:
-        sys.exit('Authentication has failed: %s' % e)
+        sys.exit('Authentication has failed: {0}'.format(e))
 
     storage.put(credential)
     credential.set_store(storage)
@@ -255,4 +263,4 @@ def run_flow(flow, storage, flags=None, http=None):
 
 def message_if_missing(filename):
     """Helpful message to display if the CLIENT_SECRETS file is missing."""
-    return _CLIENT_SECRETS_MESSAGE % filename
+    return _CLIENT_SECRETS_MESSAGE.format(file_path=filename)

src/oauth2client/transport.py

@@ -0,0 +1,245 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+import httplib2
+import six
+from six.moves import http_client
+
+from oauth2client._helpers import _to_bytes
+
+
+_LOGGER = logging.getLogger(__name__)
+# Properties present in file-like streams / buffers.
+_STREAM_PROPERTIES = ('read', 'seek', 'tell')
+
+# Google Data client libraries may need to set this to [401, 403].
+REFRESH_STATUS_CODES = (http_client.UNAUTHORIZED,)
+
+
+class MemoryCache(object):
+    """httplib2 Cache implementation which only caches locally."""
+
+    def __init__(self):
+        self.cache = {}
+
+    def get(self, key):
+        return self.cache.get(key)
+
+    def set(self, key, value):
+        self.cache[key] = value
+
+    def delete(self, key):
+        self.cache.pop(key, None)
+
+
+def get_cached_http():
+    """Return an HTTP object which caches results returned.
+
+    This is intended to be used in methods like
+    oauth2client.client.verify_id_token(), which calls to the same URI
+    to retrieve certs.
+
+    Returns:
+        httplib2.Http, an HTTP object with a MemoryCache
+    """
+    return _CACHED_HTTP
+
+
+def get_http_object():
+    """Return a new HTTP object.
+
+    Returns:
+        httplib2.Http, an HTTP object.
+    """
+    return httplib2.Http()
+
+
+def _initialize_headers(headers):
+    """Creates a copy of the headers.
+
+    Args:
+        headers: dict, request headers to copy.
+
+    Returns:
+        dict, the copied headers or a new dictionary if the headers
+        were None.
+    """
+    return {} if headers is None else dict(headers)
+
+
+def _apply_user_agent(headers, user_agent):
+    """Adds a user-agent to the headers.
+
+    Args:
+        headers: dict, request headers to add / modify user
+                 agent within.
+        user_agent: str, the user agent to add.
+
+    Returns:
+        dict, the original headers passed in, but modified if the
+        user agent is not None.
+    """
+    if user_agent is not None:
+        if 'user-agent' in headers:
+            headers['user-agent'] = (user_agent + ' ' + headers['user-agent'])
+        else:
+            headers['user-agent'] = user_agent
+
+    return headers
+
+
+def clean_headers(headers):
+    """Forces header keys and values to be strings, i.e not unicode.
+
+    The httplib module just concats the header keys and values in a way that
+    may make the message header a unicode string, which, if it then tries to
+    contatenate to a binary request body may result in a unicode decode error.
+
+    Args:
+        headers: dict, A dictionary of headers.
+
+    Returns:
+        The same dictionary but with all the keys converted to strings.
+    """
+    clean = {}
+    try:
+        for k, v in six.iteritems(headers):
+            if not isinstance(k, six.binary_type):
+                k = str(k)
+            if not isinstance(v, six.binary_type):
+                v = str(v)
+            clean[_to_bytes(k)] = _to_bytes(v)
+    except UnicodeEncodeError:
+        from oauth2client.client import NonAsciiHeaderError
+        raise NonAsciiHeaderError(k, ': ', v)
+    return clean
+
+
+def wrap_http_for_auth(credentials, http):
+    """Prepares an HTTP object's request method for auth.
+
+    Wraps HTTP requests with logic to catch auth failures (typically
+    identified via a 401 status code). In the event of failure, tries
+    to refresh the token used and then retry the original request.
+
+    Args:
+        credentials: Credentials, the credentials used to identify
+                     the authenticated user.
+        http: httplib2.Http, an http object to be used to make
+              auth requests.
+    """
+    orig_request_method = http.request
+
+    # The closure that will replace 'httplib2.Http.request'.
+    def new_request(uri, method='GET', body=None, headers=None,
+                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
+                    connection_type=None):
+        if not credentials.access_token:
+            _LOGGER.info('Attempting refresh to obtain '
+                         'initial access_token')
+            credentials._refresh(orig_request_method)
+
+        # Clone and modify the request headers to add the appropriate
+        # Authorization header.
+        headers = _initialize_headers(headers)
+        credentials.apply(headers)
+        _apply_user_agent(headers, credentials.user_agent)
+
+        body_stream_position = None
+        # Check if the body is a file-like stream.
+        if all(getattr(body, stream_prop, None) for stream_prop in
+               _STREAM_PROPERTIES):
+            body_stream_position = body.tell()
+
+        resp, content = orig_request_method(uri, method, body,
+                                            clean_headers(headers),
+                                            redirections, connection_type)
+
+        # A stored token may expire between the time it is retrieved and
+        # the time the request is made, so we may need to try twice.
+        max_refresh_attempts = 2
+        for refresh_attempt in range(max_refresh_attempts):
+            if resp.status not in REFRESH_STATUS_CODES:
+                break
+            _LOGGER.info('Refreshing due to a %s (attempt %s/%s)',
+                         resp.status, refresh_attempt + 1,
+                         max_refresh_attempts)
+            credentials._refresh(orig_request_method)
+            credentials.apply(headers)
+            if body_stream_position is not None:
+                body.seek(body_stream_position)
+
+            resp, content = orig_request_method(uri, method, body,
+                                                clean_headers(headers),
+                                                redirections, connection_type)
+
+        return resp, content
+
+    # Replace the request method with our own closure.
+    http.request = new_request
+
+    # Set credentials as a property of the request method.
+    setattr(http.request, 'credentials', credentials)
+
+
+def wrap_http_for_jwt_access(credentials, http):
+    """Prepares an HTTP object's request method for JWT access.
+
+    Wraps HTTP requests with logic to catch auth failures (typically
+    identified via a 401 status code). In the event of failure, tries
+    to refresh the token used and then retry the original request.
+
+    Args:
+        credentials: _JWTAccessCredentials, the credentials used to identify
+                     a service account that uses JWT access tokens.
+        http: httplib2.Http, an http object to be used to make
+              auth requests.
+    """
+    orig_request_method = http.request
+    wrap_http_for_auth(credentials, http)
+    # The new value of ``http.request`` set by ``wrap_http_for_auth``.
+    authenticated_request_method = http.request
+
+    # The closure that will replace 'httplib2.Http.request'.
+    def new_request(uri, method='GET', body=None, headers=None,
+                    redirections=httplib2.DEFAULT_MAX_REDIRECTS,
+                    connection_type=None):
+        if 'aud' in credentials._kwargs:
+            # Preemptively refresh token, this is not done for OAuth2
+            if (credentials.access_token is None or
+                    credentials.access_token_expired):
+                credentials.refresh(None)
+            return authenticated_request_method(uri, method, body,
+                                                headers, redirections,
+                                                connection_type)
+        else:
+            # If we don't have an 'aud' (audience) claim,
+            # create a 1-time token with the uri root as the audience
+            headers = _initialize_headers(headers)
+            _apply_user_agent(headers, credentials.user_agent)
+            uri_root = uri.split('?', 1)[0]
+            token, unused_expiry = credentials._create_token({'aud': uri_root})
+
+            headers['Authorization'] = 'Bearer ' + token
+            return orig_request_method(uri, method, body,
+                                       clean_headers(headers),
+                                       redirections, connection_type)
+
+    # Replace the request method with our own closure.
+    http.request = new_request
+
+
+_CACHED_HTTP = httplib2.Http(MemoryCache())

src/oauth2client/util.py

@@ -124,16 +124,16 @@ def positional(max_positional_args):
                 plural_s = ''
                 if max_positional_args != 1:
                     plural_s = 's'
-                message = ('%s() takes at most %d positional '
-                           'argument%s (%d given)' % (
-                               wrapped.__name__, max_positional_args,
-                               plural_s, len(args)))
+                message = ('{function}() takes at most {args_max} positional '
+                           'argument{plural} ({args_given} given)'.format(
+                               function=wrapped.__name__,
+                               args_max=max_positional_args,
+                               args_given=len(args),
+                               plural=plural_s))
                 if positional_parameters_enforcement == POSITIONAL_EXCEPTION:
                     raise TypeError(message)
                 elif positional_parameters_enforcement == POSITIONAL_WARNING:
                     logger.warning(message)
-                else:  # IGNORE
-                    pass
             return wrapped(*args, **kwargs)
         return positional_wrapper
 

src/rsa/__init__.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -22,24 +22,21 @@ WARNING: this implementation does not use random padding, compression of the
 cleartext input to prevent repetitions, or other common security improvements.
 Use with care.
 
-If you want to have a more secure implementation, use the functions from the
-``rsa.pkcs1`` module.
-
 """
 
-__author__ = "Sybren Stuvel, Barry Mead and Yesudeep Mangalapilly"
-__date__ = "2016-01-13"
-__version__ = '3.3'
-
 from rsa.key import newkeys, PrivateKey, PublicKey
 from rsa.pkcs1 import encrypt, decrypt, sign, verify, DecryptionError, \
     VerificationError
 
+__author__ = "Sybren Stuvel, Barry Mead and Yesudeep Mangalapilly"
+__date__ = "2016-03-29"
+__version__ = '3.4.2'
+
 # Do doctest if we're run directly
 if __name__ == "__main__":
     import doctest
+
     doctest.testmod()
 
 __all__ = ["newkeys", "encrypt", "decrypt", "sign", "verify", 'PublicKey',
-    'PrivateKey', 'DecryptionError', 'VerificationError']
-
+           'PrivateKey', 'DecryptionError', 'VerificationError']

src/rsa/_compat.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,7 +16,6 @@
 
 """Python compatibility wrappers."""
 
-
 from __future__ import absolute_import
 
 import sys
@@ -42,15 +41,12 @@ else:
     # Else we just assume 64-bit processor keeping up with modern times.
     MACHINE_WORD_SIZE = 64
 
-
 try:
     # < Python3
     unicode_type = unicode
-    have_python3 = False
 except NameError:
     # Python3.
     unicode_type = str
-    have_python3 = True
 
 # Fake byte literals.
 if str is unicode_type:
@@ -68,14 +64,6 @@ except NameError:
 
 b = byte_literal
 
-try:
-    # Python 2.6 or higher.
-    bytes_type = bytes
-except NameError:
-    # Python 2.5
-    bytes_type = str
-
-
 # To avoid calling b() multiple times in tight loops.
 ZERO_BYTE = b('\x00')
 EMPTY_BYTE = b('')
@@ -90,7 +78,7 @@ def is_bytes(obj):
     :returns:
         ``True`` if ``value`` is a byte string; ``False`` otherwise.
     """
-    return isinstance(obj, bytes_type)
+    return isinstance(obj, bytes)
 
 
 def is_integer(obj):

src/rsa/_version133.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,8 +14,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""RSA module
-pri = k[1]                               	//Private part of keys d,p,q
+"""Deprecated version of the RSA module
+
+.. deprecated:: 2.0
+
+    This submodule is deprecated and will be completely removed as of version 4.0.
 
 Module for calculating large primes, and RSA encryption, decryption,
 signing and verification. Includes generating public and private keys.
@@ -34,7 +37,11 @@ __version__ = '1.3.3'
 # NOTE: Python's modulo can return negative numbers. We compensate for
 # this behaviour using the abs() function
 
-from cPickle import dumps, loads
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+from pickle import dumps, loads
 import base64
 import math
 import os
@@ -49,6 +56,9 @@ from rsa._compat import byte
 import warnings
 warnings.warn('Insecure version of the RSA module is imported as %s, be careful'
         % __name__)
+warnings.warn('This submodule is deprecated and will be completely removed as of version 4.0.',
+              DeprecationWarning)
+
 
 def gcd(p, q):
     """Returns the greatest common divisor of p and q
@@ -63,12 +73,6 @@ def gcd(p, q):
 
 def bytes2int(bytes):
     """Converts a list of bytes or a string to an integer
-
-    >>> (128*256 + 64)*256 + + 15
-    8405007
-    >>> l = [128, 64, 15]
-    >>> bytes2int(l)
-    8405007
     """
 
     if not (type(bytes) is types.ListType or type(bytes) is types.StringType):
@@ -85,9 +89,6 @@ def bytes2int(bytes):
 
 def int2bytes(number):
     """Converts a number to a string of bytes
-    
-    >>> bytes2int(int2bytes(123456789))
-    123456789
     """
 
     if not (type(number) is types.LongType or type(number) is types.IntType):
@@ -204,11 +205,6 @@ def randomized_primality_testing(n, k):
 
 def is_prime(number):
     """Returns True if the number is prime, and False otherwise.
-
-    >>> is_prime(42)
-    0
-    >>> is_prime(41)
-    1
     """
 
     """
@@ -228,14 +224,6 @@ def is_prime(number):
 def getprime(nbits):
     """Returns a prime number of max. 'math.ceil(nbits/8)*8' bits. In
     other words: nbits is rounded up to whole bytes.
-
-    >>> p = getprime(8)
-    >>> is_prime(p-1)
-    0
-    >>> is_prime(p)
-    1
-    >>> is_prime(p+1)
-    0
     """
 
     nbytes = int(math.ceil(nbits/8.))
@@ -256,11 +244,6 @@ def getprime(nbits):
 def are_relatively_prime(a, b):
     """Returns True if a and b are relatively prime, and False if they
     are not.
-
-    >>> are_relatively_prime(2, 3)
-    1
-    >>> are_relatively_prime(2, 4)
-    0
     """
 
     d = gcd(a, b)

src/rsa/_version200.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,14 +14,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-"""RSA module
+"""Deprecated version of the RSA module
 
-Module for calculating large primes, and RSA encryption, decryption,
-signing and verification. Includes generating public and private keys.
+.. deprecated:: 3.0
 
-WARNING: this implementation does not use random padding, compression of the
-cleartext input to prevent repetitions, or other common security improvements.
-Use with care.
+    This submodule is deprecated and will be completely removed as of version 4.0.
 
 """
 
@@ -39,6 +36,8 @@ from rsa._compat import byte
 # Display a warning that this insecure version is imported.
 import warnings
 warnings.warn('Insecure version of the RSA module is imported as %s' % __name__)
+warnings.warn('This submodule is deprecated and will be completely removed as of version 4.0.',
+              DeprecationWarning)
 
 
 def bit_size(number):
@@ -59,13 +58,7 @@ def gcd(p, q):
     
 
 def bytes2int(bytes):
-    """Converts a list of bytes or a string to an integer
-
-    >>> (((128 * 256) + 64) * 256) + 15
-    8405007
-    >>> l = [128, 64, 15]
-    >>> bytes2int(l)              #same as bytes2int('\x80@\x0f')
-    8405007
+    r"""Converts a list of bytes or a string to an integer
     """
 
     if not (type(bytes) is types.ListType or type(bytes) is types.StringType):
@@ -99,9 +92,6 @@ def int2bytes(number):
 def to64(number):
     """Converts a number in the range of 0 to 63 into base 64 digit
     character in the range of '0'-'9', 'A'-'Z', 'a'-'z','-','_'.
-    
-    >>> to64(10)
-    'A'
     """
 
     if not (type(number) is types.LongType or type(number) is types.IntType):
@@ -128,9 +118,6 @@ def to64(number):
 def from64(number):
     """Converts an ordinal character value in the range of
     0-9,A-Z,a-z,-,_ to a number in the range of 0-63.
-    
-    >>> from64(49)
-    1
     """
 
     if not (type(number) is types.LongType or type(number) is types.IntType):
@@ -157,9 +144,6 @@ def from64(number):
 def int2str64(number):
     """Converts a number to a string of base64 encoded characters in
     the range of '0'-'9','A'-'Z,'a'-'z','-','_'.
-    
-    >>> int2str64(123456789)
-    '7MyqL'
     """
 
     if not (type(number) is types.LongType or type(number) is types.IntType):
@@ -177,9 +161,6 @@ def int2str64(number):
 def str642int(string):
     """Converts a base64 encoded string into an integer.
     The chars of this string in in the range '0'-'9','A'-'Z','a'-'z','-','_'
-    
-    >>> str642int('7MyqL')
-    123456789
     """
 
     if not (type(string) is types.ListType or type(string) is types.StringType):
@@ -270,11 +251,6 @@ def randomized_primality_testing(n, k):
 
 def is_prime(number):
     """Returns True if the number is prime, and False otherwise.
-
-    >>> is_prime(42)
-    0
-    >>> is_prime(41)
-    1
     """
 
     if randomized_primality_testing(number, 6):
@@ -288,14 +264,6 @@ def is_prime(number):
 def getprime(nbits):
     """Returns a prime number of max. 'math.ceil(nbits/8)*8' bits. In
     other words: nbits is rounded up to whole bytes.
-
-    >>> p = getprime(8)
-    >>> is_prime(p-1)
-    0
-    >>> is_prime(p)
-    1
-    >>> is_prime(p+1)
-    0
     """
 
     while True:

src/rsa/asn1.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,38 +14,40 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''ASN.1 definitions.
+"""ASN.1 definitions.
 
 Not all ASN.1-handling code use these definitions, but when it does, they should be here.
-'''
+"""
 
 from pyasn1.type import univ, namedtype, tag
 
+
 class PubKeyHeader(univ.Sequence):
     componentType = namedtype.NamedTypes(
-        namedtype.NamedType('oid', univ.ObjectIdentifier()),
-        namedtype.NamedType('parameters', univ.Null()),
+            namedtype.NamedType('oid', univ.ObjectIdentifier()),
+            namedtype.NamedType('parameters', univ.Null()),
     )
 
+
 class OpenSSLPubKey(univ.Sequence):
     componentType = namedtype.NamedTypes(
-        namedtype.NamedType('header', PubKeyHeader()),
-        
-        # This little hack (the implicit tag) allows us to get a Bit String as Octet String
-        namedtype.NamedType('key', univ.OctetString().subtype(
-                                          implicitTag=tag.Tag(tagClass=0, tagFormat=0, tagId=3))),
+            namedtype.NamedType('header', PubKeyHeader()),
+
+            # This little hack (the implicit tag) allows us to get a Bit String as Octet String
+            namedtype.NamedType('key', univ.OctetString().subtype(
+                    implicitTag=tag.Tag(tagClass=0, tagFormat=0, tagId=3))),
     )
 
 
 class AsnPubKey(univ.Sequence):
-    '''ASN.1 contents of DER encoded public key:
-    
+    """ASN.1 contents of DER encoded public key:
+
     RSAPublicKey ::= SEQUENCE {
          modulus           INTEGER,  -- n
          publicExponent    INTEGER,  -- e
-    '''
+    """
 
     componentType = namedtype.NamedTypes(
-        namedtype.NamedType('modulus', univ.Integer()),
-        namedtype.NamedType('publicExponent', univ.Integer()),
+            namedtype.NamedType('modulus', univ.Integer()),
+            namedtype.NamedType('publicExponent', univ.Integer()),
     )

src/rsa/bigfile.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,27 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Large file support
+"""Large file support
+
+.. deprecated:: 3.4
+
+    The VARBLOCK format is NOT recommended for general use, has been deprecated since
+    Python-RSA 3.4, and will be removed in a future release. It's vulnerable to a
+    number of attacks:
+
+    1. decrypt/encrypt_bigfile() does not implement `Authenticated encryption`_ nor
+       uses MACs to verify messages before decrypting public key encrypted messages.
+
+    2. decrypt/encrypt_bigfile() does not use hybrid encryption (it uses plain RSA)
+       and has no method for chaining, so block reordering is possible.
+
+    See `issue #19 on Github`_ for more information.
+
+.. _Authenticated encryption: https://en.wikipedia.org/wiki/Authenticated_encryption
+.. _issue #19 on Github: https://github.com/sybrenstuvel/python-rsa/issues/13
+
+
+This module contains functions to:
 
     - break a file into smaller blocks, and encrypt them, and store the
       encrypted blocks in another file.
@@ -37,25 +57,40 @@ The encrypted file format is as follows, where || denotes byte concatenation:
 This file format is called the VARBLOCK format, in line with the varint format
 used to denote the block sizes.
 
-'''
+"""
+
+import warnings
 
 from rsa import key, common, pkcs1, varblock
 from rsa._compat import byte
 
+
 def encrypt_bigfile(infile, outfile, pub_key):
-    '''Encrypts a file, writing it to 'outfile' in VARBLOCK format.
-    
+    """Encrypts a file, writing it to 'outfile' in VARBLOCK format.
+
+    .. deprecated:: 3.4
+        This function was deprecated in Python-RSA version 3.4 due to security issues
+        in the VARBLOCK format. See the documentation_ for more information.
+
+    .. _documentation: https://stuvel.eu/python-rsa-doc/usage.html#working-with-big-files
+
     :param infile: file-like object to read the cleartext from
     :param outfile: file-like object to write the crypto in VARBLOCK format to
     :param pub_key: :py:class:`rsa.PublicKey` to encrypt with
 
-    '''
+    """
+
+    warnings.warn("The 'rsa.bigfile.encrypt_bigfile' function was deprecated in Python-RSA version "
+                  "3.4 due to security issues in the VARBLOCK format. See "
+                  "https://stuvel.eu/python-rsa-doc/usage.html#working-with-big-files "
+                  "for more information.",
+                  DeprecationWarning, stacklevel=2)
 
     if not isinstance(pub_key, key.PublicKey):
         raise TypeError('Public key required, but got %r' % pub_key)
 
     key_bytes = common.bit_size(pub_key.n) // 8
-    blocksize = key_bytes - 11 # keep space for PKCS#1 padding
+    blocksize = key_bytes - 11  # keep space for PKCS#1 padding
 
     # Write the version number to the VARBLOCK file
     outfile.write(byte(varblock.VARBLOCK_VERSION))
@@ -67,21 +102,34 @@ def encrypt_bigfile(infile, outfile, pub_key):
         varblock.write_varint(outfile, len(crypto))
         outfile.write(crypto)
 
+
 def decrypt_bigfile(infile, outfile, priv_key):
-    '''Decrypts an encrypted VARBLOCK file, writing it to 'outfile'
-    
+    """Decrypts an encrypted VARBLOCK file, writing it to 'outfile'
+
+    .. deprecated:: 3.4
+        This function was deprecated in Python-RSA version 3.4 due to security issues
+        in the VARBLOCK format. See the documentation_ for more information.
+
+    .. _documentation: https://stuvel.eu/python-rsa-doc/usage.html#working-with-big-files
+
     :param infile: file-like object to read the crypto in VARBLOCK format from
     :param outfile: file-like object to write the cleartext to
     :param priv_key: :py:class:`rsa.PrivateKey` to decrypt with
 
-    '''
+    """
+
+    warnings.warn("The 'rsa.bigfile.decrypt_bigfile' function was deprecated in Python-RSA version "
+                  "3.4 due to security issues in the VARBLOCK format. See "
+                  "https://stuvel.eu/python-rsa-doc/usage.html#working-with-big-files "
+                  "for more information.",
+                  DeprecationWarning, stacklevel=2)
 
     if not isinstance(priv_key, key.PrivateKey):
         raise TypeError('Private key required, but got %r' % priv_key)
-    
+
     for block in varblock.yield_varblocks(infile):
         cleartext = pkcs1.decrypt(block, priv_key)
         outfile.write(cleartext)
 
-__all__ = ['encrypt_bigfile', 'decrypt_bigfile']
 
+__all__ = ['encrypt_bigfile', 'decrypt_bigfile']

src/rsa/cli.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,10 +14,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Commandline scripts.
+"""Commandline scripts.
 
 These scripts are called by the executables defined in setup.py.
-'''
+"""
 
 from __future__ import with_statement, print_function
 
@@ -31,32 +31,33 @@ import rsa.pkcs1
 
 HASH_METHODS = sorted(rsa.pkcs1.HASH_METHODS.keys())
 
+
 def keygen():
-    '''Key generator.'''
+    """Key generator."""
 
     # Parse the CLI options
     parser = OptionParser(usage='usage: %prog [options] keysize',
-            description='Generates a new RSA keypair of "keysize" bits.')
-    
+                          description='Generates a new RSA keypair of "keysize" bits.')
+
     parser.add_option('--pubout', type='string',
-            help='Output filename for the public key. The public key is '
-            'not saved if this option is not present. You can use '
-            'pyrsa-priv2pub to create the public key file later.')
-    
+                      help='Output filename for the public key. The public key is '
+                           'not saved if this option is not present. You can use '
+                           'pyrsa-priv2pub to create the public key file later.')
+
     parser.add_option('-o', '--out', type='string',
-            help='Output filename for the private key. The key is '
-            'written to stdout if this option is not present.')
+                      help='Output filename for the private key. The key is '
+                           'written to stdout if this option is not present.')
 
     parser.add_option('--form',
-            help='key format of the private and public keys - default PEM',
-            choices=('PEM', 'DER'), default='PEM')
+                      help='key format of the private and public keys - default PEM',
+                      choices=('PEM', 'DER'), default='PEM')
 
     (cli, cli_args) = parser.parse_args(sys.argv[1:])
 
     if len(cli_args) != 1:
         parser.print_help()
         raise SystemExit(1)
-    
+
     try:
         keysize = int(cli_args[0])
     except ValueError:
@@ -67,7 +68,6 @@ def keygen():
     print('Generating %i-bit key' % keysize, file=sys.stderr)
     (pub_key, priv_key) = rsa.newkeys(keysize)
 
-
     # Save public key
     if cli.pubout:
         print('Writing public key to %s' % cli.pubout, file=sys.stderr)
@@ -77,7 +77,7 @@ def keygen():
 
     # Save private key
     data = priv_key.save_pkcs1(format=cli.form)
-    
+
     if cli.out:
         print('Writing private key to %s' % cli.out, file=sys.stderr)
         with open(cli.out, 'wb') as outfile:
@@ -88,20 +88,20 @@ def keygen():
 
 
 class CryptoOperation(object):
-    '''CLI callable that operates with input, output, and a key.'''
+    """CLI callable that operates with input, output, and a key."""
 
     __metaclass__ = abc.ABCMeta
 
-    keyname = 'public' # or 'private'
+    keyname = 'public'  # or 'private'
     usage = 'usage: %%prog [options] %(keyname)s_key'
     description = None
     operation = 'decrypt'
     operation_past = 'decrypted'
     operation_progressive = 'decrypting'
     input_help = 'Name of the file to %(operation)s. Reads from stdin if ' \
-            'not specified.'
+                 'not specified.'
     output_help = 'Name of the file to write the %(operation_past)s file ' \
-            'to. Written to stdout if this option is not present.'
+                  'to. Written to stdout if this option is not present.'
     expected_cli_args = 1
     has_output = True
 
@@ -114,15 +114,15 @@ class CryptoOperation(object):
 
     @abc.abstractmethod
     def perform_operation(self, indata, key, cli_args=None):
-        '''Performs the program's operation.
+        """Performs the program's operation.
 
         Implement in a subclass.
 
         :returns: the data to write to the output.
-        '''
+        """
 
     def __call__(self):
-        '''Runs the program.'''
+        """Runs the program."""
 
         (cli, cli_args) = self.parse_cli()
 
@@ -137,21 +137,21 @@ class CryptoOperation(object):
             self.write_outfile(outdata, cli.output)
 
     def parse_cli(self):
-        '''Parse the CLI options
-        
+        """Parse the CLI options
+
         :returns: (cli_opts, cli_args)
-        '''
+        """
 
         parser = OptionParser(usage=self.usage, description=self.description)
-        
+
         parser.add_option('-i', '--input', type='string', help=self.input_help)
 
         if self.has_output:
             parser.add_option('-o', '--output', type='string', help=self.output_help)
 
         parser.add_option('--keyform',
-                help='Key format of the %s key - default PEM' % self.keyname,
-                choices=('PEM', 'DER'), default='PEM')
+                          help='Key format of the %s key - default PEM' % self.keyname,
+                          choices=('PEM', 'DER'), default='PEM')
 
         (cli, cli_args) = parser.parse_args(sys.argv[1:])
 
@@ -159,19 +159,19 @@ class CryptoOperation(object):
             parser.print_help()
             raise SystemExit(1)
 
-        return (cli, cli_args)
+        return cli, cli_args
 
     def read_key(self, filename, keyform):
-        '''Reads a public or private key.'''
+        """Reads a public or private key."""
 
         print('Reading %s key from %s' % (self.keyname, filename), file=sys.stderr)
         with open(filename, 'rb') as keyfile:
             keydata = keyfile.read()
 
         return self.key_class.load_pkcs1(keydata, keyform)
-    
+
     def read_infile(self, inname):
-        '''Read the input file'''
+        """Read the input file"""
 
         if inname:
             print('Reading input from %s' % inname, file=sys.stderr)
@@ -182,7 +182,7 @@ class CryptoOperation(object):
         return sys.stdin.read()
 
     def write_outfile(self, outdata, outname):
-        '''Write the output file'''
+        """Write the output file"""
 
         if outname:
             print('Writing output to %s' % outname, file=sys.stderr)
@@ -192,47 +192,49 @@ class CryptoOperation(object):
             print('Writing output to stdout', file=sys.stderr)
             sys.stdout.write(outdata)
 
+
 class EncryptOperation(CryptoOperation):
-    '''Encrypts a file.'''
+    """Encrypts a file."""
 
     keyname = 'public'
     description = ('Encrypts a file. The file must be shorter than the key '
-            'length in order to be encrypted. For larger files, use the '
-            'pyrsa-encrypt-bigfile command.')
+                   'length in order to be encrypted. For larger files, use the '
+                   'pyrsa-encrypt-bigfile command.')
     operation = 'encrypt'
     operation_past = 'encrypted'
     operation_progressive = 'encrypting'
 
-
     def perform_operation(self, indata, pub_key, cli_args=None):
-        '''Encrypts files.'''
+        """Encrypts files."""
 
         return rsa.encrypt(indata, pub_key)
 
+
 class DecryptOperation(CryptoOperation):
-    '''Decrypts a file.'''
+    """Decrypts a file."""
 
     keyname = 'private'
     description = ('Decrypts a file. The original file must be shorter than '
-            'the key length in order to have been encrypted. For larger '
-            'files, use the pyrsa-decrypt-bigfile command.')
+                   'the key length in order to have been encrypted. For larger '
+                   'files, use the pyrsa-decrypt-bigfile command.')
     operation = 'decrypt'
     operation_past = 'decrypted'
     operation_progressive = 'decrypting'
     key_class = rsa.PrivateKey
 
     def perform_operation(self, indata, priv_key, cli_args=None):
-        '''Decrypts files.'''
+        """Decrypts files."""
 
         return rsa.decrypt(indata, priv_key)
 
+
 class SignOperation(CryptoOperation):
-    '''Signs a file.'''
+    """Signs a file."""
 
     keyname = 'private'
     usage = 'usage: %%prog [options] private_key hash_method'
     description = ('Signs a file, outputs the signature. Choose the hash '
-            'method from %s' % ', '.join(HASH_METHODS))
+                   'method from %s' % ', '.join(HASH_METHODS))
     operation = 'sign'
     operation_past = 'signature'
     operation_progressive = 'Signing'
@@ -240,25 +242,26 @@ class SignOperation(CryptoOperation):
     expected_cli_args = 2
 
     output_help = ('Name of the file to write the signature to. Written '
-            'to stdout if this option is not present.')
+                   'to stdout if this option is not present.')
 
     def perform_operation(self, indata, priv_key, cli_args):
-        '''Decrypts files.'''
+        """Signs files."""
 
         hash_method = cli_args[1]
         if hash_method not in HASH_METHODS:
-            raise SystemExit('Invalid hash method, choose one of %s' % 
-                    ', '.join(HASH_METHODS))
+            raise SystemExit('Invalid hash method, choose one of %s' %
+                             ', '.join(HASH_METHODS))
 
         return rsa.sign(indata, priv_key, hash_method)
 
+
 class VerifyOperation(CryptoOperation):
-    '''Verify a signature.'''
+    """Verify a signature."""
 
     keyname = 'public'
     usage = 'usage: %%prog [options] public_key signature_file'
     description = ('Verifies a signature, exits with status 0 upon success, '
-        'prints an error message and exits with status 1 upon error.')
+                   'prints an error message and exits with status 1 upon error.')
     operation = 'verify'
     operation_past = 'verified'
     operation_progressive = 'Verifying'
@@ -267,10 +270,10 @@ class VerifyOperation(CryptoOperation):
     has_output = False
 
     def perform_operation(self, indata, pub_key, cli_args):
-        '''Decrypts files.'''
+        """Verifies files."""
 
         signature_file = cli_args[1]
-        
+
         with open(signature_file, 'rb') as sigfile:
             signature = sigfile.read()
 
@@ -283,7 +286,7 @@ class VerifyOperation(CryptoOperation):
 
 
 class BigfileOperation(CryptoOperation):
-    '''CryptoOperation that doesn't read the entire file into memory.'''
+    """CryptoOperation that doesn't read the entire file into memory."""
 
     def __init__(self):
         CryptoOperation.__init__(self)
@@ -291,13 +294,13 @@ class BigfileOperation(CryptoOperation):
         self.file_objects = []
 
     def __del__(self):
-        '''Closes any open file handles.'''
+        """Closes any open file handles."""
 
         for fobj in self.file_objects:
             fobj.close()
 
     def __call__(self):
-        '''Runs the program.'''
+        """Runs the program."""
 
         (cli, cli_args) = self.parse_cli()
 
@@ -312,7 +315,7 @@ class BigfileOperation(CryptoOperation):
         self.perform_operation(infile, outfile, key, cli_args)
 
     def get_infile(self, inname):
-        '''Returns the input file object'''
+        """Returns the input file object"""
 
         if inname:
             print('Reading input from %s' % inname, file=sys.stderr)
@@ -325,7 +328,7 @@ class BigfileOperation(CryptoOperation):
         return fobj
 
     def get_outfile(self, outname):
-        '''Returns the output file object'''
+        """Returns the output file object"""
 
         if outname:
             print('Will write output to %s' % outname, file=sys.stderr)
@@ -337,35 +340,37 @@ class BigfileOperation(CryptoOperation):
 
         return fobj
 
+
 class EncryptBigfileOperation(BigfileOperation):
-    '''Encrypts a file to VARBLOCK format.'''
+    """Encrypts a file to VARBLOCK format."""
 
     keyname = 'public'
     description = ('Encrypts a file to an encrypted VARBLOCK file. The file '
-            'can be larger than the key length, but the output file is only '
-            'compatible with Python-RSA.')
+                   'can be larger than the key length, but the output file is only '
+                   'compatible with Python-RSA.')
     operation = 'encrypt'
     operation_past = 'encrypted'
     operation_progressive = 'encrypting'
 
     def perform_operation(self, infile, outfile, pub_key, cli_args=None):
-        '''Encrypts files to VARBLOCK.'''
+        """Encrypts files to VARBLOCK."""
 
         return rsa.bigfile.encrypt_bigfile(infile, outfile, pub_key)
 
+
 class DecryptBigfileOperation(BigfileOperation):
-    '''Decrypts a file in VARBLOCK format.'''
+    """Decrypts a file in VARBLOCK format."""
 
     keyname = 'private'
     description = ('Decrypts an encrypted VARBLOCK file that was encrypted '
-            'with pyrsa-encrypt-bigfile')
+                   'with pyrsa-encrypt-bigfile')
     operation = 'decrypt'
     operation_past = 'decrypted'
     operation_progressive = 'decrypting'
     key_class = rsa.PrivateKey
 
     def perform_operation(self, infile, outfile, priv_key, cli_args=None):
-        '''Decrypts a VARBLOCK file.'''
+        """Decrypts a VARBLOCK file."""
 
         return rsa.bigfile.decrypt_bigfile(infile, outfile, priv_key)
 
@@ -376,4 +381,3 @@ sign = SignOperation()
 verify = VerifyOperation()
 encrypt_bigfile = EncryptBigfileOperation()
 decrypt_bigfile = DecryptBigfileOperation()
-

src/rsa/common.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,19 +14,19 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Common functionality shared by several modules.'''
+"""Common functionality shared by several modules."""
 
 
 def bit_size(num):
-    '''
+    """
     Number of bits needed to represent a integer excluding any prefix
     0 bits.
 
-    As per definition from http://wiki.python.org/moin/BitManipulation and
+    As per definition from https://wiki.python.org/moin/BitManipulation and
     to match the behavior of the Python 3 API.
 
     Usage::
-    
+
         >>> bit_size(1023)
         10
         >>> bit_size(1024)
@@ -40,7 +40,7 @@ def bit_size(num):
         before the number's bit length is determined.
     :returns:
         Returns the number of bits in the integer.
-    '''
+    """
     if num == 0:
         return 0
     if num < 0:
@@ -51,23 +51,23 @@ def bit_size(num):
 
     hex_num = "%x" % num
     return ((len(hex_num) - 1) * 4) + {
-        '0':0, '1':1, '2':2, '3':2,
-        '4':3, '5':3, '6':3, '7':3,
-        '8':4, '9':4, 'a':4, 'b':4,
-        'c':4, 'd':4, 'e':4, 'f':4,
-     }[hex_num[0]]
+        '0': 0, '1': 1, '2': 2, '3': 2,
+        '4': 3, '5': 3, '6': 3, '7': 3,
+        '8': 4, '9': 4, 'a': 4, 'b': 4,
+        'c': 4, 'd': 4, 'e': 4, 'f': 4,
+    }[hex_num[0]]
 
 
 def _bit_size(number):
-    '''
+    """
     Returns the number of bits required to hold a specific long number.
-    '''
+    """
     if number < 0:
         raise ValueError('Only nonnegative numbers possible: %s' % number)
 
     if number == 0:
         return 0
-    
+
     # This works, even with very large numbers. When using math.log(number, 2),
     # you'll get rounding errors and it'll fail.
     bits = 0
@@ -79,9 +79,9 @@ def _bit_size(number):
 
 
 def byte_size(number):
-    '''
+    """
     Returns the number of bytes required to hold a specific long number.
-    
+
     The number of bytes is rounded up.
 
     Usage::
@@ -97,17 +97,17 @@ def byte_size(number):
         An unsigned integer
     :returns:
         The number of bytes required to hold a specific long number.
-    '''
+    """
     quanta, mod = divmod(bit_size(number), 8)
     if mod or number == 0:
         quanta += 1
     return quanta
-    #return int(math.ceil(bit_size(number) / 8.0))
+    # return int(math.ceil(bit_size(number) / 8.0))
 
 
 def extended_gcd(a, b):
-    '''Returns a tuple (r, i, j) such that r = gcd(a, b) = ia + jb
-    '''
+    """Returns a tuple (r, i, j) such that r = gcd(a, b) = ia + jb
+    """
     # r = gcd(a,b) i = multiplicitive inverse of a mod b
     #      or      j = multiplicitive inverse of b mod a
     # Neg return values for i or j are made positive mod b or a respectively
@@ -116,26 +116,28 @@ def extended_gcd(a, b):
     y = 1
     lx = 1
     ly = 0
-    oa = a                             #Remember original a/b to remove 
-    ob = b                             #negative values from return results
+    oa = a  # Remember original a/b to remove
+    ob = b  # negative values from return results
     while b != 0:
         q = a // b
-        (a, b)  = (b, a % b)
-        (x, lx) = ((lx - (q * x)),x)
-        (y, ly) = ((ly - (q * y)),y)
-    if (lx < 0): lx += ob              #If neg wrap modulo orignal b
-    if (ly < 0): ly += oa              #If neg wrap modulo orignal a
-    return (a, lx, ly)                 #Return only positive values
+        (a, b) = (b, a % b)
+        (x, lx) = ((lx - (q * x)), x)
+        (y, ly) = ((ly - (q * y)), y)
+    if lx < 0:
+        lx += ob  # If neg wrap modulo orignal b
+    if ly < 0:
+        ly += oa  # If neg wrap modulo orignal a
+    return a, lx, ly  # Return only positive values
 
 
 def inverse(x, n):
-    '''Returns x^-1 (mod n)
+    """Returns x^-1 (mod n)
 
     >>> inverse(7, 4)
     3
     >>> (inverse(143, 4) * 143) % 4
     1
-    '''
+    """
 
     (divider, inv, _) = extended_gcd(x, n)
 
@@ -146,14 +148,14 @@ def inverse(x, n):
 
 
 def crt(a_values, modulo_values):
-    '''Chinese Remainder Theorem.
+    """Chinese Remainder Theorem.
 
     Calculates x such that x = a[i] (mod m[i]) for each i.
 
     :param a_values: the a-values of the above equation
     :param modulo_values: the m-values of the above equation
     :returns: x such that x = a[i] (mod m[i]) for each i
-    
+
 
     >>> crt([2, 3], [3, 5])
     8
@@ -163,10 +165,10 @@ def crt(a_values, modulo_values):
 
     >>> crt([2, 3, 0], [7, 11, 15])
     135
-    '''
+    """
 
     m = 1
-    x = 0 
+    x = 0
 
     for modulo in modulo_values:
         m *= modulo
@@ -179,7 +181,8 @@ def crt(a_values, modulo_values):
 
     return x
 
+
 if __name__ == '__main__':
     import doctest
-    doctest.testmod()
 
+    doctest.testmod()

src/rsa/core.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,24 +14,24 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Core mathematical operations.
+"""Core mathematical operations.
 
 This is the actual core RSA implementation, which is only defined
 mathematically on integers.
-'''
-
+"""
 
 from rsa._compat import is_integer
 
-def assert_int(var, name):
 
+def assert_int(var, name):
     if is_integer(var):
         return
 
     raise TypeError('%s should be an integer, not %s' % (name, var.__class__))
 
+
 def encrypt_int(message, ekey, n):
-    '''Encrypts a message using encryption key 'ekey', working modulo n'''
+    """Encrypts a message using encryption key 'ekey', working modulo n"""
 
     assert_int(message, 'message')
     assert_int(ekey, 'ekey')
@@ -39,15 +39,15 @@ def encrypt_int(message, ekey, n):
 
     if message < 0:
         raise ValueError('Only non-negative numbers are supported')
-         
+
     if message > n:
         raise OverflowError("The message %i is too long for n=%i" % (message, n))
 
     return pow(message, ekey, n)
 
+
 def decrypt_int(cyphertext, dkey, n):
-    '''Decrypts a cypher text using the decryption key 'dkey', working
-    modulo n'''
+    """Decrypts a cypher text using the decryption key 'dkey', working modulo n"""
 
     assert_int(cyphertext, 'cyphertext')
     assert_int(dkey, 'dkey')
@@ -55,4 +55,3 @@ def decrypt_int(cyphertext, dkey, n):
 
     message = pow(cyphertext, dkey, n)
     return message
-

src/rsa/key.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''RSA key generation code.
+"""RSA key generation code.
 
 Create new keys with the newkeys() function. It will give you a PublicKey and a
 PrivateKey object.
@@ -23,70 +23,118 @@ Loading and saving keys requires the pyasn1 module. This module is imported as
 late as possible, such that other functionality will remain working in absence
 of pyasn1.
 
-'''
+.. note::
+
+    Storing public and private keys via the `pickle` module is possible.
+    However, it is insecure to load a key from an untrusted source.
+    The pickle module is not secure against erroneous or maliciously
+    constructed data. Never unpickle data received from an untrusted
+    or unauthenticated source.
+
+"""
 
 import logging
-from rsa._compat import b, bytes_type
+from rsa._compat import b
 
 import rsa.prime
 import rsa.pem
 import rsa.common
+import rsa.randnum
+import rsa.core
 
 log = logging.getLogger(__name__)
-
+DEFAULT_EXPONENT = 65537
 
 
 class AbstractKey(object):
-    '''Abstract superclass for private and public keys.'''
+    """Abstract superclass for private and public keys."""
+
+    __slots__ = ('n', 'e')
+
+    def __init__(self, n, e):
+        self.n = n
+        self.e = e
 
     @classmethod
     def load_pkcs1(cls, keyfile, format='PEM'):
-        r'''Loads a key in PKCS#1 DER or PEM format.
+        """Loads a key in PKCS#1 DER or PEM format.
 
         :param keyfile: contents of a DER- or PEM-encoded file that contains
             the public key.
         :param format: the format of the file to load; 'PEM' or 'DER'
 
         :return: a PublicKey object
-
-        '''
+        """
 
         methods = {
             'PEM': cls._load_pkcs1_pem,
             'DER': cls._load_pkcs1_der,
         }
 
-        if format not in methods:
-            formats = ', '.join(sorted(methods.keys()))
-            raise ValueError('Unsupported format: %r, try one of %s' % (format,
-                formats))
-
-        method = methods[format]
+        method = cls._assert_format_exists(format, methods)
         return method(keyfile)
 
+    @staticmethod
+    def _assert_format_exists(file_format, methods):
+        """Checks whether the given file format exists in 'methods'.
+        """
+
+        try:
+            return methods[file_format]
+        except KeyError:
+            formats = ', '.join(sorted(methods.keys()))
+            raise ValueError('Unsupported format: %r, try one of %s' % (file_format,
+                                                                        formats))
+
     def save_pkcs1(self, format='PEM'):
-        '''Saves the public key in PKCS#1 DER or PEM format.
+        """Saves the public key in PKCS#1 DER or PEM format.
 
         :param format: the format to save; 'PEM' or 'DER'
         :returns: the DER- or PEM-encoded public key.
-
-        '''
+        """
 
         methods = {
             'PEM': self._save_pkcs1_pem,
             'DER': self._save_pkcs1_der,
         }
 
-        if format not in methods:
-            formats = ', '.join(sorted(methods.keys()))
-            raise ValueError('Unsupported format: %r, try one of %s' % (format,
-                formats))
-
-        method = methods[format]
+        method = self._assert_format_exists(format, methods)
         return method()
 
+    def blind(self, message, r):
+        """Performs blinding on the message using random number 'r'.
+
+        :param message: the message, as integer, to blind.
+        :type message: int
+        :param r: the random number to blind with.
+        :type r: int
+        :return: the blinded message.
+        :rtype: int
+
+        The blinding is such that message = unblind(decrypt(blind(encrypt(message))).
+
+        See https://en.wikipedia.org/wiki/Blinding_%28cryptography%29
+        """
+
+        return (message * pow(r, self.e, self.n)) % self.n
+
+    def unblind(self, blinded, r):
+        """Performs blinding on the message using random number 'r'.
+
+        :param blinded: the blinded message, as integer, to unblind.
+        :param r: the random number to unblind with.
+        :return: the original message.
+
+        The blinding is such that message = unblind(decrypt(blind(encrypt(message))).
+
+        See https://en.wikipedia.org/wiki/Blinding_%28cryptography%29
+        """
+
+        return (rsa.common.inverse(r, self.n) * blinded) % self.n
+
+
 class PublicKey(AbstractKey):
-    '''Represents a public RSA key.
+    """Represents a public RSA key.
 
     This key is also known as the 'encryption key'. It contains the 'n' and 'e'
     values.
@@ -107,20 +155,24 @@ class PublicKey(AbstractKey):
     >>> key['e']
     3
 
-    '''
+    """
 
     __slots__ = ('n', 'e')
 
-    def __init__(self, n, e):
-        self.n = n
-        self.e = e
-
     def __getitem__(self, key):
         return getattr(self, key)
 
     def __repr__(self):
         return 'PublicKey(%i, %i)' % (self.n, self.e)
 
+    def __getstate__(self):
+        """Returns the key as tuple for pickling."""
+        return self.n, self.e
+
+    def __setstate__(self, state):
+        """Sets the key from tuple."""
+        self.n, self.e = state
+
     def __eq__(self, other):
         if other is None:
             return False
@@ -135,36 +187,36 @@ class PublicKey(AbstractKey):
 
     @classmethod
     def _load_pkcs1_der(cls, keyfile):
-        r'''Loads a key in PKCS#1 DER format.
+        """Loads a key in PKCS#1 DER format.
 
-        @param keyfile: contents of a DER-encoded file that contains the public
+        :param keyfile: contents of a DER-encoded file that contains the public
             key.
-        @return: a PublicKey object
+        :return: a PublicKey object
 
         First let's construct a DER encoded key:
 
         >>> import base64
         >>> b64der = 'MAwCBQCNGmYtAgMBAAE='
-        >>> der = base64.decodestring(b64der)
+        >>> der = base64.standard_b64decode(b64der)
 
         This loads the file:
 
         >>> PublicKey._load_pkcs1_der(der)
         PublicKey(2367317549, 65537)
 
-        '''
+        """
 
         from pyasn1.codec.der import decoder
         from rsa.asn1 import AsnPubKey
-        
+
         (priv, _) = decoder.decode(keyfile, asn1Spec=AsnPubKey())
         return cls(n=int(priv['modulus']), e=int(priv['publicExponent']))
 
     def _save_pkcs1_der(self):
-        '''Saves the public key in PKCS#1 DER format.
+        """Saves the public key in PKCS#1 DER format.
 
         @returns: the DER-encoded public key.
-        '''
+        """
 
         from pyasn1.codec.der import encoder
         from rsa.asn1 import AsnPubKey
@@ -178,71 +230,70 @@ class PublicKey(AbstractKey):
 
     @classmethod
     def _load_pkcs1_pem(cls, keyfile):
-        '''Loads a PKCS#1 PEM-encoded public key file.
+        """Loads a PKCS#1 PEM-encoded public key file.
 
         The contents of the file before the "-----BEGIN RSA PUBLIC KEY-----" and
         after the "-----END RSA PUBLIC KEY-----" lines is ignored.
 
-        @param keyfile: contents of a PEM-encoded file that contains the public
+        :param keyfile: contents of a PEM-encoded file that contains the public
             key.
-        @return: a PublicKey object
-        '''
+        :return: a PublicKey object
+        """
 
         der = rsa.pem.load_pem(keyfile, 'RSA PUBLIC KEY')
         return cls._load_pkcs1_der(der)
 
     def _save_pkcs1_pem(self):
-        '''Saves a PKCS#1 PEM-encoded public key file.
+        """Saves a PKCS#1 PEM-encoded public key file.
 
-        @return: contents of a PEM-encoded file that contains the public key.
-        '''
+        :return: contents of a PEM-encoded file that contains the public key.
+        """
 
         der = self._save_pkcs1_der()
         return rsa.pem.save_pem(der, 'RSA PUBLIC KEY')
 
     @classmethod
     def load_pkcs1_openssl_pem(cls, keyfile):
-        '''Loads a PKCS#1.5 PEM-encoded public key file from OpenSSL.
-        
+        """Loads a PKCS#1.5 PEM-encoded public key file from OpenSSL.
+
         These files can be recognised in that they start with BEGIN PUBLIC KEY
         rather than BEGIN RSA PUBLIC KEY.
-        
+
         The contents of the file before the "-----BEGIN PUBLIC KEY-----" and
         after the "-----END PUBLIC KEY-----" lines is ignored.
 
-        @param keyfile: contents of a PEM-encoded file that contains the public
+        :param keyfile: contents of a PEM-encoded file that contains the public
             key, from OpenSSL.
-        @return: a PublicKey object
-        '''
+        :return: a PublicKey object
+        """
 
         der = rsa.pem.load_pem(keyfile, 'PUBLIC KEY')
         return cls.load_pkcs1_openssl_der(der)
 
     @classmethod
     def load_pkcs1_openssl_der(cls, keyfile):
-        '''Loads a PKCS#1 DER-encoded public key file from OpenSSL.
+        """Loads a PKCS#1 DER-encoded public key file from OpenSSL.
 
-        @param keyfile: contents of a DER-encoded file that contains the public
+        :param keyfile: contents of a DER-encoded file that contains the public
             key, from OpenSSL.
-        @return: a PublicKey object
-        '''
-    
+        :return: a PublicKey object
+
+        """
+
         from rsa.asn1 import OpenSSLPubKey
         from pyasn1.codec.der import decoder
         from pyasn1.type import univ
-        
+
         (keyinfo, _) = decoder.decode(keyfile, asn1Spec=OpenSSLPubKey())
-        
+
         if keyinfo['header']['oid'] != univ.ObjectIdentifier('1.2.840.113549.1.1.1'):
             raise TypeError("This is not a DER-encoded OpenSSL-compatible public key")
-                
+
         return cls._load_pkcs1_der(keyinfo['key'][1:])
-        
-        
 
 
 class PrivateKey(AbstractKey):
-    '''Represents a private RSA key.
+    """Represents a private RSA key.
 
     This key is also known as the 'decryption key'. It contains the 'n', 'e',
     'd', 'p', 'q' and other values.
@@ -253,13 +304,13 @@ class PrivateKey(AbstractKey):
     >>> PrivateKey(3247, 65537, 833, 191, 17)
     PrivateKey(3247, 65537, 833, 191, 17)
 
-    exp1, exp2 and coef don't have to be given, they will be calculated:
+    exp1, exp2 and coef can be given, but if None or omitted they will be calculated:
 
-    >>> pk = PrivateKey(3727264081, 65537, 3349121513, 65063, 57287)
+    >>> pk = PrivateKey(3727264081, 65537, 3349121513, 65063, 57287, exp2=4)
     >>> pk.exp1
     55063
-    >>> pk.exp2
-    10095
+    >>> pk.exp2  # this is of course not a correct value, but it is the one we passed.
+    4
     >>> pk.coef
     50797
 
@@ -273,13 +324,12 @@ class PrivateKey(AbstractKey):
     >>> pk.coef
     8
 
-    '''
+    """
 
     __slots__ = ('n', 'e', 'd', 'p', 'q', 'exp1', 'exp2', 'coef')
 
     def __init__(self, n, e, d, p, q, exp1=None, exp2=None, coef=None):
-        self.n = n
-        self.e = e
+        AbstractKey.__init__(self, n, e)
         self.d = d
         self.p = p
         self.q = q
@@ -290,7 +340,7 @@ class PrivateKey(AbstractKey):
         else:
             self.exp1 = exp1
 
-        if exp1 is None:
+        if exp2 is None:
             self.exp2 = int(d % (q - 1))
         else:
             self.exp2 = exp2
@@ -306,6 +356,14 @@ class PrivateKey(AbstractKey):
     def __repr__(self):
         return 'PrivateKey(%(n)i, %(e)i, %(d)i, %(p)i, %(q)i)' % self
 
+    def __getstate__(self):
+        """Returns the key as tuple for pickling."""
+        return self.n, self.e, self.d, self.p, self.q, self.exp1, self.exp2, self.coef
+
+    def __setstate__(self, state):
+        """Sets the key from tuple."""
+        self.n, self.e, self.d, self.p, self.q, self.exp1, self.exp2, self.coef = state
+
     def __eq__(self, other):
         if other is None:
             return False
@@ -314,37 +372,68 @@ class PrivateKey(AbstractKey):
             return False
 
         return (self.n == other.n and
-            self.e == other.e and
-            self.d == other.d and
-            self.p == other.p and
-            self.q == other.q and
-            self.exp1 == other.exp1 and
-            self.exp2 == other.exp2 and
-            self.coef == other.coef)
+                self.e == other.e and
+                self.d == other.d and
+                self.p == other.p and
+                self.q == other.q and
+                self.exp1 == other.exp1 and
+                self.exp2 == other.exp2 and
+                self.coef == other.coef)
 
     def __ne__(self, other):
         return not (self == other)
 
+    def blinded_decrypt(self, encrypted):
+        """Decrypts the message using blinding to prevent side-channel attacks.
+
+        :param encrypted: the encrypted message
+        :type encrypted: int
+
+        :returns: the decrypted message
+        :rtype: int
+        """
+
+        blind_r = rsa.randnum.randint(self.n - 1)
+        blinded = self.blind(encrypted, blind_r)  # blind before decrypting
+        decrypted = rsa.core.decrypt_int(blinded, self.d, self.n)
+
+        return self.unblind(decrypted, blind_r)
+
+    def blinded_encrypt(self, message):
+        """Encrypts the message using blinding to prevent side-channel attacks.
+
+        :param message: the message to encrypt
+        :type message: int
+
+        :returns: the encrypted message
+        :rtype: int
+        """
+
+        blind_r = rsa.randnum.randint(self.n - 1)
+        blinded = self.blind(message, blind_r)  # blind before encrypting
+        encrypted = rsa.core.encrypt_int(blinded, self.d, self.n)
+        return self.unblind(encrypted, blind_r)
+
     @classmethod
     def _load_pkcs1_der(cls, keyfile):
-        r'''Loads a key in PKCS#1 DER format.
+        """Loads a key in PKCS#1 DER format.
 
-        @param keyfile: contents of a DER-encoded file that contains the private
+        :param keyfile: contents of a DER-encoded file that contains the private
             key.
-        @return: a PrivateKey object
+        :return: a PrivateKey object
 
         First let's construct a DER encoded key:
 
         >>> import base64
         >>> b64der = 'MC4CAQACBQDeKYlRAgMBAAECBQDHn4npAgMA/icCAwDfxwIDANcXAgInbwIDAMZt'
-        >>> der = base64.decodestring(b64der)
+        >>> der = base64.standard_b64decode(b64der)
 
         This loads the file:
 
         >>> PrivateKey._load_pkcs1_der(der)
         PrivateKey(3727264081, 65537, 3349121513, 65063, 57287)
 
-        '''
+        """
 
         from pyasn1.codec.der import decoder
         (priv, _) = decoder.decode(keyfile)
@@ -352,16 +441,16 @@ class PrivateKey(AbstractKey):
         # ASN.1 contents of DER encoded private key:
         #
         # RSAPrivateKey ::= SEQUENCE {
-        #     version           Version, 
+        #     version           Version,
         #     modulus           INTEGER,  -- n
         #     publicExponent    INTEGER,  -- e
         #     privateExponent   INTEGER,  -- d
         #     prime1            INTEGER,  -- p
         #     prime2            INTEGER,  -- q
         #     exponent1         INTEGER,  -- d mod (p-1)
-        #     exponent2         INTEGER,  -- d mod (q-1) 
+        #     exponent2         INTEGER,  -- d mod (q-1)
         #     coefficient       INTEGER,  -- (inverse of q) mod p
-        #     otherPrimeInfos   OtherPrimeInfos OPTIONAL 
+        #     otherPrimeInfos   OtherPrimeInfos OPTIONAL
         # }
 
         if priv[0] != 0:
@@ -371,25 +460,25 @@ class PrivateKey(AbstractKey):
         return cls(*as_ints)
 
     def _save_pkcs1_der(self):
-        '''Saves the private key in PKCS#1 DER format.
+        """Saves the private key in PKCS#1 DER format.
 
         @returns: the DER-encoded private key.
-        '''
+        """
 
         from pyasn1.type import univ, namedtype
         from pyasn1.codec.der import encoder
 
         class AsnPrivKey(univ.Sequence):
             componentType = namedtype.NamedTypes(
-                namedtype.NamedType('version', univ.Integer()),
-                namedtype.NamedType('modulus', univ.Integer()),
-                namedtype.NamedType('publicExponent', univ.Integer()),
-                namedtype.NamedType('privateExponent', univ.Integer()),
-                namedtype.NamedType('prime1', univ.Integer()),
-                namedtype.NamedType('prime2', univ.Integer()),
-                namedtype.NamedType('exponent1', univ.Integer()),
-                namedtype.NamedType('exponent2', univ.Integer()),
-                namedtype.NamedType('coefficient', univ.Integer()),
+                    namedtype.NamedType('version', univ.Integer()),
+                    namedtype.NamedType('modulus', univ.Integer()),
+                    namedtype.NamedType('publicExponent', univ.Integer()),
+                    namedtype.NamedType('privateExponent', univ.Integer()),
+                    namedtype.NamedType('prime1', univ.Integer()),
+                    namedtype.NamedType('prime2', univ.Integer()),
+                    namedtype.NamedType('exponent1', univ.Integer()),
+                    namedtype.NamedType('exponent2', univ.Integer()),
+                    namedtype.NamedType('coefficient', univ.Integer()),
             )
 
         # Create the ASN object
@@ -408,31 +497,32 @@ class PrivateKey(AbstractKey):
 
     @classmethod
     def _load_pkcs1_pem(cls, keyfile):
-        '''Loads a PKCS#1 PEM-encoded private key file.
+        """Loads a PKCS#1 PEM-encoded private key file.
 
         The contents of the file before the "-----BEGIN RSA PRIVATE KEY-----" and
         after the "-----END RSA PRIVATE KEY-----" lines is ignored.
 
-        @param keyfile: contents of a PEM-encoded file that contains the private
+        :param keyfile: contents of a PEM-encoded file that contains the private
             key.
-        @return: a PrivateKey object
-        '''
+        :return: a PrivateKey object
+        """
 
         der = rsa.pem.load_pem(keyfile, b('RSA PRIVATE KEY'))
         return cls._load_pkcs1_der(der)
 
     def _save_pkcs1_pem(self):
-        '''Saves a PKCS#1 PEM-encoded private key file.
+        """Saves a PKCS#1 PEM-encoded private key file.
 
-        @return: contents of a PEM-encoded file that contains the private key.
-        '''
+        :return: contents of a PEM-encoded file that contains the private key.
+        """
 
         der = self._save_pkcs1_der()
         return rsa.pem.save_pem(der, b('RSA PRIVATE KEY'))
 
+
 def find_p_q(nbits, getprime_func=rsa.prime.getprime, accurate=True):
-    ''''Returns a tuple of two different primes of nbits bits each.
-    
+    """Returns a tuple of two different primes of nbits bits each.
+
     The resulting p * q has exacty 2 * nbits bits, and the returned p and q
     will not be equal.
 
@@ -458,9 +548,9 @@ def find_p_q(nbits, getprime_func=rsa.prime.getprime, accurate=True):
     True
     >>> common.bit_size(p * q) > 240
     True
-    
-    '''
-    
+
+    """
+
     total_bits = nbits * 2
 
     # Make sure that p and q aren't too close or the factoring programs can
@@ -468,7 +558,7 @@ def find_p_q(nbits, getprime_func=rsa.prime.getprime, accurate=True):
     shift = nbits // 16
     pbits = nbits + shift
     qbits = nbits - shift
-    
+
     # Choose the two initial primes
     log.debug('find_p_q(%i): Finding p', nbits)
     p = getprime_func(pbits)
@@ -476,11 +566,11 @@ def find_p_q(nbits, getprime_func=rsa.prime.getprime, accurate=True):
     q = getprime_func(qbits)
 
     def is_acceptable(p, q):
-        '''Returns True iff p and q are acceptable:
-            
+        """Returns True iff p and q are acceptable:
+
             - p and q differ
             - (p * q) has the right nr of bits (when accurate=True)
-        '''
+        """
 
         if p == q:
             return False
@@ -505,49 +595,80 @@ def find_p_q(nbits, getprime_func=rsa.prime.getprime, accurate=True):
 
     # We want p > q as described on
     # http://www.di-mgt.com.au/rsa_alg.html#crt
-    return (max(p, q), min(p, q))
+    return max(p, q), min(p, q)
 
-def calculate_keys(p, q, nbits):
-    '''Calculates an encryption and a decryption key given p and q, and
-    returns them as a tuple (e, d)
 
-    '''
+def calculate_keys_custom_exponent(p, q, exponent):
+    """Calculates an encryption and a decryption key given p, q and an exponent,
+    and returns them as a tuple (e, d)
+
+    :param p: the first large prime
+    :param q: the second large prime
+    :param exponent: the exponent for the key; only change this if you know
+        what you're doing, as the exponent influences how difficult your
+        private key can be cracked. A very common choice for e is 65537.
+    :type exponent: int
+
+    """
 
     phi_n = (p - 1) * (q - 1)
 
-    # A very common choice for e is 65537
-    e = 65537
-
     try:
-        d = rsa.common.inverse(e, phi_n)
+        d = rsa.common.inverse(exponent, phi_n)
     except ValueError:
         raise ValueError("e (%d) and phi_n (%d) are not relatively prime" %
-                (e, phi_n))
+                         (exponent, phi_n))
 
-    if (e * d) % phi_n != 1:
+    if (exponent * d) % phi_n != 1:
         raise ValueError("e (%d) and d (%d) are not mult. inv. modulo "
-                "phi_n (%d)" % (e, d, phi_n))
+                         "phi_n (%d)" % (exponent, d, phi_n))
 
-    return (e, d)
+    return exponent, d
 
-def gen_keys(nbits, getprime_func, accurate=True):
-    '''Generate RSA keys of nbits bits. Returns (p, q, e, d).
+
+def calculate_keys(p, q):
+    """Calculates an encryption and a decryption key given p and q, and
+    returns them as a tuple (e, d)
+
+    :param p: the first large prime
+    :param q: the second large prime
+
+    :return: tuple (e, d) with the encryption and decryption exponents.
+    """
+
+    return calculate_keys_custom_exponent(p, q, DEFAULT_EXPONENT)
+
+
+def gen_keys(nbits, getprime_func, accurate=True, exponent=DEFAULT_EXPONENT):
+    """Generate RSA keys of nbits bits. Returns (p, q, e, d).
 
     Note: this can take a long time, depending on the key size.
-    
+
     :param nbits: the total number of bits in ``p`` and ``q``. Both ``p`` and
         ``q`` will use ``nbits/2`` bits.
     :param getprime_func: either :py:func:`rsa.prime.getprime` or a function
         with similar signature.
-    '''
+    :param exponent: the exponent for the key; only change this if you know
+        what you're doing, as the exponent influences how difficult your
+        private key can be cracked. A very common choice for e is 65537.
+    :type exponent: int
+    """
 
-    (p, q) = find_p_q(nbits // 2, getprime_func, accurate)
-    (e, d) = calculate_keys(p, q, nbits // 2)
+    # Regenerate p and q values, until calculate_keys doesn't raise a
+    # ValueError.
+    while True:
+        (p, q) = find_p_q(nbits // 2, getprime_func, accurate)
+        try:
+            (e, d) = calculate_keys_custom_exponent(p, q, exponent=exponent)
+            break
+        except ValueError:
+            pass
 
-    return (p, q, e, d)
+    return p, q, e, d
 
-def newkeys(nbits, accurate=True, poolsize=1):
-    '''Generates public and private keys, and returns them as (pub, priv).
+
+def newkeys(nbits, accurate=True, poolsize=1, exponent=DEFAULT_EXPONENT):
+    """Generates public and private keys, and returns them as (pub, priv).
 
     The public key is also known as the 'encryption key', and is a
     :py:class:`rsa.PublicKey` object. The private key is also known as the
@@ -560,13 +681,17 @@ def newkeys(nbits, accurate=True, poolsize=1):
     :param poolsize: the number of processes to use to generate the prime
         numbers. If set to a number > 1, a parallel algorithm will be used.
         This requires Python 2.6 or newer.
+    :param exponent: the exponent for the key; only change this if you know
+        what you're doing, as the exponent influences how difficult your
+        private key can be cracked. A very common choice for e is 65537.
+    :type exponent: int
 
     :returns: a tuple (:py:class:`rsa.PublicKey`, :py:class:`rsa.PrivateKey`)
 
     The ``poolsize`` parameter was added in *Python-RSA 3.1* and requires
     Python 2.6 or newer.
-    
-    '''
+
+    """
 
     if nbits < 16:
         raise ValueError('Key too small')
@@ -580,11 +705,12 @@ def newkeys(nbits, accurate=True, poolsize=1):
         import functools
 
         getprime_func = functools.partial(parallel.getprime, poolsize=poolsize)
-    else: getprime_func = rsa.prime.getprime
+    else:
+        getprime_func = rsa.prime.getprime
 
     # Generate the key components
-    (p, q, e, d) = gen_keys(nbits, getprime_func)
-    
+    (p, q, e, d) = gen_keys(nbits, getprime_func, accurate=accurate, exponent=exponent)
+
     # Create the key objects
     n = p * q
 
@@ -593,11 +719,12 @@ def newkeys(nbits, accurate=True, poolsize=1):
         PrivateKey(n, e, d, p, q)
     )
 
+
 __all__ = ['PublicKey', 'PrivateKey', 'newkeys']
 
 if __name__ == '__main__':
     import doctest
-    
+
     try:
         for count in range(100):
             (failures, tests) = doctest.testmod()

src/rsa/parallel.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Functions for parallel computation on multiple cores.
+"""Functions for parallel computation on multiple cores.
 
 Introduced in Python-RSA 3.1.
 
@@ -22,7 +22,7 @@ Introduced in Python-RSA 3.1.
 
     Requires Python 2.6 or newer.
 
-'''
+"""
 
 from __future__ import print_function
 
@@ -31,20 +31,19 @@ import multiprocessing as mp
 import rsa.prime
 import rsa.randnum
 
+
 def _find_prime(nbits, pipe):
     while True:
-        integer = rsa.randnum.read_random_int(nbits)
-
-        # Make sure it's odd
-        integer |= 1
+        integer = rsa.randnum.read_random_odd_int(nbits)
 
         # Test for primeness
         if rsa.prime.is_prime(integer):
             pipe.send(integer)
             return
 
+
 def getprime(nbits, poolsize):
-    '''Returns a prime number that can be stored in 'nbits' bits.
+    """Returns a prime number that can be stored in 'nbits' bits.
 
     Works in multiple threads at the same time.
 
@@ -55,40 +54,47 @@ def getprime(nbits, poolsize):
     True
     >>> rsa.prime.is_prime(p+1)
     False
-    
+
     >>> from rsa import common
     >>> common.bit_size(p) == 128
     True
-    
-    '''
+
+    """
 
     (pipe_recv, pipe_send) = mp.Pipe(duplex=False)
 
     # Create processes
-    procs = [mp.Process(target=_find_prime, args=(nbits, pipe_send))
-             for _ in range(poolsize)]
-    [p.start() for p in procs]
+    try:
+        procs = [mp.Process(target=_find_prime, args=(nbits, pipe_send))
+                 for _ in range(poolsize)]
+        # Start processes
+        for p in procs:
+            p.start()
 
-    result = pipe_recv.recv()
+        result = pipe_recv.recv()
+    finally:
+        pipe_recv.close()
+        pipe_send.close()
 
-    [p.terminate() for p in procs]
+    # Terminate processes
+    for p in procs:
+        p.terminate()
 
     return result
 
+
 __all__ = ['getprime']
 
-    
 if __name__ == '__main__':
     print('Running doctests 1000x or until failure')
     import doctest
-    
+
     for count in range(100):
         (failures, tests) = doctest.testmod()
         if failures:
             break
-        
+
         if count and count % 10 == 0:
             print('%i times' % count)
-    
-    print('Doctests done')
 
+    print('Doctests done')

src/rsa/pem.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,15 +14,16 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Functions that load and write PEM-encoded files.'''
+"""Functions that load and write PEM-encoded files."""
 
 import base64
 from rsa._compat import b, is_bytes
 
+
 def _markers(pem_marker):
-    '''
+    """
     Returns the start and end PEM markers
-    '''
+    """
 
     if is_bytes(pem_marker):
         pem_marker = pem_marker.decode('utf-8')
@@ -30,20 +31,25 @@ def _markers(pem_marker):
     return (b('-----BEGIN %s-----' % pem_marker),
             b('-----END %s-----' % pem_marker))
 
-def load_pem(contents, pem_marker):
-    '''Loads a PEM file.
 
-    @param contents: the contents of the file to interpret
-    @param pem_marker: the marker of the PEM content, such as 'RSA PRIVATE KEY'
+def load_pem(contents, pem_marker):
+    """Loads a PEM file.
+
+    :param contents: the contents of the file to interpret
+    :param pem_marker: the marker of the PEM content, such as 'RSA PRIVATE KEY'
         when your file has '-----BEGIN RSA PRIVATE KEY-----' and
         '-----END RSA PRIVATE KEY-----' markers.
 
-    @return the base64-decoded content between the start and end markers.
+    :return: the base64-decoded content between the start and end markers.
 
     @raise ValueError: when the content is invalid, for example when the start
         marker cannot be found.
 
-    '''
+    """
+
+    # We want bytes, not text. If it's text, it can be converted to ASCII bytes.
+    if not is_bytes(contents):
+        contents = contents.encode('ascii')
 
     (pem_start, pem_end) = _markers(pem_marker)
 
@@ -89,26 +95,26 @@ def load_pem(contents, pem_marker):
 
     # Base64-decode the contents
     pem = b('').join(pem_lines)
-    return base64.decodestring(pem)
+    return base64.standard_b64decode(pem)
 
 
 def save_pem(contents, pem_marker):
-    '''Saves a PEM file.
+    """Saves a PEM file.
 
-    @param contents: the contents to encode in PEM format
-    @param pem_marker: the marker of the PEM content, such as 'RSA PRIVATE KEY'
+    :param contents: the contents to encode in PEM format
+    :param pem_marker: the marker of the PEM content, such as 'RSA PRIVATE KEY'
         when your file has '-----BEGIN RSA PRIVATE KEY-----' and
         '-----END RSA PRIVATE KEY-----' markers.
 
-    @return the base64-encoded content between the start and end markers.
+    :return: the base64-encoded content between the start and end markers.
 
-    '''
+    """
 
     (pem_start, pem_end) = _markers(pem_marker)
 
-    b64 = base64.encodestring(contents).replace(b('\n'), b(''))
+    b64 = base64.standard_b64encode(contents).replace(b('\n'), b(''))
     pem_lines = [pem_start]
-    
+
     for block_start in range(0, len(b64), 64):
         block = b64[block_start:block_start + 64]
         pem_lines.append(block)
@@ -117,4 +123,3 @@ def save_pem(contents, pem_marker):
     pem_lines.append(b(''))
 
     return b('\n').join(pem_lines)
-    

src/rsa/pkcs1.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Functions for PKCS#1 version 1.5 encryption and signing
+"""Functions for PKCS#1 version 1.5 encryption and signing
 
 This module implements certain functionality from PKCS#1 version 1.5. For a
 very clear example, read http://www.di-mgt.com.au/rsa_alg.html#pkcs1schemes
@@ -26,13 +26,13 @@ WARNING: this module leaks information when decryption fails. The exceptions
 that are raised contain the Python traceback information, which can be used to
 deduce where in the process the failure occurred. DO NOT PASS SUCH INFORMATION
 to your users.
-'''
+"""
 
 import hashlib
 import os
 
 from rsa._compat import b
-from rsa import common, transform, core, varblock
+from rsa import common, transform, core
 
 # ASN.1 codes that describe the hash algorithm used.
 HASH_ASN1 = {
@@ -51,133 +51,138 @@ HASH_METHODS = {
     'SHA-512': hashlib.sha512,
 }
 
+
 class CryptoError(Exception):
-    '''Base class for all exceptions in this module.'''
+    """Base class for all exceptions in this module."""
+
 
 class DecryptionError(CryptoError):
-    '''Raised when decryption fails.'''
+    """Raised when decryption fails."""
+
 
 class VerificationError(CryptoError):
-    '''Raised when verification fails.'''
- 
+    """Raised when verification fails."""
+
+
 def _pad_for_encryption(message, target_length):
-    r'''Pads the message for encryption, returning the padded message.
-    
+    r"""Pads the message for encryption, returning the padded message.
+
     :return: 00 02 RANDOM_DATA 00 MESSAGE
-    
-    >>> block = _pad_for_encryption('hello', 16)
+
+    >>> block = _pad_for_encryption(b'hello', 16)
     >>> len(block)
     16
     >>> block[0:2]
-    '\x00\x02'
+    b'\x00\x02'
     >>> block[-6:]
-    '\x00hello'
+    b'\x00hello'
 
-    '''
+    """
 
     max_msglength = target_length - 11
     msglength = len(message)
-    
+
     if msglength > max_msglength:
         raise OverflowError('%i bytes needed for message, but there is only'
-            ' space for %i' % (msglength, max_msglength))
-    
+                            ' space for %i' % (msglength, max_msglength))
+
     # Get random padding
     padding = b('')
     padding_length = target_length - msglength - 3
-    
+
     # We remove 0-bytes, so we'll end up with less padding than we've asked for,
     # so keep adding data until we're at the correct length.
     while len(padding) < padding_length:
         needed_bytes = padding_length - len(padding)
-        
+
         # Always read at least 8 bytes more than we need, and trim off the rest
         # after removing the 0-bytes. This increases the chance of getting
         # enough bytes, especially when needed_bytes is small
         new_padding = os.urandom(needed_bytes + 5)
         new_padding = new_padding.replace(b('\x00'), b(''))
         padding = padding + new_padding[:needed_bytes]
-    
+
     assert len(padding) == padding_length
-    
+
     return b('').join([b('\x00\x02'),
-                    padding,
-                    b('\x00'),
-                    message])
-    
+                       padding,
+                       b('\x00'),
+                       message])
+
 
 def _pad_for_signing(message, target_length):
-    r'''Pads the message for signing, returning the padded message.
-    
+    r"""Pads the message for signing, returning the padded message.
+
     The padding is always a repetition of FF bytes.
-    
+
     :return: 00 01 PADDING 00 MESSAGE
-    
-    >>> block = _pad_for_signing('hello', 16)
+
+    >>> block = _pad_for_signing(b'hello', 16)
     >>> len(block)
     16
     >>> block[0:2]
-    '\x00\x01'
+    b'\x00\x01'
     >>> block[-6:]
-    '\x00hello'
+    b'\x00hello'
     >>> block[2:-6]
-    '\xff\xff\xff\xff\xff\xff\xff\xff'
-    
-    '''
+    b'\xff\xff\xff\xff\xff\xff\xff\xff'
+
+    """
 
     max_msglength = target_length - 11
     msglength = len(message)
-    
+
     if msglength > max_msglength:
         raise OverflowError('%i bytes needed for message, but there is only'
-            ' space for %i' % (msglength, max_msglength))
-    
+                            ' space for %i' % (msglength, max_msglength))
+
     padding_length = target_length - msglength - 3
-    
+
     return b('').join([b('\x00\x01'),
-                    padding_length * b('\xff'),
-                    b('\x00'),
-                    message])
-    
-    
+                       padding_length * b('\xff'),
+                       b('\x00'),
+                       message])
+
+
 def encrypt(message, pub_key):
-    '''Encrypts the given message using PKCS#1 v1.5
-    
+    """Encrypts the given message using PKCS#1 v1.5
+
     :param message: the message to encrypt. Must be a byte string no longer than
         ``k-11`` bytes, where ``k`` is the number of bytes needed to encode
         the ``n`` component of the public key.
     :param pub_key: the :py:class:`rsa.PublicKey` to encrypt with.
     :raise OverflowError: when the message is too large to fit in the padded
         block.
-        
+
     >>> from rsa import key, common
     >>> (pub_key, priv_key) = key.newkeys(256)
-    >>> message = 'hello'
+    >>> message = b'hello'
     >>> crypto = encrypt(message, pub_key)
-    
+
     The crypto text should be just as long as the public key 'n' component:
 
     >>> len(crypto) == common.byte_size(pub_key.n)
     True
-    
-    '''
-    
+
+    """
+
     keylength = common.byte_size(pub_key.n)
     padded = _pad_for_encryption(message, keylength)
-    
+
     payload = transform.bytes2int(padded)
     encrypted = core.encrypt_int(payload, pub_key.e, pub_key.n)
     block = transform.int2bytes(encrypted, keylength)
-    
+
     return block
 
+
 def decrypt(crypto, priv_key):
-    r'''Decrypts the given message using PKCS#1 v1.5
-    
+    r"""Decrypts the given message using PKCS#1 v1.5
+
     The decryption is considered 'failed' when the resulting cleartext doesn't
     start with the bytes 00 02, or when the 00 byte between the padding and
     the message cannot be found.
-    
+
     :param crypto: the crypto text as returned by :py:func:`rsa.encrypt`
     :param priv_key: the :py:class:`rsa.PrivateKey` to decrypt with.
     :raise DecryptionError: when the decryption fails. No details are given as
@@ -190,15 +195,15 @@ def decrypt(crypto, priv_key):
 
     It works with strings:
 
-    >>> crypto = encrypt('hello', pub_key)
+    >>> crypto = encrypt(b'hello', pub_key)
     >>> decrypt(crypto, priv_key)
-    'hello'
-    
+    b'hello'
+
     And with binary data:
 
-    >>> crypto = encrypt('\x00\x00\x00\x00\x01', pub_key)
+    >>> crypto = encrypt(b'\x00\x00\x00\x00\x01', pub_key)
     >>> decrypt(crypto, priv_key)
-    '\x00\x00\x00\x00\x01'
+    b'\x00\x00\x00\x00\x01'
 
     Altering the encrypted information will *likely* cause a
     :py:class:`rsa.pkcs1.DecryptionError`. If you want to be *sure*, use
@@ -213,38 +218,39 @@ def decrypt(crypto, priv_key):
         It's only a tiny bit of information, but every bit makes cracking the
         keys easier.
 
-    >>> crypto = encrypt('hello', pub_key)
-    >>> crypto = crypto[0:5] + 'X' + crypto[6:] # change a byte
+    >>> crypto = encrypt(b'hello', pub_key)
+    >>> crypto = crypto[0:5] + b'X' + crypto[6:] # change a byte
     >>> decrypt(crypto, priv_key)
     Traceback (most recent call last):
     ...
-    DecryptionError: Decryption failed
+    rsa.pkcs1.DecryptionError: Decryption failed
+
+    """
 
-    '''
-    
     blocksize = common.byte_size(priv_key.n)
     encrypted = transform.bytes2int(crypto)
-    decrypted = core.decrypt_int(encrypted, priv_key.d, priv_key.n)
+    decrypted = priv_key.blinded_decrypt(encrypted)
     cleartext = transform.int2bytes(decrypted, blocksize)
 
     # If we can't find the cleartext marker, decryption failed.
     if cleartext[0:2] != b('\x00\x02'):
         raise DecryptionError('Decryption failed')
-    
+
     # Find the 00 separator between the padding and the message
     try:
         sep_idx = cleartext.index(b('\x00'), 2)
     except ValueError:
         raise DecryptionError('Decryption failed')
-    
-    return cleartext[sep_idx+1:]
-    
+
+    return cleartext[sep_idx + 1:]
+
+
 def sign(message, priv_key, hash):
-    '''Signs the message with the private key.
+    """Signs the message with the private key.
 
     Hashes the message, then signs the hash with the given key. This is known
     as a "detached signature", because the message itself isn't altered.
-    
+
     :param message: the message to sign. Can be an 8-bit string or a file-like
         object. If ``message`` has a ``read()`` method, it is assumed to be a
         file-like object.
@@ -255,13 +261,13 @@ def sign(message, priv_key, hash):
     :raise OverflowError: if the private key is too small to contain the
         requested hash.
 
-    '''
+    """
 
     # Get the ASN1 code for this hash method
     if hash not in HASH_ASN1:
         raise ValueError('Invalid hash method: %s' % hash)
     asn1code = HASH_ASN1[hash]
-    
+
     # Calculate the hash
     hash = _hash(message, hash)
 
@@ -269,18 +275,19 @@ def sign(message, priv_key, hash):
     cleartext = asn1code + hash
     keylength = common.byte_size(priv_key.n)
     padded = _pad_for_signing(cleartext, keylength)
-    
+
     payload = transform.bytes2int(padded)
-    encrypted = core.encrypt_int(payload, priv_key.d, priv_key.n)
+    encrypted = priv_key.blinded_encrypt(payload)
     block = transform.int2bytes(encrypted, keylength)
-    
+
     return block
 
+
 def verify(message, signature, pub_key):
-    '''Verifies that the signature matches the message.
-    
+    """Verifies that the signature matches the message.
+
     The hash method is detected automatically from the signature.
-    
+
     :param message: the signed message. Can be an 8-bit string or a file-like
         object. If ``message`` has a ``read()`` method, it is assumed to be a
         file-like object.
@@ -288,13 +295,13 @@ def verify(message, signature, pub_key):
     :param pub_key: the :py:class:`rsa.PublicKey` of the person signing the message.
     :raise VerificationError: when the signature doesn't match the message.
 
-    '''
-    
+    """
+
     keylength = common.byte_size(pub_key.n)
     encrypted = transform.bytes2int(signature)
     decrypted = core.decrypt_int(encrypted, pub_key.e, pub_key.n)
     clearsig = transform.int2bytes(decrypted, keylength)
-    
+
     # Get the hash method
     method_name = _find_method_hash(clearsig)
     message_hash = _hash(message, method_name)
@@ -309,24 +316,28 @@ def verify(message, signature, pub_key):
 
     return True
 
+
 def _hash(message, method_name):
-    '''Returns the message digest.
-    
+    """Returns the message digest.
+
     :param message: the signed message. Can be an 8-bit string or a file-like
         object. If ``message`` has a ``read()`` method, it is assumed to be a
         file-like object.
     :param method_name: the hash method, must be a key of
         :py:const:`HASH_METHODS`.
-    
-    '''
+
+    """
 
     if method_name not in HASH_METHODS:
         raise ValueError('Invalid hash method: %s' % method_name)
-    
+
     method = HASH_METHODS[method_name]
     hasher = method()
 
     if hasattr(message, 'read') and hasattr(message.read, '__call__'):
+        # Late import to prevent DeprecationWarnings.
+        from . import varblock
+
         # read as 1K blocks
         for block in varblock.yield_fixedblocks(message, 1024):
             hasher.update(block)
@@ -338,20 +349,17 @@ def _hash(message, method_name):
 
 
 def _find_method_hash(clearsig):
-    '''Finds the hash method.
-    
-    :param clearsig: full padded ASN1 and hash.
-    
-    :return: the used hash method.
-    
-    :raise VerificationFailed: when the hash method cannot be found
+    """Finds the hash method.
 
-    '''
+    :param clearsig: full padded ASN1 and hash.
+    :return: the used hash method.
+    :raise VerificationFailed: when the hash method cannot be found
+    """
 
     for (hashname, asn1code) in HASH_ASN1.items():
         if asn1code in clearsig:
             return hashname
-    
+
     raise VerificationError('Verification failed')
 
 
@@ -361,13 +369,13 @@ __all__ = ['encrypt', 'decrypt', 'sign', 'verify',
 if __name__ == '__main__':
     print('Running doctests 1000x or until failure')
     import doctest
-    
+
     for count in range(1000):
         (failures, tests) = doctest.testmod()
         if failures:
             break
-        
+
         if count and count % 100 == 0:
             print('%i times' % count)
-    
+
     print('Doctests done')

src/rsa/prime.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,102 +14,115 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Numerical functions related to primes.
+"""Numerical functions related to primes.
 
 Implementation based on the book Algorithm Design by Michael T. Goodrich and
 Roberto Tamassia, 2002.
-'''
-
-__all__ = [ 'getprime', 'are_relatively_prime']
+"""
 
 import rsa.randnum
 
+__all__ = ['getprime', 'are_relatively_prime']
+
+
 def gcd(p, q):
-    '''Returns the greatest common divisor of p and q
+    """Returns the greatest common divisor of p and q
 
     >>> gcd(48, 180)
     12
-    '''
+    """
 
     while q != 0:
-        if p < q: (p,q) = (q,p)
-        (p,q) = (q, p % q)
+        (p, q) = (q, p % q)
     return p
-    
 
-def jacobi(a, b):
-    '''Calculates the value of the Jacobi symbol (a/b) where both a and b are
-    positive integers, and b is odd
 
-    :returns: -1, 0 or 1
-    '''
+def miller_rabin_primality_testing(n, k):
+    """Calculates whether n is composite (which is always correct) or prime
+    (which theoretically is incorrect with error probability 4**-k), by
+    applying Miller-Rabin primality testing.
 
-    assert a > 0
-    assert b > 0
+    For reference and implementation example, see:
+    https://en.wikipedia.org/wiki/Miller%E2%80%93Rabin_primality_test
 
-    if a == 0: return 0
-    result = 1
-    while a > 1:
-        if a & 1:
-            if ((a-1)*(b-1) >> 2) & 1:
-                result = -result
-            a, b = b % a, a
-        else:
-            if (((b * b) - 1) >> 3) & 1:
-                result = -result
-            a >>= 1
-    if a == 0: return 0
-    return result
+    :param n: Integer to be tested for primality.
+    :type n: int
+    :param k: Number of rounds (witnesses) of Miller-Rabin testing.
+    :type k: int
+    :return: False if the number is composite, True if it's probably prime.
+    :rtype: bool
+    """
 
-def jacobi_witness(x, n):
-    '''Returns False if n is an Euler pseudo-prime with base x, and
-    True otherwise.
-    '''
+    # prevent potential infinite loop when d = 0
+    if n < 2:
+        return False
 
-    j = jacobi(x, n) % n
+    # Decompose (n - 1) to write it as (2 ** r) * d
+    # While d is even, divide it by 2 and increase the exponent.
+    d = n - 1
+    r = 0
 
-    f = pow(x, n >> 1, n)
-
-    if j == f: return False
-    return True
-
-def randomized_primality_testing(n, k):
-    '''Calculates whether n is composite (which is always correct) or
-    prime (which is incorrect with error probability 2**-k)
-
-    Returns False if the number is composite, and True if it's
-    probably prime.
-    '''
-
-    # 50% of Jacobi-witnesses can report compositness of non-prime numbers
-
-    # The implemented algorithm using the Jacobi witness function has error
-    # probability q <= 0.5, according to Goodrich et. al
-    #
-    # q = 0.5
-    # t = int(math.ceil(k / log(1 / q, 2)))
-    # So t = k / log(2, 2) = k / 1 = k
-    # this means we can use range(k) rather than range(t)
+    while not (d & 1):
+        r += 1
+        d >>= 1
 
+    # Test k witnesses.
     for _ in range(k):
-        x = rsa.randnum.randint(n-1)
-        if jacobi_witness(x, n): return False
-    
+        # Generate random integer a, where 2 <= a <= (n - 2)
+        a = rsa.randnum.randint(n - 4) + 2
+
+        x = pow(a, d, n)
+        if x == 1 or x == n - 1:
+            continue
+
+        for _ in range(r - 1):
+            x = pow(x, 2, n)
+            if x == 1:
+                # n is composite.
+                return False
+            if x == n - 1:
+                # Exit inner loop and continue with next witness.
+                break
+        else:
+            # If loop doesn't break, n is composite.
+            return False
+
     return True
 
+
 def is_prime(number):
-    '''Returns True if the number is prime, and False otherwise.
+    """Returns True if the number is prime, and False otherwise.
 
+    >>> is_prime(2)
+    True
     >>> is_prime(42)
     False
     >>> is_prime(41)
     True
-    '''
+    >>> [x for x in range(901, 1000) if is_prime(x)]
+    [907, 911, 919, 929, 937, 941, 947, 953, 967, 971, 977, 983, 991, 997]
+    """
+
+    # Check for small numbers.
+    if number < 10:
+        return number in [2, 3, 5, 7]
+
+    # Check for even numbers.
+    if not (number & 1):
+        return False
+
+    # According to NIST FIPS 186-4, Appendix C, Table C.3, minimum number of
+    # rounds of M-R testing, using an error probability of 2 ** (-100), for
+    # different p, q bitsizes are:
+    #   * p, q bitsize: 512; rounds: 7
+    #   * p, q bitsize: 1024; rounds: 4
+    #   * p, q bitsize: 1536; rounds: 3
+    # See: http://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.186-4.pdf
+    return miller_rabin_primality_testing(number, 7)
 
-    return randomized_primality_testing(number, 6)
 
 def getprime(nbits):
-    '''Returns a prime number that can be stored in 'nbits' bits.
+    """Returns a prime number that can be stored in 'nbits' bits.
 
     >>> p = getprime(128)
     >>> is_prime(p-1)
@@ -118,49 +131,48 @@ def getprime(nbits):
     True
     >>> is_prime(p+1)
     False
-    
+
     >>> from rsa import common
     >>> common.bit_size(p) == 128
     True
-    
-    '''
+    """
+
+    assert nbits > 3  # the loop wil hang on too small numbers
 
     while True:
-        integer = rsa.randnum.read_random_int(nbits)
-
-        # Make sure it's odd
-        integer |= 1
+        integer = rsa.randnum.read_random_odd_int(nbits)
 
         # Test for primeness
         if is_prime(integer):
             return integer
 
-        # Retry if not prime
+            # Retry if not prime
 
 
 def are_relatively_prime(a, b):
-    '''Returns True if a and b are relatively prime, and False if they
+    """Returns True if a and b are relatively prime, and False if they
     are not.
 
     >>> are_relatively_prime(2, 3)
-    1
+    True
     >>> are_relatively_prime(2, 4)
-    0
-    '''
+    False
+    """
 
     d = gcd(a, b)
-    return (d == 1)
-    
+    return d == 1
+
+
 if __name__ == '__main__':
     print('Running doctests 1000x or until failure')
     import doctest
-    
+
     for count in range(1000):
         (failures, tests) = doctest.testmod()
         if failures:
             break
-        
+
         if count and count % 100 == 0:
             print('%i times' % count)
-    
+
     print('Doctests done')

src/rsa/randnum.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Functions for generating random numbers.'''
+"""Functions for generating random numbers."""
 
 # Source inspired by code by Yesudeep Mangalapilly <yesudeep@gmail.com>
 
@@ -23,12 +23,13 @@ import os
 from rsa import common, transform
 from rsa._compat import byte
 
+
 def read_random_bits(nbits):
-    '''Reads 'nbits' random bits.
+    """Reads 'nbits' random bits.
 
     If nbits isn't a whole number of bytes, an extra byte will be appended with
     only the lower bits set.
-    '''
+    """
 
     nbytes, rbits = divmod(nbits, 8)
 
@@ -45,8 +46,8 @@ def read_random_bits(nbits):
 
 
 def read_random_int(nbits):
-    '''Reads a random integer of approximately nbits bits.
-    '''
+    """Reads a random integer of approximately nbits bits.
+    """
 
     randomdata = read_random_bits(nbits)
     value = transform.bytes2int(randomdata)
@@ -57,13 +58,27 @@ def read_random_int(nbits):
 
     return value
 
+
+def read_random_odd_int(nbits):
+    """Reads a random odd integer of approximately nbits bits.
+
+    >>> read_random_odd_int(512) & 1
+    1
+    """
+
+    value = read_random_int(nbits)
+
+    # Make sure it's odd
+    return value | 1
+
+
 def randint(maxvalue):
-    '''Returns a random integer x with 1 <= x <= maxvalue
-    
+    """Returns a random integer x with 1 <= x <= maxvalue
+
     May take a very long time in specific situations. If maxvalue needs N bits
     to store, the closer maxvalue is to (2 ** N) - 1, the faster this function
     is.
-    '''
+    """
 
     bit_size = common.bit_size(maxvalue)
 
@@ -81,5 +96,3 @@ def randint(maxvalue):
         tries += 1
 
     return value
-
-

src/rsa/transform.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,10 +14,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Data transformation functions.
+"""Data transformation functions.
 
 From bytes to a number, number to bytes, etc.
-'''
+"""
 
 from __future__ import absolute_import
 
@@ -26,6 +26,7 @@ try:
     # Using psyco (if available) cuts down the execution time on Python 2.5
     # at least by half.
     import psyco
+
     psyco.full()
 except ImportError:
     pass
@@ -37,32 +38,32 @@ from rsa._compat import is_integer, b, byte, get_word_alignment, ZERO_BYTE, EMPT
 
 
 def bytes2int(raw_bytes):
-    r'''Converts a list of bytes or an 8-bit string to an integer.
+    r"""Converts a list of bytes or an 8-bit string to an integer.
 
     When using unicode strings, encode it to some encoding like UTF8 first.
 
     >>> (((128 * 256) + 64) * 256) + 15
     8405007
-    >>> bytes2int('\x80@\x0f')
+    >>> bytes2int(b'\x80@\x0f')
     8405007
 
-    '''
+    """
 
     return int(binascii.hexlify(raw_bytes), 16)
 
 
 def _int2bytes(number, block_size=None):
-    r'''Converts a number to a string of bytes.
+    r"""Converts a number to a string of bytes.
 
     Usage::
 
         >>> _int2bytes(123456789)
-        '\x07[\xcd\x15'
+        b'\x07[\xcd\x15'
         >>> bytes2int(_int2bytes(123456789))
         123456789
 
         >>> _int2bytes(123456789, 6)
-        '\x00\x00\x07[\xcd\x15'
+        b'\x00\x00\x07[\xcd\x15'
         >>> bytes2int(_int2bytes(123456789, 128))
         123456789
 
@@ -78,11 +79,12 @@ def _int2bytes(number, block_size=None):
 
     @throws OverflowError when block_size is given and the number takes up more
         bytes than fit into the block.
-    '''
+    """
+
     # Type checking
     if not is_integer(number):
         raise TypeError("You must pass an integer for 'number', not %s" %
-            number.__class__)
+                        number.__class__)
 
     if number < 0:
         raise ValueError('Negative numbers cannot be used: %i' % number)
@@ -99,7 +101,7 @@ def _int2bytes(number, block_size=None):
     if block_size and block_size > 0:
         if needed_bytes > block_size:
             raise OverflowError('Needed %i bytes for number, but block size '
-                'is %i' % (needed_bytes, block_size))
+                                'is %i' % (needed_bytes, block_size))
 
     # Convert the number to bytes.
     while number > 0:
@@ -116,7 +118,7 @@ def _int2bytes(number, block_size=None):
 
 
 def bytes_leading(raw_bytes, needle=ZERO_BYTE):
-    '''
+    """
     Finds the number of prefixed byte occurrences in the haystack.
 
     Useful when you want to deal with padding.
@@ -127,7 +129,8 @@ def bytes_leading(raw_bytes, needle=ZERO_BYTE):
         The byte to count. Default \000.
     :returns:
         The number of leading needle bytes.
-    '''
+    """
+
     leading = 0
     # Indexing keeps compatibility between Python 2.x and Python 3.x
     _byte = needle[0]
@@ -140,7 +143,7 @@ def bytes_leading(raw_bytes, needle=ZERO_BYTE):
 
 
 def int2bytes(number, fill_size=None, chunk_size=None, overflow=False):
-    '''
+    """
     Convert an unsigned integer to bytes (base-256 representation)::
 
     Does not preserve leading zeros if you don't specify a chunk size or
@@ -172,7 +175,8 @@ def int2bytes(number, fill_size=None, chunk_size=None, overflow=False):
         bytes than fit into the block. This requires the ``overflow``
         argument to this function to be set to ``False`` otherwise, no
         error will be raised.
-    '''
+    """
+
     if number < 0:
         raise ValueError("Number must be an unsigned integer: %d" % number)
 
@@ -202,8 +206,8 @@ def int2bytes(number, fill_size=None, chunk_size=None, overflow=False):
     if fill_size and fill_size > 0:
         if not overflow and length > fill_size:
             raise OverflowError(
-                "Need %d bytes for number, but fill size is %d" %
-                (length, fill_size)
+                    "Need %d bytes for number, but fill size is %d" %
+                    (length, fill_size)
             )
         raw_bytes = raw_bytes.rjust(fill_size, ZERO_BYTE)
     elif chunk_size and chunk_size > 0:
@@ -216,5 +220,5 @@ def int2bytes(number, fill_size=None, chunk_size=None, overflow=False):
 
 if __name__ == '__main__':
     import doctest
-    doctest.testmod()
 
+    doctest.testmod()

src/rsa/util.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''Utility functions.'''
+"""Utility functions."""
 
 from __future__ import with_statement, print_function
 
@@ -23,34 +23,35 @@ from optparse import OptionParser
 
 import rsa.key
 
+
 def private_to_public():
-    '''Reads a private key and outputs the corresponding public key.'''
+    """Reads a private key and outputs the corresponding public key."""
 
     # Parse the CLI options
     parser = OptionParser(usage='usage: %prog [options]',
-            description='Reads a private key and outputs the '
-            'corresponding public key. Both private and public keys use '
-            'the format described in PKCS#1 v1.5')
+                          description='Reads a private key and outputs the '
+                                      'corresponding public key. Both private and public keys use '
+                                      'the format described in PKCS#1 v1.5')
 
     parser.add_option('-i', '--input', dest='infilename', type='string',
-            help='Input filename. Reads from stdin if not specified')
+                      help='Input filename. Reads from stdin if not specified')
     parser.add_option('-o', '--output', dest='outfilename', type='string',
-            help='Output filename. Writes to stdout of not specified')
+                      help='Output filename. Writes to stdout of not specified')
 
     parser.add_option('--inform', dest='inform',
-            help='key format of input - default PEM',
-            choices=('PEM', 'DER'), default='PEM')
+                      help='key format of input - default PEM',
+                      choices=('PEM', 'DER'), default='PEM')
 
     parser.add_option('--outform', dest='outform',
-            help='key format of output - default PEM',
-            choices=('PEM', 'DER'), default='PEM')
+                      help='key format of output - default PEM',
+                      choices=('PEM', 'DER'), default='PEM')
 
     (cli, cli_args) = parser.parse_args(sys.argv)
 
     # Read the input data
     if cli.infilename:
-        print('Reading private key from %s in %s format' % \
-            (cli.infilename, cli.inform), file=sys.stderr)
+        print('Reading private key from %s in %s format' %
+              (cli.infilename, cli.inform), file=sys.stderr)
         with open(cli.infilename, 'rb') as infile:
             in_data = infile.read()
     else:
@@ -60,7 +61,6 @@ def private_to_public():
 
     assert type(in_data) == bytes, type(in_data)
 
-
     # Take the public fields and create a public key
     priv_key = rsa.key.PrivateKey.load_pkcs1(in_data, cli.inform)
     pub_key = rsa.key.PublicKey(priv_key.n, priv_key.e)
@@ -69,13 +69,11 @@ def private_to_public():
     out_data = pub_key.save_pkcs1(cli.outform)
 
     if cli.outfilename:
-        print('Writing public key to %s in %s format' % \
-            (cli.outfilename, cli.outform), file=sys.stderr)
+        print('Writing public key to %s in %s format' %
+              (cli.outfilename, cli.outform), file=sys.stderr)
         with open(cli.outfilename, 'wb') as outfile:
             outfile.write(out_data)
     else:
         print('Writing public key to stdout in %s format' % cli.outform,
               file=sys.stderr)
         sys.stdout.write(out_data.decode('ascii'))
-
-    

src/rsa/varblock.py

@@ -6,7 +6,7 @@
 #  you may not use this file except in compliance with the License.
 #  You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 #  Unless required by applicable law or agreed to in writing, software
 #  distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,25 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-'''VARBLOCK file support
+"""VARBLOCK file support
+
+.. deprecated:: 3.4
+
+    The VARBLOCK format is NOT recommended for general use, has been deprecated since
+    Python-RSA 3.4, and will be removed in a future release. It's vulnerable to a
+    number of attacks:
+
+    1. decrypt/encrypt_bigfile() does not implement `Authenticated encryption`_ nor
+       uses MACs to verify messages before decrypting public key encrypted messages.
+
+    2. decrypt/encrypt_bigfile() does not use hybrid encryption (it uses plain RSA)
+       and has no method for chaining, so block reordering is possible.
+
+    See `issue #19 on Github`_ for more information.
+
+.. _Authenticated encryption: https://en.wikipedia.org/wiki/Authenticated_encryption
+.. _issue #19 on Github: https://github.com/sybrenstuvel/python-rsa/issues/13
+
 
 The VARBLOCK file format is as follows, where || denotes byte concatenation:
 
@@ -31,25 +49,32 @@ The VARBLOCK file format is as follows, where || denotes byte concatenation:
 This file format is called the VARBLOCK format, in line with the varint format
 used to denote the block sizes.
 
-'''
+"""
+
+import warnings
 
 from rsa._compat import byte, b
 
-
 ZERO_BYTE = b('\x00')
 VARBLOCK_VERSION = 1
 
+warnings.warn("The 'rsa.varblock' module was deprecated in Python-RSA version "
+              "3.4 due to security issues in the VARBLOCK format. See "
+              "https://github.com/sybrenstuvel/python-rsa/issues/13 for more information.",
+              DeprecationWarning)
+
+
 def read_varint(infile):
-    '''Reads a varint from the file.
+    """Reads a varint from the file.
 
     When the first byte to be read indicates EOF, (0, 0) is returned. When an
     EOF occurs when at least one byte has been read, an EOFError exception is
     raised.
 
-    @param infile: the file-like object to read from. It should have a read()
+    :param infile: the file-like object to read from. It should have a read()
         method.
-    @returns (varint, length), the read varint and the number of read bytes.
-    '''
+    :returns: (varint, length), the read varint and the number of read bytes.
+    """
 
     varint = 0
     read_bytes = 0
@@ -58,7 +83,7 @@ def read_varint(infile):
         char = infile.read(1)
         if len(char) == 0:
             if read_bytes == 0:
-                return (0, 0)
+                return 0, 0
             raise EOFError('EOF while reading varint, value is %i so far' %
                            varint)
 
@@ -68,16 +93,16 @@ def read_varint(infile):
         read_bytes += 1
 
         if not byte & 0x80:
-            return (varint, read_bytes)
+            return varint, read_bytes
 
 
 def write_varint(outfile, value):
-    '''Writes a varint to a file.
+    """Writes a varint to a file.
 
-    @param outfile: the file-like object to write to. It should have a write()
+    :param outfile: the file-like object to write to. It should have a write()
         method.
-    @returns the number of written bytes.
-    '''
+    :returns: the number of written bytes.
+    """
 
     # there is a big difference between 'write the value 0' (this case) and
     # 'there is nothing left to write' (the false-case of the while loop)
@@ -89,7 +114,7 @@ def write_varint(outfile, value):
     written_bytes = 0
     while value > 0:
         to_write = value & 0x7f
-        value = value >> 7
+        value >>= 7
 
         if value > 0:
             to_write |= 0x80
@@ -101,12 +126,12 @@ def write_varint(outfile, value):
 
 
 def yield_varblocks(infile):
-    '''Generator, yields each block in the input file.
+    """Generator, yields each block in the input file.
 
-    @param infile: file to read, is expected to have the VARBLOCK format as
+    :param infile: file to read, is expected to have the VARBLOCK format as
         described in the module's docstring.
     @yields the contents of each block.
-    '''
+    """
 
     # Check the version number
     first_char = infile.read(1)
@@ -135,11 +160,11 @@ def yield_varblocks(infile):
 
 
 def yield_fixedblocks(infile, blocksize):
-    '''Generator, yields each block of ``blocksize`` bytes in the input file.
+    """Generator, yields each block of ``blocksize`` bytes in the input file.
 
     :param infile: file to read and separate in blocks.
     :returns: a generator that yields the contents of each block
-    '''
+    """
 
     while True:
         block = infile.read(blocksize)
@@ -152,4 +177,3 @@ def yield_fixedblocks(infile, blocksize):
 
         if read_bytes < blocksize:
             break
-

src/six.py

@@ -29,12 +29,13 @@ import sys
 import types
 
 __author__ = "Benjamin Peterson <benjamin@python.org>"
-__version__ = "1.9.0"
+__version__ = "1.10.0"
 
 
 # Useful for very coarse version differentiation.
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] == 3
+PY34 = sys.version_info[0:2] >= (3, 4)
 
 if PY3:
     string_types = str,
@@ -57,6 +58,7 @@ else:
     else:
         # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
         class X(object):
+
             def __len__(self):
                 return 1 << 31
         try:
@@ -88,7 +90,7 @@ class _LazyDescr(object):
 
     def __get__(self, obj, tp):
         result = self._resolve()
-        setattr(obj, self.name, result) # Invokes __set__.
+        setattr(obj, self.name, result)  # Invokes __set__.
         try:
             # This is a bit ugly, but it avoids running this again by
             # removing this descriptor.
@@ -160,12 +162,14 @@ class MovedAttribute(_LazyDescr):
 
 
 class _SixMetaPathImporter(object):
+
     """
     A meta path importer to import six.moves and its submodules.
 
     This class implements a PEP302 finder and loader. It should be compatible
     with Python 2.5 and all existing versions of Python3
     """
+
     def __init__(self, six_module_name):
         self.name = six_module_name
         self.known_modules = {}
@@ -223,6 +227,7 @@ _importer = _SixMetaPathImporter(__name__)
 
 
 class _MovedItems(_LazyModule):
+
     """Lazy loading of moved objects"""
     __path__ = []  # mark as package
 
@@ -234,8 +239,10 @@ _moved_attributes = [
     MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
     MovedAttribute("intern", "__builtin__", "sys"),
     MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"),
+    MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"),
     MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
-    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
+    MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"),
     MovedAttribute("reduce", "__builtin__", "functools"),
     MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
     MovedAttribute("StringIO", "StringIO", "io"),
@@ -245,7 +252,6 @@ _moved_attributes = [
     MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
     MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
     MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"),
-
     MovedModule("builtins", "__builtin__"),
     MovedModule("configparser", "ConfigParser"),
     MovedModule("copyreg", "copy_reg"),
@@ -292,8 +298,13 @@ _moved_attributes = [
     MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
     MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"),
     MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"),
-    MovedModule("winreg", "_winreg"),
 ]
+# Add windows specific modules.
+if sys.platform == "win32":
+    _moved_attributes += [
+        MovedModule("winreg", "_winreg"),
+    ]
+
 for attr in _moved_attributes:
     setattr(_MovedItems, attr.name, attr)
     if isinstance(attr, MovedModule):
@@ -307,6 +318,7 @@ _importer._add_module(moves, "moves")
 
 
 class Module_six_moves_urllib_parse(_LazyModule):
+
     """Lazy loading of moved objects in six.moves.urllib_parse"""
 
 
@@ -346,6 +358,7 @@ _importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_pa
 
 
 class Module_six_moves_urllib_error(_LazyModule):
+
     """Lazy loading of moved objects in six.moves.urllib_error"""
 
 
@@ -365,6 +378,7 @@ _importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.er
 
 
 class Module_six_moves_urllib_request(_LazyModule):
+
     """Lazy loading of moved objects in six.moves.urllib_request"""
 
 
@@ -414,6 +428,7 @@ _importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.
 
 
 class Module_six_moves_urllib_response(_LazyModule):
+
     """Lazy loading of moved objects in six.moves.urllib_response"""
 
 
@@ -434,6 +449,7 @@ _importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib
 
 
 class Module_six_moves_urllib_robotparser(_LazyModule):
+
     """Lazy loading of moved objects in six.moves.urllib_robotparser"""
 
 
@@ -451,6 +467,7 @@ _importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.url
 
 
 class Module_six_moves_urllib(types.ModuleType):
+
     """Create a six.moves.urllib namespace that resembles the Python 3 namespace"""
     __path__ = []  # mark as package
     parse = _importer._get_module("moves.urllib_parse")
@@ -521,6 +538,9 @@ if PY3:
 
     create_bound_method = types.MethodType
 
+    def create_unbound_method(func, cls):
+        return func
+
     Iterator = object
 else:
     def get_unbound_function(unbound):
@@ -529,6 +549,9 @@ else:
     def create_bound_method(func, obj):
         return types.MethodType(func, obj, obj.__class__)
 
+    def create_unbound_method(func, cls):
+        return types.MethodType(func, None, cls)
+
     class Iterator(object):
 
         def next(self):
@@ -567,16 +590,16 @@ if PY3:
     viewitems = operator.methodcaller("items")
 else:
     def iterkeys(d, **kw):
-        return iter(d.iterkeys(**kw))
+        return d.iterkeys(**kw)
 
     def itervalues(d, **kw):
-        return iter(d.itervalues(**kw))
+        return d.itervalues(**kw)
 
     def iteritems(d, **kw):
-        return iter(d.iteritems(**kw))
+        return d.iteritems(**kw)
 
     def iterlists(d, **kw):
-        return iter(d.iterlists(**kw))
+        return d.iterlists(**kw)
 
     viewkeys = operator.methodcaller("viewkeys")
 
@@ -595,15 +618,13 @@ _add_doc(iterlists,
 if PY3:
     def b(s):
         return s.encode("latin-1")
+
     def u(s):
         return s
     unichr = chr
-    if sys.version_info[1] <= 1:
-        def int2byte(i):
-            return bytes((i,))
-    else:
-        # This is about 2x faster than the implementation above on 3.2+
-        int2byte = operator.methodcaller("to_bytes", 1, "big")
+    import struct
+    int2byte = struct.Struct(">B").pack
+    del struct
     byte2int = operator.itemgetter(0)
     indexbytes = operator.getitem
     iterbytes = iter
@@ -611,18 +632,25 @@ if PY3:
     StringIO = io.StringIO
     BytesIO = io.BytesIO
     _assertCountEqual = "assertCountEqual"
-    _assertRaisesRegex = "assertRaisesRegex"
-    _assertRegex = "assertRegex"
+    if sys.version_info[1] <= 1:
+        _assertRaisesRegex = "assertRaisesRegexp"
+        _assertRegex = "assertRegexpMatches"
+    else:
+        _assertRaisesRegex = "assertRaisesRegex"
+        _assertRegex = "assertRegex"
 else:
     def b(s):
         return s
     # Workaround for standalone backslash
+
     def u(s):
         return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape")
     unichr = unichr
     int2byte = chr
+
     def byte2int(bs):
         return ord(bs[0])
+
     def indexbytes(buf, i):
         return ord(buf[i])
     iterbytes = functools.partial(itertools.imap, ord)
@@ -650,7 +678,6 @@ def assertRegex(self, *args, **kwargs):
 if PY3:
     exec_ = getattr(moves.builtins, "exec")
 
-
     def reraise(tp, value, tb=None):
         if value is None:
             value = tp()
@@ -671,7 +698,6 @@ else:
             _locs_ = _globs_
         exec("""exec _code_ in _globs_, _locs_""")
 
-
     exec_("""def reraise(tp, value, tb=None):
     raise tp, value, tb
 """)
@@ -699,13 +725,14 @@ if print_ is None:
         fp = kwargs.pop("file", sys.stdout)
         if fp is None:
             return
+
         def write(data):
             if not isinstance(data, basestring):
                 data = str(data)
             # If the file has an encoding, encode unicode with it.
             if (isinstance(fp, file) and
-                isinstance(data, unicode) and
-                fp.encoding is not None):
+                    isinstance(data, unicode) and
+                    fp.encoding is not None):
                 errors = getattr(fp, "errors", None)
                 if errors is None:
                     errors = "strict"
@@ -748,6 +775,7 @@ if print_ is None:
         write(end)
 if sys.version_info[:2] < (3, 3):
     _print = print_
+
     def print_(*args, **kwargs):
         fp = kwargs.get("file", sys.stdout)
         flush = kwargs.pop("flush", False)
@@ -768,12 +796,14 @@ if sys.version_info[0:2] < (3, 4):
 else:
     wraps = functools.wraps
 
+
 def with_metaclass(meta, *bases):
     """Create a base class with a metaclass."""
     # This requires a bit of explanation: the basic idea is to make a dummy
     # metaclass for one level of class instantiation that replaces itself with
     # the actual metaclass.
     class metaclass(meta):
+
         def __new__(cls, name, this_bases, d):
             return meta(name, bases, d)
     return type.__new__(metaclass, 'temporary_class', (), {})
@@ -830,7 +860,7 @@ if sys.meta_path:
         # the six meta path importer, since the other six instance will have
         # inserted an importer with different class.
         if (type(importer).__name__ == "_SixMetaPathImporter" and
-            importer.name == __name__):
+                importer.name == __name__):
             del sys.meta_path[i]
             break
     del i, importer

src/whatsnew.txt

@@ -1,3 +1,253 @@
+GAM 3.72
+- Chrome OS device actions, disable, re-enable and deprovision devices.
+- (beta) MSI Windows build
+- (beta) binary Linux and MacOS builds
+- Numerous fixes and updates by Ross
+
+GAM 3.71
+- Fix update first / last name.
+- upgrade GAM versions of oauth2client, googleapiclient, RSA and six
+- Improved UTF-8 support for CSV commands (Ross)
+- Authorization flow improvements by Ross
+- Other minor cleanups and fixes
+
+GAM 3.7
+- Classroom Guardians API. Invite, list and delete guardians for a student.
+- Includes number of improvements from Ross in 3.66 (see below)
+
+To use this version, you must update the list of authorized scopes for your Gam OAuth2 Client and Service Account.
+Go here: https://github.com/jay0lee/GAM/wiki/CreatingClientSecretsFile
+Log on to the admin console as in steps 6.ii.c, d, e.
+In the list of Authorized API clients, locate your Gam OAuth2 Client, copy the Client ID and then remove the entry.
+Paste the Client ID into the Client name box as in step 6.ii.f, then do steps 6.ii.g and 6.ii.h.
+You'll notice that the API Scopes - OAuth2 list has additional entries, these are what is required in Gam 3.7.
+Skip down to step 6.iii.
+In the list of Authorized API clients, locate your Gam Service Account, copy the Client ID and then remove the entry.
+Paste the Client ID into the Client name box as in step 6.iii.c, then do steps 6.iii.d and 6.iii.e.
+You'll notice that the API Scopes - Service Account list has additional entries, these are what is required in Gam 3.7.
+
+GAM 3.66
+See GamCommands.txt for a complete syntax description.
+
+Added arguments to gam info group to suppress aliases listing and include groups of which this group is a member.
+gam info group <Group> ... [noaliases] [groups]
+
+Added argument to gam print cros to limit number of activeTimeRanges and recentUsers entries
+gam print cros ... [listlimit <Number>]
+
+Added argument to gam <UserTypeEntity> signature and gam <UserTypeEntity> vacation to allow specification of file character set so that extended characters can be read.
+Credit to Steve Main for suggesting the following enhancement.
+Added argument to gam <UserTypeEntity> signature and gam <UserTypeEntity> vacation to allow pattern substitution in the signature and vacation message.
+gam <UserTypeEntity> signature <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)*
+gam <UserTypeEntity> vacation <TrueValues> subject <String> (message <String>)|(file <FileName> [charset <CharSet>]) (replace <Tag> <String>)*
+	[contactsonly] [domainonly] [startdate <Date>] [enddate <Date>]
+Every instance of {<Tag>} in the signature/message will be replaced by <String>. Instances of the form {RT}...{Text}...{/RT} will be eliminated
+if there was no <Tag> specified that matches Text or if a <Tag> matching Text was specified but the matching <String> is empty.
+This is especially useful with CSV files.
+gam csv Users.csv gam user "~User" signature file SignatureTemplate.txt replace "#User#" "~User" replace "#Title#" "~Title"
+
+Added argument to gam <UserTypeEntity> show signature to format the signature.
+gam <UserTypeEntity> show signature [format]
+
+Added argument to gam add/update calendar to allow specification of event notifications
+gam <UserTypeEntity> add/update calendar <Calendar> notification email|sms eventcreation|eventchange|eventcancellation|eventresponce|agenda
+
+Added option to reminder and notification arguments of update calendar to allowing clearing of reminders/notifications.
+gam <UserTypeEntity> update calendar <Calendar> [reminder clear] [notification clear]
+
+Added arguments to gam print group-members to allow selecting subsets of groups.
+Added argument to gam print group-members to add member full name to output,
+Added argument to gam print group-members to allow output field selection.
+gam print group-members [todrive] ([domain <DomainName>] [member <UserItem>])|[group <GroupItem>] [membernames] [fields <MembersFieldNameList>]
+MembersFieldNameList is a comma separated list of field names: email | group | id | name | role | type
+
+Added argument to gam info user to specify SKUs for which license information is desired.
+gam info user [<UserItem>] ... [skus <SKUIDList>]
+
+Added argument to gam print printjobs and gam printjob <PrinterID> fetch to allow specifying the maximum number of print jobs to retrieve.
+gam printjob <PrinterID> fetch		... [limit <Number>]
+gam print printjobs  ... [limit <Number>]
+limit <Number> specifies the maximum number of print jobs to retrieve; defaults to 25, set limit to 0 to retrieve all print jobs.
+
+Credit to Seth Stein for the following enhancements.
+Added argument to gam <UserTypeEntity> get drivefile to allow downloading a specific revision of a drive file.
+gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(query <Query>) [format <FileFormatList>] [targetfolder <FilePath>] [revision <Number>]
+Added command to show drive file revisions.
+gam <UserTypeEntity> show filerevisions <DriveFileID>
+
+Added argument to gam print adminroles to allow uploading to Google Drive.
+gam print adminroles [todrive]
+
+Added group, groups, mobile arguments to gam report.
+gam report admin|calendar|calendars|drive|docs|doc|groups|group|logins|login|mobile|tokens|token ...
+
+Added command to show user Google+ profile.
+gam <UserTypeEntity> show gplusprofile [todrive]
+To enable this command, visit: https://github.com/jay0lee/GAM/wiki/CreatingClientSecretsFile
+In step 3.v, enable the Google+ API
+In step 6.iii.d, add https://www.googleapis.com/auth/plus.me to the API scopes - Service Account list, then remove and re-add the authorization.
+
+Added argument to gam <UserTypeEntity> show labels to allow seeing message counts for each label.
+gam <UserTypeEntity> show labels|label [onlyuser] [showcounts]
+
+Added argument to gam <UserTypeEntity> show fileinfo to allow field selection.
+gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
+
+Added argument to gam update group to allow removing members by role.
+gam update group <Group> clear [members] [managers] [owners]
+If no option follows clear, all members will removed.
+
+Changed gam print admins to include 'id:' in OrgUnitID column as with other gam print commands.
+
+Fixed GAM to handle both future date error messages in gam report
+
+Fixed gam <UserTypeEntity> show delegates to handle Unicode characters in delagator name.
+
+Fixed gam <UserTypeEntity> get drivefile to properly handle file extension.
+
+Fixed gam create alias <Name> target <Group>.
+
+2016/07/29
+
+Added command to empty drive drive trash.
+gam <UserTypeEntity> empty drivetrash
+
+Added alternative command to add delegates and command to print delegates.
+gam <UserTypeEntity> add delegate|delegates <UserEntity>
+gam <UserTypeEntity> print delegates [todrive]
+
+Improved Gmail filter processing.
+gam <UserTypeEntity> [add] filter [from <EmailAddress>] [to <EmailAddress>] [subject <String>] [haswords|query <List>] [nowords|negatedquery <List>] [musthaveattachment|hasattachment] [excludechats] [size larger|smaller <ByteCount>]
+	[label <LabelID>] [important|notimportant] [star] [trash] [markread] [archive] [neverspam] [forward <EmailAddress>]
+gam <UserTypeEntity> delete filters <FilterIDEntity>
+gam <UserTypeEntity> show filters
+gam <UserTypeEntity> info filters <FilterIDEntity>
+gam <UserTypeEntity> print filters [todrive]
+
+Added commands to process Gmail forwarding addresses.
+gam <UserTypeEntity> add forwardingaddress|forwardingaddresses <EmailAddressEntity>
+gam <UserTypeEntity> delete forwardingaddress|forwardingaddresses <EmailAddressEntity>
+gam <UserTypeEntity> show forwardingaddress|forwardingaddresses
+gam <UserTypeEntity> info forwardingaddress|forwardingaddresses <EmailAddressEntity>
+gam <UserTypeEntity> print forwardingaddress|forwardingaddresses [todrive]
+
+Improved Gmail forward processing.
+gam <UserTypeEntity> forward <FalseValues>
+gam <UserTypeEntity> forward <TrueValues> keep|leaveininbox|archive|delete|trash|markread <EmailAddress>
+gam <UserTypeEntity> show forward
+gam <UserTypeEntity> print forward [todrive]
+
+Improved Gmail sendas processing.
+gam <UserTypeEntity> [add] sendas <EmailAddress> <Name> [replyto <EmailAddress>] [default] [treatasalias <Boolean>] [signature|sig <String>|(file <FileName> [charset <CharSet>]) (replace <REPattern> <String>)*]
+gam <UserTypeEntity> update sendas <EmailAddress> [name <Name>] [replyto <EmailAddress>] [default] [treatasalias <Boolean>] [signature|sig <String>|(file <FileName> [charset <CharSet>]) (replace <REPattern> <String>)*]
+gam <UserTypeEntity> delete sendas <EmailAddressEntity>
+gam <UserTypeEntity> show sendas [format]
+gam <UserTypeEntity> info sendas <EmailAddressEntity> [format]
+gam <UserTypeEntity> print sendas [todrive]
+
+Improved Gmail signature processing.
+gam <UserTypeEntity> signature|sig <String>|(file <FileName> [charset <Charset>]) (replace <Tag> <String>)* [name <String>] [replyto <EmailAddress>] 
+gam <UserTypeEntity> show signature|sig [format]
+
+Use Gmail API for POP/IMAP/Vacation processing.
+gam <UserTypeEntity> imap|imap4 <Boolean> [noautoexpunge] [expungebehavior archive|deleteforever|trash] [maxfoldersize 0|1000|2000|5000|10000]
+gam <UserTypeEntity> pop|pop3 <Boolean> [for allmail|newmail|mailfromnowon|fromnowown] [action keep|leaveininbox|archive|delete|trash|markread]
+gam <UserTypeEntity> vacation <FalseValues>
+gam <UserTypeEntity> vacation <TrueValues> subject <String> (message <String>)|(file <FileName> [charset <CharSet>]) (replace <Tag> <String>)* [html]
+	[contactsonly] [domainonly] [startdate <Date>] [enddate <Date>]
+gam <UserTypeEntity> show vacation [format]
+
+Added command toGet information about a specific calendar.
+gam <UserTypeEntity> info calendar <EmailAddress>
+
+Added command to print calendars to CSV file, dropped all arguments from gam show calendars.
+gam <UserTypeEntity> print calendars [todrive]
+gam <UserTypeEntity> show calendars
+
+Added command to print Gmail Profiles to CSV file, dropped all arguments from gam show gmailprofile.
+gam <UserTypeEntity> print gmailprofile [todrive]
+gam <UserTypeEntity> show gmailprofile
+
+Added command to print Gplus Profiles to CSV file, dropped all arguments from gam show gplusprofile.
+gam <UserTypeEntity> print gplusprofile [todrive]
+gam <UserTypeEntity> show gplusprofile
+
+Added command to print user schemas to CSV file, renamed command to display formatted user schemas to gam show schemas.
+gam print schemas [todrive]
+gam show schemas
+
+Added command to print user access tokens to CSV file.
+gam <UserTypeEntity> print tokens|token|3lo|oauth [todrive]
+
+Added arguments to gam info cros to allow specification of desired output fields.
+gam info cros <CrosDeviceEntity> [nolists] [listlimit <Number>]
+	[basic|full|allfields] <CrOSFieldName>* [fields <CrOSFieldNameList>]
+
+Added drivedir and targetfolder <FilePath> arguments to gam printjob fetch and gam get photo to
+allow specification of the destination folder for the file retrieved from Google. The default
+location for these commands is the current working directory, drivedir specifies the value of the environment variable GAMDRIVEDIR and
+targetfolder <FilePath> specifies a user-choosen path.
+gam printjob <PrinterID>|any fetch
+	[olderthan|newerthan <PrintJobAge>] [query <QueryPrintJob>]
+	[status <PrintJobStatus>]
+	[orderby <PrintJobOrderByFieldName> [ascending|descending]]
+	[owner|user <EmailAddress>]
+	[limit <Number>] [drivedir|(targetfolder <FilePath>)]
+gam <UserTypeEntity> get photo [drivedir|(targetfolder <FilePath>)]
+
+Added noshow argument to gam get photo to suppress displaying of photo data
+gam <UserTypeEntity> get photo [drivedir|(targetfolder <FilePath>)] [noshow]
+
+Changed gam print cros to match gam print users. Previously, gam print cros produced a full listing of CrOS devices
+and gam print users produced a listing of primaryEmail addresses. Now, gam print cros produces a listing of deviceIds.
+To get the previous behavior, say gam print cros full. See GamCommands.txt for a summary of CrOS and User printing.
+
+Commands that produce CSV file output have been changed to make the leftmost column(s) be the key fields.
+If you have scripts that process the CSV files as flat files, expecting the columns to be in a particular
+order, they will have to be updated. If your scripts process the CSV files by column header, no changes should be required.
+
+2016/07/31
+
+Changed gam get drivefile to take a list of file formats rather than a single format. The first format in the list that is available will be used.
+Added drivefilename argument to allow downloading file by name.
+<FileFormat> ::= csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
+<FileFormatList> ::= '<FileFormat>(,<FileFormat)*'
+gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>) [format <FileFormatList>] [targetfolder <FilePath>] [revision <Number>]
+
+2016/08/01
+
+Added delimiter <String> argument to gam print courses to allow choice of delimiter to separate aliases.
+gam print courses [todrive] [teacher] [student] [alias|aliases] [delimiter <String>]
+
+Added notsuspended argument to gam update group add/sync to prevent adding suspended users to a group
+when specifying all users, org <OrgUnitPath> or query <QueryUser> for <UserTypeEntity>.
+gam update group <GroupItem> add [owner|manager|member] [notsuspended] <UserTypeEntity>
+gam update group <GroupItem> sync [owner|manager|member] [notsuspended] <UserTypeEntity>
+
+Added basic|full argument to gam print mobile to allow specification of output detail desired.
+gam print mobile [todrive] [query <QueryMobile>] [basic|full] [orderby <MobileOrderByFieldName> [ascending|descending]]
+
+GAM 3.65
+  -fix vacation issues (Ross)
+  -fix Windows path issues (Ross)
+  -Add message undelete (Ross) and modify (Jay) commands
+  -Improve message delete performance
+  -Upgrade to new versions of oauth2client and googleapiclient
+
+GAM 3.63
+  -"gam update group ... sync users" now does batch add/remove of users for faster sync
+  -"gam file" can now use - to read list of users from stdin
+  -"gam csv file:column" can read CSV file of users and specify column to be used.
+
+GAM 3.62
+  -New Admin Roles API allows you to create, delete and print delegated and super admins
+  -New Resource Calendar API replaces old GData API and allows you to create, update, get info, delete and print all resource calendars.
+  -Major cleanups and design updates from Ross Scroggs @taers232c mean GAM is more reliable and stable for your needs. Huge thanks Ross!
+  -Guard additional fields with convertUTF8
+  -Correct gam create resource to replace resType with type
+  -Add type as an argument to gam print resources to make resource type visible
+  -Handle students/teachers with missing emails in gam course sync
+
 GAM 3.61
   -Various fixes by Ross Scroggs for Domain API and Data Transfer commands
   -Remove duplicate DoCreateDomain which broke "gam create domain"

src/windows-build.bat

@@ -0,0 +1,24 @@
+rmdir /q /s gam
+rmdir /q /s gam-64
+rmdir /q /s build
+rmdir /q /s dist
+del /q /f gam-%1-windows.zip
+del /q /f gam-%1-windows-x64.zip
+del /q /f gam-%1-windows-x64.msi
+del /q /f gam.wixobj
+del /q /f gam.wixpdb
+
+c:\python27-32\scripts\pyinstaller --clean -F --distpath=gam windows-gam.spec
+xcopy LICENSE gam\
+xcopy whatsnew.txt gam\
+del gam\w9xpopen.exe
+"%ProgramFiles%\7-Zip\7z.exe" a -tzip gam-%1-windows.zip gam\ -xr!.svn
+
+c:\python27-64\scripts\pyinstaller --clean -F --distpath=gam-64 windows-gam.spec
+xcopy LICENSE gam-64\
+xcopy whatsnew.txt gam-64\
+"%ProgramFiles%\7-Zip\7z.exe" a -tzip gam-%1-windows-x64.zip gam-64\ -xr!.svn
+
+set GAMVERSION=%1
+"%ProgramFiles(x86)%\WiX Toolset v3.10\bin\candle.exe" -arch x64 gam.wxs
+"%ProgramFiles(x86)%\WiX Toolset v3.10\bin\light.exe" -ext "%ProgramFiles(x86)%\WiX Toolset v3.10\bin\WixUIExtension.dll" gam.wixobj -o gam-%1-windows-x64.msi

src/windows-gam.spec

@@ -1,24 +1,27 @@
-# -*- mode: python -*-
-a = Analysis(['gam.py'],
-             pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
-             hiddenimports=[],
-             hookspath=None,
-             runtime_hooks=None)
-for d in a.datas:
-    if 'pyconfig' in d[0]: 
-        a.datas.remove(d)
-        break
-a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')]
-a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
-a.datas += [('admin-settings-v1.json', 'admin-settings-v1.json', 'DATA')]
-pyz = PYZ(a.pure)
-exe = EXE(pyz,
-          a.scripts,
-          a.binaries,
-          a.zipfiles,
-          a.datas,
-          name='gam.exe',
-          debug=False,
-          strip=None,
-          upx=True,
-          console=True )
+# -*- mode: python -*-
+a = Analysis(['gam.py'],
+             pathex=['C:\\Users\\jlee\\Documents\\GitHub\\GAM'],
+             hiddenimports=[],
+             hookspath=None,
+             excludes=['_tkinter'],
+             runtime_hooks=None)
+for d in a.datas:
+    if 'pyconfig' in d[0]:
+        a.datas.remove(d)
+        break
+a.datas += [('httplib2/cacerts.txt', 'httplib2\cacerts.txt', 'DATA')]
+a.datas += [('admin-settings-v2.json', 'admin-settings-v2.json', 'DATA')]
+a.datas += [('cloudprint-v2.json', 'cloudprint-v2.json', 'DATA')]
+a.datas += [('email-audit-v1.json', 'email-audit-v1.json', 'DATA')]
+a.datas += [('email-settings-v2.json', 'email-settings-v2.json', 'DATA')]
+pyz = PYZ(a.pure)
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='gam.exe',
+          debug=False,
+          strip=None,
+          upx=True,
+          console=True )