GAM 5.05

Linux/MacOS
fileId=`gam user user@domain.com create drivefile ...`
Windows PowerShell
$fileId = & gam user user@domain.com create drivefile ...`

.travis.yml

@@ -1,14 +1,15 @@
 if: tag IS blank
 os: linux
 language: python
+dist: xenial
 
 env:
   global:
     - BUILD_PYTHON_VERSION=3.8.2
-    - MIN_PYTHON_VERSION=3.8.1
-    - BUILD_OPENSSL_VERSION=1.1.1e
-    - MIN_OPENSSL_VERSION=1.1.1d
-    - PATCHELF_VERSION=0.9
+    - MIN_PYTHON_VERSION=3.8.2
+    - BUILD_OPENSSL_VERSION=1.1.1f
+    - MIN_OPENSSL_VERSION=1.1.1f
+    - PATCHELF_VERSION=0.10
     - PYINSTALLER_VERSION=3.5
     - secure: "FSKvLaiqhKz21SVgAQZI3bSX34Ffyev4l+R2G//QXNDu6UVQcuFsykzw+eZEG7fkhotXr8BMDL7xIkookiL8eLwUtcd/Z95HCjPBBHcmCSQleyvuuJBxdrQ9xldmiGLzMCYiumSH9OH4uJhQ39Yjnjsa8TK+PlTci6a/BTzlYyBSyDYDf7Iv/uhfQPDHL3pNwrQPHf4fL6/jcvo+uaPcv83AVZkNzZjjyoi9Aa+uh9xlbyHg11jp44463qqxoxTdYik3pYuXRBPjknjOGcnFHqn+QOVSdRQoiwbmT8xVuYuCzTv9THhuJ//i5u7s4y3Xyl7u17B3tdm86UlMpQHy/w9EsYaSBPOU4oPNomRtOnTSugh0v9ZBwptP5XfbslII/iA+LQdzTHhchn0W0CRyDqjOMSestWlrsq5NZJtBJTYHbebllOhEI7xbj9tY+re1zFWSPMOPgHJP23ovsdk3hD9OT93AzRHInCx5IxL6QvEgRhAancRuGkf2rGP0g/vX9fQ0Il3rNMSQxHB5CyHUBtUJ9nhU79YkMDZicD0jFMEwjWJO3itAp3ynoLXRgktgQCYUfgc9SpdWKD5SXLCYnSo22JD3D1P6h2EertRHaoKRLb+CRXQC/lM8uh/W+BjA2Xe6Vut2I/72ndjM+10T7E2xk1CFyCH37a5p8cH26Fs="
     - secure: "J9380tGLOZWa7dSH1y5Il8T5JQpN6ad81gI6VR1HIU0svpRdjgikyDA7ca2MKYDUYYY9yVSkTV6gCl6iIU/9+SKaYugpP+tkvdGYkC2moJdcTgYM/WOnIK9ExQ3BPhN1neGxJjPTwKo1ft27mtZ2I5vuCiBwIcnKWLnKPyW3PD+mWpfqiLuEzkHoAh6G3jC4qbcCrZDeX/knE+PzqESUEi+8k1G8gYcSDWujba9ypSsqZ8T/MXagGla6l7y2Rz+/KZTJmFHwKAA10V+xPLVqxoiqi4ar66yUqy0BamwRXPcseI+ns3Q+4lUpMqVQ5GlRy7LF1xC8myjmcAexXk0F9hg+CMzewKI8UgmQH/ZJvQZEh8s6mW26+CqA4d3zMQkWaR0WtEtpiuH7AGHCflIqvEQ6UiG7ia3B8iZfW2wl0j/kqx4OuHkS3r0pWKVVIIvCj9Ow2BHP7SpiV1AcUGsVxzwbgTh67fitna3Z3c6Uj8ccQlNr7ZIt1az6Wf3w5njijkLOiBpQSLKunTTCTSge/JzBTKUcie3RE9vzirl58gUxAt36nDtPWnory+RttMZrOkBVbTeSxp+IUe8pNwLFPHABsafXsjkfzBOtFmm+0ZXWt2Rlog5NvlemJfQUWDlsL4g+BSakzN+4sIPKzSauWDHyaEeULY7Uprkil6c5zwo="
@@ -43,134 +44,70 @@ jobs:
       name: "Linux 64-bit Bionic"
       dist: bionic
       language: shell
-      env:
-         - GAMOS=linux
-         - PLATFORM=x86_64
-         - VMTYPE=build
     - os: linux
       name: "Linux 64-bit Xenial"
       dist: xenial
       language: shell
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: linux
       dist: bionic
       arch: arm64
       name: "Linux ARM64 Bionic"
       language: shell
       filter_secrets: false
-      env:
-        - GAMOS=linux
-        - PLATFORM=arm64
-        - VMTYPE=build
     - os: linux
       dist: xenial
       arch: arm64
       name: "Linux ARM64 Xenial"
       language: shell
       filter_secrets: false
-      env:
-        - GAMOS=linux
-        - PLATFORM=arm64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Trusty"
-      dist: trusty
-      language: shell
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
-    - os: linux
-      name: "Linux 64-bit Precise"
-      dist: precise
-      language: shell
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: linux
       name: "Python 3.6 Source Testing"
-      dist: bionic
       language: python
       python: 3.6
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
     - os: linux
       name: "Python 3.7 Source Testing"
-      dist: bionic
       language: python
       python: 3.7
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
     - os: linux
       name: "Python nightly Source Testing"
-      dist: bionic
       language: python
       python: nightly
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
     - os: linux
       name: "Python PyPi Source Testing"
-      dist: xenial
       language: python
       python: pypy3
-      env:
-        - GAMOS=linux
-        - PLATFORM=x86_64
-        - VMTYPE=test
-    - os: osx
-      name: "MacOS 10.12"
-      language: generic
-      osx_image: xcode9.2
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: osx
       name: "MacOS 10.13"
       language: generic
       osx_image: xcode10.1
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: osx
       name: "MacOS 10.14"
       language: generic
       osx_image: xcode11.3
-      env:
-        - GAMOS=macos
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: windows
       name: "Windows 64-bit"
       language: shell
-      env:
-        - GAMOS=windows
-        - PLATFORM=x86_64
-        - VMTYPE=build
     - os: windows
       name: "Windows 32-bit"
       language: shell
-      env:
-        - GAMOS=windows
-        - PLATFORM=x86
-        - VMTYPE=build
 
 before_install:
-- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-before-install.sh
+- if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
+    export GAMOS="macos";
+  else
+    export GAMOS="${TRAVIS_OS_NAME}";
+  fi
+- if [ "${TRAVIS_JOB_NAME}" == "Windows 32-bit" ]; then
+    export PLATFORM="x86";
+  elif [ "${TRAVIS_CPU_ARCH}" == "amd64" ]; then
+    export PLATFORM="x86_64";
+  else
+    export PLATFORM="${TRAVIS_CPU_ARCH}";
+  fi
+- source src/travis/${TRAVIS_OS_NAME}-before-install.sh
 
 install:
-- source src/travis/$TRAVIS_OS_NAME-$PLATFORM-install.sh
+- source src/travis/${TRAVIS_OS_NAME}-install.sh
 
 script:
 # Discover and run all Python unit tests. Buffer output so that it's not sent to the build log.
@@ -179,16 +116,16 @@ script:
 - $gam version extended
 - $gam version | grep travis # travis should be part of the path (not /tmp or such)
 # determine which Python version GAM is built with and ensure it's at least build version from above.
-- if [ "$VMTYPE" == "build" ]; then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $MIN_PYTHON_VERSION; fi
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]];  then vline=$($gam version | grep "Python "); python_line=($vline); this_python=${python_line[1]}; $python tools/a_atleast_b.py $this_python $MIN_PYTHON_VERSION; fi
 # determine which OpenSSL version GAM is built with and ensure it's at least build version from above.
-- if [ "$VMTYPE" == "build" ]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $MIN_OPENSSL_VERSION; fi
-- if [ "$VMTYPE" == "build" ]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
-- if [ "$VMTYPE" == "build" ]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then vline=$($gam version extended | grep "OpenSSL "); openssl_line=($vline); this_openssl=${openssl_line[1]}; $python tools/a_atleast_b.py $this_openssl $MIN_OPENSSL_VERSION; fi
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then $gam version extended | grep TLSv1\.[23]; fi # Builds should default TLS 1.2 or 1.3 to Google
+- if [[ "$TRAVIS_JOB_NAME" != *"Testing" ]]; then GAM_TLS_MIN_VERSION=TLSv1_2 $gam version extended location tls-v1-0.badssl.com:1010; [[ $? == 3 ]]; fi # expect fail since server doesn't support our TLS version
 - export jid="$(cut -d'.' -f2 <<<"$TRAVIS_JOB_NUMBER")"
 - if [ "$TRAVIS_EVENT_TYPE" != "pull_request" ]; then export e2e=true; fi
 - if [ "$e2e" = true ]; then export gam_user=gam-travis-$jid@pdl.jaylee.us; fi
 - if [ "$e2e" = true ]; then openssl aes-256-cbc -K $encrypted_ab10ec38326e_key -iv $encrypted_ab10ec38326e_iv -in travis/oauth2service.json.enc -out $gampath/oauth2service.json -d; fi
-- if [ "$e2e" = true ]; then cat travis/cfg_template.json | python travis/svars-write.py &> /dev/null; fi
+- if [ "$e2e" = true ]; then cat travis/cfg_template.json | $python travis/svars-write.py &> /dev/null; fi
 - if [ "$e2e" = true ]; then $gam info domain; fi
 - if [ "$e2e" = true ]; then $gam oauth info; fi
 - if [ "$e2e" = true ]; then $gam oauth refresh; fi
@@ -250,7 +187,7 @@ script:
 - if [ "$e2e" = true ]; then $gam calendar $gam_user printevents after -0d; fi
 - if [ "$e2e" = true ]; then $gam create vaultmatter name "Travis matter $newbase" description "test matter" collaborators $newuser; fi
 - if [ "$e2e" = true ]; then $gam create vaulthold matter "Travis matter $newbase" name "Travis hold $newbase" corpus mail accounts $newuser; fi
-- if [ "$e2e" = true ]; then $gam print vaultmatters; fi
+- if [ "$e2e" = true ]; then $gam print vaultmatters matterstate open; fi
 - if [ "$e2e" = true ]; then $gam print vaultholds matter "Travis matter $newbase"; fi
 - if [ "$e2e" = true ]; then $gam create vaultexport matter "Travis matter $newbase" name "Travis export $newbase" corpus mail accounts $newuser; fi
 - if [ "$e2e" = true ]; then $gam print exports matter "Travis matter $newbase" | $gam csv - gam info export id:~~matterId~~ id:~~id~~; fi
@@ -271,8 +208,10 @@ script:
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
 - if [ "$e2e" = true ]; then $gam print mobile; fi
 - if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$e2e" = true ]; then $gam report usageparameters customer; fi
+- if [ "$e2e" = true ]; then $gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins; fi
 - if [ "$e2e" = true ]; then $gam report customer todrive; fi
-- if [ "$e2e" = true ]; then $gam report users fulldatarequired accounts,gmail fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
+- if [ "$e2e" = true ]; then $gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive; fi
 - if [ "$e2e" = true ]; then $gam report admin start -3d todrive; fi
 
 before_deploy:
@@ -290,4 +229,4 @@ deploy:
   draft: true
   on:
     repo: jay0lee/GAM
-    condition: $VMTYPE = build
+    condition: $TRAVIS_JOB_NAME != *"Testing"

src/GamCommands.txt

@@ -38,6 +38,7 @@ If an item contains spaces, it should be surrounded by ".
 	papayawhip|peachpuff|peru|pink|plum|powderblue|purple|red|rosybrown|royalblue|
 	saddlebrown|salmon|sandybrown|seagreen|seashell|sienna|silver|skyblue|slateblue|slategray|slategrey|snow|springgreen|steelblue|
 	tan|teal|thistle|tomato|turquoise|violet|wheat|white|whitesmoke|yellow|yellowgreen
+<DayOfWeek> ::= mon|tue|wed|thu|fri|sat|sun
 <FileFormat> ::=
 	csv|html|txt|tsv|jpeg|jpg|png|svg|pdf|rtf|pptx|xlsx|docx|odt|ods|openoffice|ms|microsoft|micro$oft
 <LabelColorHex> ::=
@@ -914,12 +915,28 @@ gam info resoldsubscriptions <CustomerID> [customer_auth_token <String>]
 	sites
 <ReportsAppList> ::= "<ReportsApp>(,<ReportsApp>)*"
 
-gam report users|user [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
-	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)] [filter|filters <String>] [fields|parameters <String>]
-gam report customers|customer|domain [todrive] [date <Date>] [fulldatarequired all|<ReportsAppList>]
+gam report usageparameters customer|user [todrive]
+gam report usage user [todrive]
+	[<UserTypeItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>(,<Date>)*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+gam report usage customer [todrive]
+	[startdate <Date>] [enddate <Date>]
+	[skipdates <Date>(,<Date>)*] [skipdaysofweek <DayOfWeek>(,<DayOfWeek>)*]
+	[fields|parameters <String>]
+
+gam report users|user [todrive]
+	[(user <UserItem>)|(orgunit|org|ou <OrgUnitPath>)]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
+	[filter|filters <String>] [fields|parameters <String>]
+gam report customers|customer|domain [todrive]
+	[date <Date>] [fulldatarequired all|<ReportsAppList>]
 	[fields|parameters <String>]
 gam report <ActivityApplicationName> [todrive]
-	[start <Time>] [end <Time>] [(user all|<UserItem>)] [event <String>] [filter|filters <String>] [ip <String>]
+	[user all|<UserItem>]
+	[start <Time>] [end <Time>]
+	[filter|filters <String>] [event <String>] [ip <String>]
 
 gam create admin <UserItem> <RoleItem> customer|(org_unit <OrgUnitItem>)
 gam delete admin <RoleAssignmentId>
@@ -1005,7 +1022,6 @@ The following attributes are equivalent:
 	guestscantinviteothers|
 	guestscantseeothers|
 	hangoutsmeet|
-	(id <String>)|
 	(location <String>)|
 	(noreminders| (reminder <Number> email|popup|sms))|
 	(optionalattendee <EmailAddress>)|
@@ -1019,6 +1035,11 @@ The following attributes are equivalent:
 	(timezone <Timezone>)|
 	(visibility default|public|prvate)
 
+<EventUpdateAttributes> ::=
+	<EventAttributes>|
+	(removeattendee <EmailAddress>)|
+	(replacedescription <RegularExpression> <String>)
+
 <EventSelectProperty:> ::=
 	(after <Time>)|
 	(before <Time>)|
@@ -1030,9 +1051,10 @@ The following attributes are equivalent:
 <EventDisplayProperty> ::=
 	(timezone <TimeZone>)
 
-gam calendar <CalendarItem> addevent <EventAttributes>+ [<EventNotificationAttribute>]
+gam calendar <CalendarItem> addevent [id <String>] <EventAttributes>+ [<EventNotificationAttribute>]
 gam calendar <CalendarItem> deleteevent id|eventid <EventID> [doit] [<EventNotificationAttribute>]
 gam calendar <CalendarItem> moveevent id|eventid <EventID> [doit] [<EventNotificationAttribute>]
+gam calendar <CalendarItem> updateevent <EventID> <EventUpdateAttributes>+ [<EventNotificationAttribute>]
 gam calendar <CalendarItem> wipe
 gam calendar <CalendarItem> printevents <EventSelectProperty>* <EventDisplayProperty>* [todrive]
 
@@ -1248,7 +1270,7 @@ gam reopen vaultmatter|matter <MatterItem>
 gam delete vaultmatter|matter <MatterItem>
 gam undelete vaultmatter|matter <MatterItem>
 gam info vaultmatter|matter <MatterItem>
-gam print vaultmatters|matters [todrive] [basic|full]
+gam print vaultmatters|matters [todrive] [basic|full] [matterstate open|closed|deleted]
 
 gam <UserTypeEntity> delete|del asp|asps|applicationspecificpasswords all|<ASPIDList>
 gam <UserTypeEntity> show asps|asp|applicationspecificpasswords
@@ -1277,7 +1299,7 @@ gam <UserTypeEntity> show fileinfo <DriveFileID> [allfields|<DriveFieldName>*]
 gam <UserTypeEntity> show filerevisions <DriveFileID>
 gam <UserTypeEntity> show filetree [anyowner] (orderby <DriveOrderByFieldName> [ascending|descending])*
 
-gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive]
+gam <UserTypeEntity> create|add drivefile [drivefilename <DriveFileName>] <DriveFileAddAttributes>* [csv] [todrive] [returnidonly]
 gam <UserTypeEntity> update drivefile (id <DriveFileID)|(drivefilename <DriveFileName>)|(query <QueryDriveFile) [copy] [newfilename <DriveFileName>] <DriveFileUpdateAttributes>*
 gam <UserTypeEntity> get drivefile (id <DriveFileID>)|(drivefilename <DriveFileName>)|(query <QueryDriveFile>)
 	[revision <Number>] [(format <FileFormatList>)|(csvsheet <String>)]

src/auth/__init__.py

@@ -1,10 +1,10 @@
 """Authentication/Credentials general purpose and convenience methods."""
 
-from . import oauth
+import transport
 from var import _FN_OAUTH2_TXT
 from var import GC_OAUTH2_TXT
 from var import GC_Values
-
+from . import oauth
 # TODO: Move logic that determines file name into this module. We should be able
 # to discover the file location without accessing a private member or waiting
 # for a global initialization.
@@ -12,16 +12,15 @@ DEFAULT_OAUTH_STORAGE_FILE = _FN_OAUTH2_TXT
 
 
 def get_admin_credentials_filename():
-  """Gets the name of the file that stores the admin account credentials."""
-  # If the environment globals are loaded, use the set global value. It may have
-  # some custom name in it. Otherwise, just use the default name.
-  if GC_Values[GC_OAUTH2_TXT]:
-    return GC_Values[GC_OAUTH2_TXT]
-  else:
+    """Gets the name of the file that stores the admin account credentials."""
+    # If the environment globals are loaded, use the set global value. It may have
+    # some custom name in it. Otherwise, just use the default name.
+    if GC_Values[GC_OAUTH2_TXT]:
+        return GC_Values[GC_OAUTH2_TXT]
     return DEFAULT_OAUTH_STORAGE_FILE
 
 
 def get_admin_credentials():
-  """Gets oauth.Credentials that are authenticated as the domain's admin user."""
-  credential_file = get_admin_credentials_filename()
-  return oauth.Credentials.from_credentials_file(credential_file)
+    """Gets oauth.Credentials that are authenticated as the domain's admin user."""
+    credential_file = get_admin_credentials_filename()
+    return oauth.Credentials.from_credentials_file(credential_file)

src/auth/oauth.py

@@ -382,7 +382,12 @@ class Credentials(google.oauth2.credentials.Credentials):
     """
     if not self.id_token:
       raise CredentialsError('Failed to fetch token data. No id_token present.')
+
     request = transport.create_request()
+    if self.expired:
+      # The id_token needs to be unexpired, in order to request data about it.
+      self.refresh(request)
+
     self._id_token_data = google.oauth2.id_token.verify_oauth2_token(
         self.id_token, request)
 

src/auth/oauth_test.py

@@ -336,6 +336,26 @@ class CredentialsTest(unittest.TestCase):
         id_token_data=self.fake_token_data)
     self.assertEqual('Unknown', creds.get_token_value('unknown-field'))
 
+  @patch.object(oauth.google.oauth2.id_token, 'verify_oauth2_token')
+  def test_get_token_value_credentials_expired(self, mock_verify_oauth2_token):
+    mock_verify_oauth2_token.return_value = {'fetched-field': 'fetched-value'}
+    time_earlier_than_now = datetime.datetime.now() - datetime.timedelta(
+        minutes=5)
+    creds = oauth.Credentials(
+        token=self.fake_token,
+        client_id=self.fake_client_id,
+        client_secret=self.fake_client_secret,
+        expiry=time_earlier_than_now,
+        id_token=self.fake_id_token,
+        id_token_data=None)
+    self.assertTrue(creds.expired)
+    creds.refresh = MagicMock()
+
+    token_value = creds.get_token_value('fetched-field')
+
+    self.assertEqual('fetched-value', token_value)
+    self.assertTrue(creds.refresh.called)
+
   def test_to_json_contains_all_required_fields(self):
     creds = oauth.Credentials(
         token=self.fake_token,
@@ -585,7 +605,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_returns_shortened_url(self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
         self.fake_client_config, scopes=self.fake_scopes)
@@ -609,7 +629,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_request_error(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
@@ -625,7 +645,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_non_200_response_status(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
@@ -644,7 +664,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_bad_json_response(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(
@@ -663,7 +683,7 @@ class ShortUrlFlowTest(unittest.TestCase):
 
   @patch.object(oauth.google_auth_oauthlib.flow.InstalledAppFlow,
                 'authorization_url')
-  @unittest.skip("disable short url tests temporarily.")
+  @unittest.skip('disable short url tests temporarily.')
   def test_shorturlflow_falls_back_to_long_url_on_empty_short_url_field(
       self, mock_super_auth_url):
     url_flow = oauth._ShortURLFlow.from_client_config(

src/display.py

@@ -1,6 +1,7 @@
 """Methods related to display of information to the user."""
 
 import csv
+import datetime
 import io
 import sys
 import webbrowser
@@ -129,8 +130,8 @@ def write_csv_file(csvRows, titles, list_type, todrive):
       return False
     return rowBoolean == filterBoolean
 
-  def headerFilterMatch(title):
-    for filterStr in GC_Values[GC_CSV_HEADER_FILTER]:
+  def headerFilterMatch(filters, title):
+    for filterStr in filters:
       if filterStr.match(title):
         return True
     return False
@@ -150,10 +151,13 @@ def write_csv_file(csvRows, titles, list_type, todrive):
         csvRows = [row for row in csvRows if rowCountFilterMatch(row.get(column, 0), filterVal[1], filterVal[2])]
       else: #boolean
         csvRows = [row for row in csvRows if rowBooleanFilterMatch(row.get(column, False), filterVal[1])]
-  if GC_Values[GC_CSV_HEADER_FILTER]:
-    titles = [t for t in titles if headerFilterMatch(t)]
+  if GC_Values[GC_CSV_HEADER_FILTER] or GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+    if GC_Values[GC_CSV_HEADER_DROP_FILTER]:
+      titles = [t for t in titles if not headerFilterMatch(GC_Values[GC_CSV_HEADER_DROP_FILTER], t)]
+    if GC_Values[GC_CSV_HEADER_FILTER]:
+      titles = [t for t in titles if headerFilterMatch(GC_Values[GC_CSV_HEADER_FILTER], t)]
     if not titles:
-      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER\n')
+      controlflow.system_error_exit(3, 'No columns selected with GAM_CSV_HEADER_FILTER and GAM_CSV_HEADER_DROP_FILTER\n')
       return
   csv.register_dialect('nixstdout', lineterminator='\n')
   if todrive:

src/gam-install.sh

@@ -28,8 +28,8 @@ upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.27 2.23 2.19 2.15"
-gam_macos_vers="10.14.6 10.13.6 10.12.6"
+gam_glibc_vers="2.27 2.23"
+gam_macos_vers="10.14.6 10.13.6"
 
 while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
@@ -238,9 +238,11 @@ fi
 if [ "$update_profile" = true ]; then
   alias_line="gam() { \"$target_dir/gam/gam\" \"\$@\" ; }"
   if [ "$gamos" == "linux" ]; then
-    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0
+    update_profile "$HOME/.zshrc" 0
   elif [ "$gamos" == "macos" ]; then
-    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.zshrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.bash_aliases" 0 || update_profile "$HOME/.bash_profile" 0 || update_profile "$HOME/.bashrc" 0 || update_profile "$HOME/.profile" 1
+    update_profile "$HOME/.zshrc" 0
   fi
 else
   echo_yellow "skipping profile update."

src/gam.py

@@ -98,12 +98,15 @@ if platform.system() == 'Linux':
 if os.environ.get('STATICX_PROG_PATH', False):
   # StaticX static executable
   GM_Globals[GM_GAM_PATH] = os.path.dirname(os.environ['STATICX_PROG_PATH'])
+  GM_Globals[GM_GAM_TYPE] = 'staticx'
   # Pyinstaller executable
 elif getattr(sys, 'frozen', False):
   GM_Globals[GM_GAM_PATH] = os.path.dirname(sys.executable)
+  GM_Globals[GM_GAM_TYPE] = 'pyinstaller'
 else:
   # Source code
   GM_Globals[GM_GAM_PATH] = os.path.dirname(os.path.realpath(__file__))
+  GM_Globals[GM_GAM_TYPE] = 'pythonsource'
 
 def showUsage():
   doGAMVersion(checkForArgs=False)
@@ -228,15 +231,6 @@ def getLabelColor(color):
     controlflow.expected_argument_exit("label color", ", ".join(LABEL_COLORS), color)
   controlflow.system_error_exit(2, f'A label color must be # and six hex characters (#012345); got {color}')
 
-def integerLimits(minVal, maxVal, item='integer'):
-  if (minVal is not None) and (maxVal is not None):
-    return f'{item} {minVal}<=x<={maxVal}'
-  if minVal is not None:
-    return f'{item} x>={minVal}'
-  if maxVal is not None:
-    return f'{item} x<={maxVal}'
-  return f'{item} x'
-
 def getInteger(value, item, minVal=None, maxVal=None):
   try:
     number = int(value.strip())
@@ -244,7 +238,7 @@ def getInteger(value, item, minVal=None, maxVal=None):
       return number
   except ValueError:
     pass
-  controlflow.system_error_exit(2, f'expected {item} in range <{integerLimits(minVal, maxVal)}>, got {value}')
+  controlflow.system_error_exit(2, f'expected {item} in range <{utils.integerLimits(minVal, maxVal)}>, got {value}')
 
 def removeCourseIdScope(courseId):
   if courseId.startswith('d:'):
@@ -437,6 +431,7 @@ def SetGlobalVariables():
   _getOldEnvVar(GC_AUTO_BATCH_MIN, 'GAM_AUTOBATCH')
   _getOldEnvVar(GC_BATCH_SIZE, 'GAM_BATCH_SIZE')
   _getOldEnvVar(GC_CSV_HEADER_FILTER, 'GAM_CSV_HEADER_FILTER')
+  _getOldEnvVar(GC_CSV_HEADER_DROP_FILTER, 'GAM_CSV_HEADER_DROP_FILTER')
   _getOldEnvVar(GC_CSV_ROW_FILTER, 'GAM_CSV_ROW_FILTER')
   _getOldEnvVar(GC_TLS_MIN_VERSION, 'GAM_TLS_MIN_VERSION')
   _getOldEnvVar(GC_TLS_MAX_VERSION, 'GAM_TLS_MAX_VERSION')
@@ -608,7 +603,7 @@ def doGAMVersion(checkForArgs=True):
     return
   pyversion = platform.python_version()
   cpu_bits = struct.calcsize('P') * 8
-  print((f'GAM {gam_version} - {GAM_URL}\n'
+  print((f'GAM {gam_version} - {GAM_URL} - {GM_Globals[GM_GAM_TYPE]}\n'
          f'{gam_author}\n'
          f'Python {pyversion} {cpu_bits}-bit {sys.version_info.releaselevel}\n'
          f'google-api-python-client {googleapiclient.__version__}\n'
@@ -3529,7 +3524,7 @@ def doUpdateDriveFile(users):
         print(f'Successfully copied {fileId} to {result["id"]}')
 
 def createDriveFile(users):
-  csv_output = to_drive = False
+  csv_output = return_id_only = to_drive = False
   csv_rows = []
   csv_titles = ['User', 'title', 'id']
   media_body = None
@@ -3546,6 +3541,9 @@ def createDriveFile(users):
     elif myarg == 'todrive':
       to_drive = True
       i += 1
+    elif myarg == 'returnidonly':
+      return_id_only = True
+      i += 1
     else:
       i = getDriveFileAttribute(i, body, parameters, myarg, False)
   for user in users:
@@ -3564,10 +3562,12 @@ def createDriveFile(users):
                        ocrLanguage=parameters[DFA_OCRLANGUAGE],
                        media_body=media_body, body=body, fields='id,title,mimeType',
                        supportsAllDrives=True)
-    titleInfo = f'{result["title"]}({result["id"]})'
-    if csv_output:
+    if return_id_only:
+      sys.stdout.write(f"{result['id']}\n")
+    elif csv_output:
       csv_rows.append({'User': user, 'title': result['title'], 'id': result['id']})
     else:
+      titleInfo = f'{result["title"]}({result["id"]})'
       if parameters[DFA_LOCALFILENAME]:
         print(f'Successfully uploaded {parameters[DFA_LOCALFILENAME]} to Drive File {titleInfo}')
       else:
@@ -6327,7 +6327,7 @@ def _createClientSecretsOauth2service(httpObj, projectId, login_hint, create_pro
 
 {console_url}
 
-1. Choose "Desktop App" for "Application type".
+1. Choose "Desktop App" or "Other" for "Application type".
 2. Enter a desired value for "Name" or leave as is.
 3. Click the blue "Create" button.
 4. Copy the "client ID" value that shows on the next page.
@@ -8754,6 +8754,8 @@ def send_email(subject, body, recipient=None, sender=None, user=None, method='se
   if not user:
     user = _getValueFromOAuth('email')
   userId, gmail = buildGmailGAPIObject(user)
+  if not gmail:
+    return
   resource = gmail.users().messages()
   if labels:
     api_body['labelIds'] = labelsToLabelIds(gmail, labels)
@@ -8763,6 +8765,11 @@ def send_email(subject, body, recipient=None, sender=None, user=None, method='se
   if not recipient:
     recipient = userId
     default_recipient = True
+  # Force ASCII for RFC compliance
+  # xmlcharref seems to work to display at least
+  # some unicode in HTML body and is ignored in
+  # plain text body.
+  body = body.encode('ascii', 'xmlcharrefreplace').decode()
   msg = message_from_string(body)
   for header, value in msgHeaders.items():
     msg.__delitem__(header) # can remove multiple case-insensitive matching headers

src/gapi/__init__.py

@@ -63,7 +63,7 @@ def call(service,
       if http_status == -1:
         # The error detail indicated that we should retry this request
         # We'll refresh credentials and make another pass
-        service._http.request.credentials.refresh(transport.create_http())
+        service._http.credentials.refresh(transport.create_http())
         continue
       if http_status == 0:
         return None

src/gapi/__init___test.py

@@ -108,7 +108,7 @@ class GapiTest(unittest.TestCase):
         self.mock_service, self.mock_method_name, soft_errors=True)
     self.assertEqual(response, fake_200_response)
     self.assertEqual(
-        self.mock_service._http.request.credentials.refresh.call_count, 1)
+        self.mock_service._http.credentials.refresh.call_count, 1)
     self.assertEqual(self.mock_method.return_value.execute.call_count, 2)
 
   def test_call_throws_for_provided_reason(self):

src/gapi/calendar.py

@@ -306,7 +306,6 @@ def addOrUpdateEvent(action):
     if not cal:
         return
     # only way for non-Google calendars to get updates is via email
-    timeZone = None
     kwargs = {}
     body = {}
     if action == 'add':
@@ -353,8 +352,9 @@ def getEventAttributes(i, calendarId, cal, body, action):
             i += 2
         elif myarg == 'removeattendee' and action == 'update':
             remove_email = sys.argv[i+1].lower()
-            body['attendees'] = _remove_attendee(body['attendees'],
-                                                 remove_email)
+            if 'attendees' in body:
+                body['attendees'] = _remove_attendee(body['attendees'],
+                                                     remove_email)
             i += 2
         elif myarg == 'optionalattendee':
             body.setdefault('attendees', [])
@@ -367,14 +367,15 @@ def getEventAttributes(i, calendarId, cal, body, action):
         elif myarg == 'description':
             body['description'] = sys.argv[i+1].replace('\\n', '\n')
             i += 2
-        elif myarg == 'replacedescription':
+        elif myarg == 'replacedescription' and action == 'update':
             search = sys.argv[i+1]
             replace = sys.argv[i+2]
-            body['description'] = re.sub(search, replace, body['description'])
+            if 'description' in body:
+                body['description'] = re.sub(search, replace, body['description'])
             i += 3
         elif myarg == 'start':
             if sys.argv[i+1].lower() == 'allday':
-                body['start'] = {'date': __main__.getYYYYMMDD(sys.argv[i+2])}
+                body['start'] = {'date': utils.get_yyyymmdd(sys.argv[i+2])}
                 i += 3
             else:
                 start_time = utils.get_time_or_delta_from_now(sys.argv[i+1])
@@ -382,7 +383,7 @@ def getEventAttributes(i, calendarId, cal, body, action):
                 i += 2
         elif myarg == 'end':
             if sys.argv[i+1].lower() == 'allday':
-                body['end'] = {'date': __main__.getYYYYMMDD(sys.argv[i+2])}
+                body['end'] = {'date': utils.get_yyyymmdd(sys.argv[i+2])}
                 i += 3
             else:
                 end_time = utils.get_time_or_delta_from_now(sys.argv[i+1])

src/gapi/directory/cros.py

@@ -421,7 +421,7 @@ def doPrintCrosActivity():
                         utils.formatMilliSeconds(active_time)
                     newrow['activeTimeRanges.minutes'] = \
                         activeTimeRange['activeTime']//60000
-                    csvRows.append(new_row)
+                    csvRows.append(newrow)
             if selectRecentUsers:
                 recentUsers = cros.get('recentUsers', [])
                 lenRU = len(recentUsers)
@@ -444,11 +444,11 @@ def doPrintCrosActivity():
                 lenDF = len(deviceFiles)
                 num_ranges = min(lenDF, listLimit or lenDF)
                 for deviceFile in deviceFiles[:num_ranges]:
-                    new_row = row.copy()
-                    new_row['deviceFiles.type'] = deviceFile['type']
+                    newrow = row.copy()
+                    newrow['deviceFiles.type'] = deviceFile['type']
                     create_time = deviceFile['createTime']
-                    new_row['deviceFiles.createTime'] = create_time
-                    csvRows.append(new_row)
+                    newrow['deviceFiles.createTime'] = create_time
+                    csvRows.append(newrow)
     display.write_csv_file(csvRows, titles, 'CrOS Activity', todrive)
 
 
@@ -707,7 +707,6 @@ def doPrintCrosDevices():
                     tempInfos = cpuStatusReports[i].get('cpuTemperatureInfo',
                                                         [])
                     for tempInfo in tempInfos:
-                        temperature = tempInfo['temperature']
                         label = tempInfo["label"].strip()
                         base = 'cpuStatusReports.cpuTemperatureInfo.'
                         nrow[f'{base}{label}'] = tempInfo['temperature']

src/gapi/reports.py

@@ -1,6 +1,11 @@
+import calendar
 import datetime
+import re
 import sys
 
+from dateutil.parser import parse
+from dateutil.relativedelta import relativedelta
+
 import __main__
 from var import *
 import controlflow
@@ -29,16 +34,220 @@ REPORT_CHOICE_MAP = {
     'logins': 'login',
     'oauthtoken': 'token',
     'tokens': 'token',
+    'usage': 'usage',
+    'usageparameters': 'usageparameters',
     'users': 'user',
     'useraccounts': 'user_accounts',
 }
 
 
+def showUsageParameters():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID,
+                     gapi.errors.ErrorReason.BAD_REQUEST]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit(
+            'user or customer', 'report usageparameters')
+    report = sys.argv[3].lower()
+    titles = ['parameter']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = {}
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = {'userKey': __main__._getValueFromOAuth('email')}
+    else:
+        controlflow.expected_argument_exit(
+            'usageparameters', ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    tryDate = datetime.date.today().strftime(YYYYMMDD_FORMAT)
+    partial_apps = []
+    all_parameters = []
+    one_day = datetime.timedelta(days=1)
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'todrive':
+            todrive = True
+            i += 1
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], "gam report usageparameters")
+    while True:
+        try:
+            response = gapi.call(endpoint, 'get',
+                                 throw_reasons=throw_reasons,
+                                 date=tryDate,
+                                 customerId=customerId,
+                                 **kwargs)
+            partial_on_thisday = []
+            for warning in response.get('warnings', []):
+                for data in warning.get('data', []):
+                    if data.get('key') == 'application':
+                        partial_on_thisday.append(data['value'])
+            if partial_apps:
+                partial_apps = [app for app in partial_apps if app in partial_on_thisday]
+            else:
+                partial_apps = partial_on_thisday
+            for parameter in response['usageReports'][0]['parameters']:
+                name = parameter.get('name')
+                if name and name not in all_parameters:
+                    all_parameters.append(name)
+            if not partial_apps:
+                break
+            tryDate = (utils.get_yyyymmdd(tryDate, returnDateTime=True) - \
+                    one_day).strftime(YYYYMMDD_FORMAT)
+        except gapi.errors.GapiInvalidError as e:
+            tryDate = _adjust_date(str(e))
+    all_parameters.sort()
+    csvRows = []
+    for parameter in all_parameters:
+        csvRows.append({'parameter': parameter})
+    display.write_csv_file(
+        csvRows, titles, f'{report.capitalize()} Report Usage Parameters', todrive)
+
+REPORTS_PARAMETERS_SIMPLE_TYPES = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
+
+def showUsage():
+    rep = buildGAPIObject()
+    throw_reasons = [gapi.errors.ErrorReason.INVALID,
+                     gapi.errors.ErrorReason.BAD_REQUEST]
+    todrive = False
+    if len(sys.argv) == 3:
+        controlflow.missing_argument_exit(
+            'user or customer', 'report usage')
+    report = sys.argv[3].lower()
+    titles = ['date']
+    if report == 'customer':
+        endpoint = rep.customerUsageReports()
+        kwargs = [{}]
+    elif report == 'user':
+        endpoint = rep.userUsageReport()
+        kwargs = [{'userKey': 'all'}]
+        titles.append('user')
+    else:
+        controlflow.expected_argument_exit(
+            'usage', ['user', 'customer'], report)
+    customerId = GC_Values[GC_CUSTOMER_ID]
+    if customerId == MY_CUSTOMER:
+        customerId = None
+    parameters = []
+    start_date = end_date = orgUnitId = None
+    skip_day_numbers = []
+    skip_dates = []
+    i = 4
+    while i < len(sys.argv):
+        myarg = sys.argv[i].lower().replace('_', '')
+        if myarg == 'startdate':
+            start_date = utils.get_yyyymmdd(sys.argv[i+1], returnDateTime=True)
+            i += 2
+        elif myarg == 'enddate':
+            end_date = utils.get_yyyymmdd(sys.argv[i+1], returnDateTime=True)
+            i += 2
+        elif myarg == 'todrive':
+            todrive = True
+            i += 1
+        elif myarg in ['fields', 'parameters']:
+            parameters = sys.argv[i+1].split(',')
+            i += 2
+        elif myarg == 'skipdates':
+            skips = sys.argv[i+1].split(',')
+            skip_dates = [utils.get_yyyymmdd(d) for d in skips]
+            i += 2
+        elif myarg == 'skipdaysofweek':
+            skipdaynames = sys.argv[i+1].split(',')
+            dow = [d.lower() for d in calendar.day_abbr]
+            skip_day_numbers = [dow.index(d) for d in skipdaynames if d in dow]
+            i += 2
+        elif report == 'user' and myarg in ['orgunit', 'org', 'ou']:
+            _, orgUnitId = __main__.getOrgUnitId(sys.argv[i+1])
+            i += 2
+        elif report == 'user' and myarg in usergroup_types:
+            users = __main__.getUsersToModify(myarg, sys.argv[i+1])
+            kwargs = [{'userKey': user} for user in users]
+            i += 2
+        else:
+            controlflow.invalid_argument_exit(sys.argv[i], f'gam report usage {report}')
+    if parameters:
+        titles.extend(parameters)
+        parameters = ','.join(parameters)
+    else:
+        parameters = None
+    if not start_date:
+        start_date = datetime.datetime.now() + relativedelta(months=-1)
+    if not end_date:
+        end_date = datetime.datetime.now()
+    if orgUnitId:
+        for kw in kwargs:
+            kw['orgUnitID'] = orgUnitId
+    one_day = datetime.timedelta(days=1)
+    usage_on_date = start_date
+    start_date = usage_on_date.strftime('%Y-%m-%d')
+    csvRows = []
+    while usage_on_date <= end_date:
+        use_date = usage_on_date.strftime('%Y-%m-%d')
+        if usage_on_date.weekday() in skip_day_numbers or \
+           use_date in skip_dates:
+            usage_on_date += one_day
+            continue
+        usage_on_date += one_day
+        try:
+            for kwarg in kwargs:
+                try:
+                    usage = gapi.get_all_pages(endpoint, 'get',
+                                               'usageReports',
+                                               throw_reasons=throw_reasons,
+                                               customerId=customerId,
+                                               date=use_date,
+                                               parameters=parameters,
+                                               **kwarg)
+                except gapi.errors.GapiBadRequestError:
+                    continue
+                for entity in usage:
+                    row = {'date': use_date}
+                    if 'userEmail' in entity['entity']:
+                        row['user'] = entity['entity']['userEmail']
+                    for item in entity.get('parameters', []):
+                        if 'name' not in item:
+                            continue
+                        name = item['name']
+                        if name == 'cros:device_version_distribution':
+                            for cros_ver in item['msgValue']:
+                                v = cros_ver['version_number']
+                                column_name = f'cros:num_devices_chrome_{v}'
+                                if column_name not in titles:
+                                    titles.append(column_name)
+                                row[column_name] = cros_ver['num_devices']
+                        else:
+                            if not name in titles:
+                                titles.append(name)
+                            for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
+                                if ptype in item:
+                                    row[name] = item[ptype]
+                                    break
+                            else:
+                                row[name] = ''
+                    csvRows.append(row)
+        except gapi.errors.GapiInvalidError as e:
+            display.print_warning(str(e))
+            break
+    display.write_csv_file(
+        csvRows, titles, f'{report.capitalize()} Usage Report - {start_date}:{use_date}', todrive)
+
+
 def showReport():
     rep = buildGAPIObject()
     throw_reasons = [gapi.errors.ErrorReason.INVALID]
     report = sys.argv[2].lower()
     report = REPORT_CHOICE_MAP.get(report.replace('_', ''), report)
+    if report == 'usage':
+      showUsage()
+      return
+    if report == 'usageparameters':
+      showUsageParameters()
+      return
     valid_apps = gapi.get_enum_values_minus_unspecified(
         rep._rootDesc['resources']['activities']['methods']['list'][
             'parameters']['applicationName']['enum'])+['customer', 'user']
@@ -130,7 +339,6 @@ def showReport():
             sys.exit(1)
         titles = ['email', 'date']
         csvRows = []
-        ptypes = ['intValue', 'boolValue', 'datetimeValue', 'stringValue']
         for user_report in usage:
             if 'entity' not in user_report:
                 continue
@@ -142,7 +350,7 @@ def showReport():
                 name = item['name']
                 if not name in titles:
                     titles.append(name)
-                for ptype in ptypes:
+                for ptype in REPORTS_PARAMETERS_SIMPLE_TYPES:
                     if ptype in item:
                         row[name] = item[ptype]
                         break

src/gapi/storage.py

@@ -18,7 +18,7 @@ def build_gapi():
 
 
 def get_cloud_storage_object(s, bucket, object_, local_file=None,
-                           expectedMd5=None):
+                             expectedMd5=None):
     if not local_file:
         local_file = object_
     if os.path.exists(local_file):

src/gapi/vault.py

@@ -653,6 +653,7 @@ def printMatters():
     initialTitles = ['matterId', 'name', 'description', 'state']
     titles = initialTitles[:]
     view = 'FULL'
+    state = None
     i = 3
     while i < len(sys.argv):
         myarg = sys.argv[i].lower().replace('_', '')
@@ -662,12 +663,22 @@ def printMatters():
         elif myarg in PROJECTION_CHOICES_MAP:
             view = PROJECTION_CHOICES_MAP[myarg]
             i += 1
+        elif myarg == 'matterstate':
+            valid_states = gapi.get_enum_values_minus_unspecified(
+                v._rootDesc['schemas']['Matter']['properties']['state'][
+                    'enum'])
+            state = sys.argv[i+1].upper()
+            if state not in valid_states:
+                controlflow.expected_argument_exit(
+                    'state', ', '.join(valid_states), state)
+            i += 2
         else:
             controlflow.invalid_argument_exit(myarg, "gam print matters")
     __main__.printGettingAllItems('Vault Matters', None)
     page_message = gapi.got_total_items_msg('Vault Matters', '...\n')
     matters = gapi.get_all_pages(
-        v.matters(), 'list', 'matters', page_message=page_message, view=view)
+        v.matters(), 'list', 'matters', page_message=page_message, view=view,
+        state=state)
     for matter in matters:
         display.add_row_titles_to_csv_file(
             utils.flatten_json(matter), csvRows, titles)
@@ -695,13 +706,11 @@ def printExports():
         else:
             controlflow.invalid_argument_exit(myarg, "gam print exports")
     if not matters:
-        fields = 'matters(matterId,state),nextPageToken'
+        fields = 'matters(matterId),nextPageToken'
         matters_results = gapi.get_all_pages(v.matters(
         ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
         for matter in matters_results:
-            matterState = matter['state']
-            matterId = matter['matterId']
-            matterIds.append(matterId)
+            matterIds.append(matter['matterId'])
     else:
         for matter in matters:
             matterIds.append(getMatterItem(v, matter))
@@ -736,13 +745,11 @@ def printHolds():
         else:
             controlflow.invalid_argument_exit(myarg, "gam print holds")
     if not matters:
-        fields = 'matters(matterId,state),nextPageToken'
+        fields = 'matters(matterId),nextPageToken'
         matters_results = gapi.get_all_pages(v.matters(
         ), 'list', 'matters', view='BASIC', state='OPEN', fields=fields)
         for matter in matters_results:
-            matterState = matter['state']
-            matterId = matter['matterId']
-            matterIds.append(matterId)
+            matterIds.append(matter['matterId'])
     else:
         for matter in matters:
             matterIds.append(getMatterItem(v, matter))

src/project-apis.txt

@@ -12,7 +12,6 @@ gmail.googleapis.com
 groupssettings.googleapis.com
 iam.googleapis.com
 licensing.googleapis.com
-plus.googleapis.com
 reseller.googleapis.com
 sheets.googleapis.com
 siteverification.googleapis.com

src/transport.py

@@ -27,13 +27,16 @@ def create_http(cache=None,
   Returns:
     httplib2.Http with the specified options.
   """
-  tls_minimum_version = override_min_tls if override_min_tls else GC_Values[GC_TLS_MIN_VERSION]
-  tls_maximum_version = override_max_tls if override_max_tls else GC_Values[GC_TLS_MAX_VERSION]
-  httpObj = httplib2.Http(ca_certs=GC_Values[GC_CA_FILE],
-                          tls_maximum_version=tls_maximum_version,
-                          tls_minimum_version=tls_minimum_version,
-                          cache=cache,
-                          timeout=timeout)
+  tls_minimum_version = override_min_tls if override_min_tls else GC_Values.get(
+      GC_TLS_MIN_VERSION)
+  tls_maximum_version = override_max_tls if override_max_tls else GC_Values.get(
+      GC_TLS_MAX_VERSION)
+  httpObj = httplib2.Http(
+      ca_certs=GC_Values.get(GC_CA_FILE),
+      tls_maximum_version=tls_maximum_version,
+      tls_minimum_version=tls_minimum_version,
+      cache=cache,
+      timeout=timeout)
   httpObj.redirect_codes = set(httpObj.redirect_codes) - {308}
   return httpObj
 
@@ -68,7 +71,9 @@ def _force_user_agent(user_agent):
         if kwargs['headers'].get('user-agent'):
           if user_agent not in kwargs['headers']['user-agent']:
             # Save the existing user-agent header and tack on our own.
-            kwargs['headers']['user-agent'] = f'{user_agent} {kwargs["headers"]["user-agent"]}'
+            kwargs['headers']['user-agent'] = (
+                f'{user_agent} '
+                f'{kwargs["headers"]["user-agent"]}')
         else:
           kwargs['headers']['user-agent'] = user_agent
       else:

src/travis/linux-arm64-before-install.sh

@@ -1,75 +0,0 @@
-export whereibelong=$(pwd)
-export dist=$(lsb_release --codename --short)
-echo "We are running on Ubuntu $dist"
-export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
-cpucount=$(nproc --all)
-echo "This device has $cpucount CPUs for compiling..."
-echo -e "nameserver 8.8.8.8\nnameserver 8.8.4.4" > /tmp/resolv.conf
-sudo cp /tmp/resolv.conf /etc
-sudo apt-get -qq --yes update > /dev/null
-sudo apt-get -qq --yes install xz-utils > /dev/null
-SSLVER=$(~/ssl/bin/openssl version)
-SSLRESULT=$?
-PYVER=$(~/python/bin/python3 -V)
-PYRESULT=$?
-if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $BUILD_PYTHON_VERSION"* ]]; then
-  echo "RUNNING: apt dist-upgrade..."
-  sudo apt-get -qq --yes dist-upgrade > /dev/null
-  echo "Installing build tools..."
-  sudo apt-get -qq --yes install build-essential
-
-  echo "Installing deps for python3"
-  sudo cp -v /etc/apt/sources.list /tmp
-  sudo chmod a+rwx /tmp/sources.list
-  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
-  sudo cp -v /tmp/sources.list /etc/apt
-  sudo apt-get -qq --yes update > /dev/null
-  sudo apt-get -qq --yes build-dep python3 > /dev/null
-  sudo apt-get -qq --yes install zlib1g-dev > /dev/null
-  sudo apt-get -qq --yes install libffi-dev > /dev/null
-
-  # Compile latest OpenSSL
-  echo "Downloading OpenSSL..."
-  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  echo "Extracting OpenSSL..."
-  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  cd openssl-$BUILD_OPENSSL_VERSION
-  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-  ./config shared --prefix=$HOME/ssl
-  echo "Running make for OpenSSL..."
-  make -j$cpucount -s
-  echo "Running make install for OpenSSL..."
-  make install > /dev/null
-  cd ~
-
-  # Compile latest Python
-  echo "Downloading Python $BUILD_PYTHON_VERSION..."
-  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
-  echo "Extracting Python..."
-  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
-  cd Python-$BUILD_PYTHON_VERSION
-  echo "Compiling Python $BUILD_PYTHON_VERSION..."
-  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
-  unsafe_flags="--enable-optimizations --with-lto"  
-  echo "running configure with safe and unsafe"
-  ./configure $safe_flags $unsafe_flags > /dev/null
-  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
-  echo "Installing Python..."
-  make install > /dev/null
-fi
-
-python=~/python/bin/python3
-pip=~/python/bin/pip3
-
-$python -V
-
-cd $whereibelong
-
-echo "Upgrading pip packages..."
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
-
-mkdir ~/.ruby
-export GEM_HOME=~/.ruby
-export PATH=$PATH:~/.ruby/bin

src/travis/linux-arm64-install.sh

@@ -1,33 +0,0 @@
-cd src
-if [ "$VMTYPE" == "test" ]; then
-  export gam="$python gam.py"
-  export gampath=$(readlink -e .)
-else
-  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
-  export gam="gam/gam"
-  export gampath=$(readlink -e gam)
-  export GAMVERSION=`$gam version simple`
-  cp LICENSE $gampath
-  cp whatsnew.txt $gampath
-  cp GamCommands.txt $gampath
-  this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
-  GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
-  rm $gampath/lastupdatecheck.txt
-  tar cfJ $GAM_ARCHIVE gam/
-  echo "PyInstaller GAM info:"
-  du -h gam/gam
-  time $gam version extended
-
-  if [[ "$dist" == "precise" ]]; then
-    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
-    $python -OO -m staticx gam/gam gam/gam-staticx
-    strip gam/gam-staticx
-    rm gam/gam
-    mv gam/gam-staticx gam/gam
-    tar cfJ $GAM_LEGACY_ARCHIVE gam/
-    echo "Legacy StaticX GAM info:"
-    du -h gam/gam
-    time $gam version extended
-  fi
-
-fi

src/travis/linux-before-install.sh

@@ -1,12 +1,11 @@
-if [ "$VMTYPE" == "test" ]; then
+if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
   export python="python"
   export pip="pip"
   echo "Travis setup Python $TRAVIS_PYTHON_VERSION"
   echo "running tests with this version"
 else
   export whereibelong=$(pwd)
-  export dist=$(lsb_release --codename --short)
-  echo "We are running on Ubuntu $dist"
+  echo "We are running on Ubuntu $TRAVIS_DIST $PLATFORM"
   export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   cpucount=$(nproc --all)
   echo "This device has $cpucount CPUs for compiling..."
@@ -42,7 +41,7 @@ else
     echo "Installing deps for python3"
     sudo cp -v /etc/apt/sources.list /tmp
     sudo chmod a+rwx /tmp/sources.list
-    echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+    echo "deb-src http://archive.ubuntu.com/ubuntu/ $TRAVIS_DIST main" >> /tmp/sources.list
     sudo cp -v /tmp/sources.list /etc/apt
     sudo apt-get -qq --yes update > /dev/null
     sudo apt-get -qq --yes build-dep python3 > /dev/null
@@ -91,9 +90,8 @@ else
   python=~/python/bin/python3
   pip=~/python/bin/pip3
 
-  if [[ "$dist" == "precise" ]]; then
+  if ([ "${TRAVIS_DIST}" == "trusty" ] || [ "${TRAVIS_DIST}" == "xenial" ]) && [ "${PLATFORM}" == "x86_64" ]; then
     echo "Installing deps for StaticX..."
-    sudo apt-get install --yes scons
     if [ ! -d patchelf-$PATCHELF_VERSION ]; then
       echo "Downloading PatchELF $PATCHELF_VERSION"
       wget https://nixos.org/releases/patchelf/patchelf-$PATCHELF_VERSION/patchelf-$PATCHELF_VERSION.tar.bz2
@@ -103,7 +101,7 @@ else
       make
       sudo make install
     fi
-    $pip install git+https://github.com/JonathonReinhart/staticx.git@master
+    $pip install staticx
   fi
 
   cd $whereibelong

src/travis/linux-install.sh

@@ -1,5 +1,5 @@
 cd src
-if [ "$VMTYPE" == "test" ]; then
+if [[ "$TRAVIS_JOB_NAME" == *"Testing" ]]; then
   export gam="$python gam.py"
   export gampath=$(readlink -e .)
 else
@@ -18,16 +18,18 @@ else
   du -h gam/gam
   time $gam version extended
 
-  if [[ "$dist" == "precise" ]]; then
-    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
-    $python -OO -m staticx gam/gam gam/gam-staticx
+  if ([ "${TRAVIS_DIST}" == "trusty" ] || [ "${TRAVIS_DIST}" == "xenial" ]) && [ "${PLATFORM}" == "x86_64" ]; then
+    GAM_LEGACY_ARCHIVE=gam-${GAMVERSION}-${GAMOS}-${PLATFORM}-legacy.tar.xz
+    $python -OO -m staticx -l /lib/x86_64-linux-gnu/libresolv.so.2 -l /lib/x86_64-linux-gnu/libnss_dns.so.2 gam/gam gam/gam-staticx
     strip gam/gam-staticx
     rm gam/gam
     mv gam/gam-staticx gam/gam
     chmod 755 gam/gam
-    tar cfJ $GAM_LEGACY_ARCHIVE gam/
+    tar cvfJ $GAM_LEGACY_ARCHIVE gam/
     echo "Legacy StaticX GAM info:"
     du -h gam/gam
     time $gam version extended
   fi
+  echo "GAM packages:"
+  ls -l gam-*.tar.xz
 fi

src/travis/svars-write.py

@@ -6,8 +6,8 @@ cfg = json.load(sys.stdin)
 cfg['client_secret'] = os.getenv('client_secret')
 jid = os.getenv('jid')
 cfg['refresh_token'] = os.getenv('refresh_%s' % jid)
-vmtype = os.getenv('VMTYPE')
-if vmtype == 'test':
+name = os.getenv('TRAVIS_JOB_NAME')
+if name.endswith('Testing'):
     out_file = 'oauth2.txt'
 else:
     out_file = 'gam/oauth2.txt'

src/travis/windows-before-install.sh

@@ -0,0 +1,81 @@
+if [[ "$PLATFORM" == "x86_64" ]]; then
+  export BITS="64"
+  export PYTHONFILE_BITS="-amd64"
+  export OPENSSL_BITS="-x64"
+  export WIX_BITS="x64"
+elif [[ "$PLATFORM" == "x86" ]]; then
+  export BITS="32"
+  export PYTHONFILE_BITS=""
+  export OPENSSL_BITS=""
+  export WIX_BITS="x86"
+fi
+echo "This is a ${BITS}-bit build for ${PLATFORM}"
+
+export mypath=$(pwd)
+cd ~
+
+# .NET Core
+echo "Installing Net-Framework-Core..."
+until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying .net again..."; done
+
+# VS 2015
+echo "Installing Visual Studio 2015.."
+until choco install vcbuildtools; do echo "Trying Visual Studio again..."; done
+
+# Python
+echo "Installing Python..."
+export python_file=python-${BUILD_PYTHON_VERSION}${PYTHONFILE_BITS}.exe
+if [ ! -e $python_file ]; then
+  echo "Downloading $python_file..."
+  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
+fi
+until powershell ".\\${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python"; do echo "trying python again..."; done
+export python=/c/python/python.exe
+export pip=/c/python/scripts/pip.exe
+until [ -f $python ]; do sleep 1; done
+export PATH=$PATH:/c/python/scripts
+
+# OpenSSL
+echo "Installing OpenSSL..."
+export exefile=Win${BITS}OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
+if [ ! -e $exefile ]; then
+  echo "Downloading $exefile..."
+  wget --quiet https://slproweb.com/download/$exefile
+fi
+until powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"; do echo "trying openssl again..."; done
+until cp -v /c/ssl/libcrypto-1_1${OPENSSL_BITS}.dll /c/python/DLLs/; do echo "trying libcrypto copy again..."; sleep 3; done
+until cp -v /c/ssl/libssl-1_1${OPENSSL_BITS}.dll /c/python/DLLs/; do echo "trying libssl copy again..."; done
+if [[ "$PLATFORM" == "x86_64" ]]; then
+  cp -v /c/python/DLLs/libssl-1_1-x64.dll /c/python/DLLs/libssl-1_1.dll
+  cp -v /c/python/DLLs/libcrypto-1_1-x64.dll /c/python/DLLs/libcrypto-1_1.dll
+fi
+
+# WIX Toolset
+until cinst -y wixtoolset; do echo "trying wix install again..."; done
+
+cd $mypath
+
+$pip install --upgrade pip
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+#$pip install --upgrade pyinstaller
+# Install PyInstaller from source and build bootloader
+# to try and avoid getting flagged as malware since
+# lots of malware uses PyInstaller default bootloader
+# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
+echo "Downloading PyInstaller..."
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+tar xf develop.tar.gz
+cd pyinstaller-develop/bootloader
+echo "bootloader before:"
+md5sum ../PyInstaller/bootloader/Windows-${BITS}bit/*
+
+$python ./waf all --target-arch=${BITS}bit --msvc_version "msvc 14.0"
+
+echo "bootloader after:"
+md5sum ../PyInstaller/bootloader/Windows-${BITS}bit/*
+echo "PATH: $PATH"
+cd ..
+$python setup.py install
+echo "cd to $mypath"
+cd $mypath

src/travis/windows-install.sh

@@ -15,6 +15,6 @@ GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
 /c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
 mkdir gam-64
 cp -rf gam/* gam-64/;
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch x64 gam.wxs
+/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch $WIX_BITS gam.wxs
 /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
 rm *.wixpdb

src/travis/windows-x86-before-install.sh

@@ -1,46 +0,0 @@
-echo "Installing Net-Framework-Core..."
-export mypath=$(pwd)
-until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~
-export exefile=Win32OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
-if [ ! -e $exefile ]; then
-  echo "Downloading $exefile..."
-  wget --quiet https://slproweb.com/download/$exefile
-fi
-echo "Installing $exefile..."
-powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
-export python_file=python-$BUILD_PYTHON_VERSION.exe
-wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
-powershell ".\\${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python"
-until cinst -y wixtoolset; do echo "trying again..."; done
-until cp -v /c/ssl/libcrypto-1_1.dll /c/python/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
-until cp -v /c/ssl/libssl-1_1.dll /c/python/DLLs/libssl-1_1.dll; do echo "trying again..."; done
-export PATH=$PATH:/c/python/scripts
-cd $mypath
-export python=/c/python/python.exe
-export pip=/c/python/scripts/pip.exe
-until [ -f $python ]; do :; done
-until [ -f $pip ]; do :; done
-
-$pip install --upgrade pip
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-#$pip install --upgrade pyinstaller
-# Install PyInstaller from source and build bootloader
-# to try and avoid getting flagged as malware since
-# lots of malware uses PyInstaller default bootloader
-# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
-echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
-tar xf develop.tar.gz
-cd pyinstaller-develop/bootloader
-echo "bootloader before:"
-md5sum ../PyInstaller/bootloader/Windows-32bit/*
-$python ./waf all --target-arch=32bit
-echo "bootloader after:"
-md5sum ../PyInstaller/bootloader/Windows-32bit/*
-echo "PATH: $PATH"
-cd ..
-$python setup.py install
-echo "cd to $mypath"
-cd $mypath

src/travis/windows-x86-install.sh

@@ -1,18 +0,0 @@
-cd src
-pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
-export gam="gam/gam"
-export gampath=$(readlink -e gam)
-$gam version extended
-export GAMVERSION=`$gam version simple`
-rm gam/lastupdatecheck.txt
-cp LICENSE gam
-cp GamCommands.txt gam
-cp whatsnew.txt gam
-cp gam-setup.bat gam
-GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM.zip
-/c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam -xr!.svn
-mkdir gam-64
-cp -rf gam/* gam-64/;
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/candle.exe -arch x86 gam.wxs
-/c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.11/bin/WixUIExtension.dll gam.wixobj -o gam-$GAMVERSION-$GAMOS-$PLATFORM.msi || true;
-rm *.wixpdb

src/travis/windows-x86_64-before-install.sh

@@ -1,48 +0,0 @@
-echo "Installing Net-Framework-Core..."
-export mypath=$(pwd)
-until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~
-export exefile=Win64OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
-if [ ! -e $exefile ]; then
-  echo "Downloading $exefile..."
-  wget --quiet https://slproweb.com/download/$exefile
-fi
-echo "Installing $exefile..."
-powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
-export python_file=python-$BUILD_PYTHON_VERSION-amd64.exe
-wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/$python_file
-powershell ".\\${python_file} /quiet InstallAllUsers=1 TargetDir=c:\\python"
-until cinst -y wixtoolset; do echo "trying again..."; done
-until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/python/DLLs/; do echo "trying libcrypto copy again..."; done
-cp -v /c/python/DLLs/libcrypto-1_1-x64.dll /c/python/DLLs/libcrypto-1_1.dll
-until cp -v /c/ssl/libssl-1_1-x64.dll /c/python/DLLs/; do echo "trying libssl copy again..."; done
-cp -v /c/python/DLLs/libssl-1_1-x64.dll /c/python/DLLs/libssl-1_1.dll
-export PATH=$PATH:/c/python/scripts
-cd $mypath
-export python=/c/python/python.exe
-export pip=/c/python/scripts/pip.exe
-until [ -f $python ]; do :; done
-until [ -f $pip ]; do :; done
-
-$pip install --upgrade pip
-$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
-$pip install --upgrade -r src/requirements.txt
-#$pip install --upgrade pyinstaller
-# Install PyInstaller from source and build bootloader
-# to try and avoid getting flagged as malware since
-# lots of malware uses PyInstaller default bootloader
-# https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
-echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
-tar xf develop.tar.gz
-cd pyinstaller-develop/bootloader
-echo "bootloader before:"
-md5sum ../PyInstaller/bootloader/Windows-64bit/*
-$python ./waf all --target-arch=64bit
-echo "bootloader after:"
-md5sum ../PyInstaller/bootloader/Windows-64bit/*
-echo "PATH: $PATH"
-cd ..
-$python setup.py install
-echo "cd to $mypath"
-cd $mypath

src/utils.py

@@ -1,14 +1,17 @@
 import datetime
 import re
 import sys
+import time
 from hashlib import md5
 from html.entities import name2codepoint
 from html.parser import HTMLParser
 import json
+import dateutil.parser
 
-from var import *
+import controlflow
 import fileutils
 import transport
+from var import *
 
 class _DeHTMLParser(HTMLParser):
 
@@ -111,6 +114,15 @@ def formatMilliSeconds(millis):
   hours, minutes = divmod(minutes, 60)
   return f'{hours:02d}:{minutes:02d}:{seconds:02d}'
 
+def integerLimits(minVal, maxVal, item='integer'):
+  if (minVal is not None) and (maxVal is not None):
+    return f'{item} {minVal}<=x<={maxVal}'
+  if minVal is not None:
+    return f'{item} x>={minVal}'
+  if maxVal is not None:
+    return f'{item} x<={maxVal}'
+  return f'{item} x'
+
 def get_string(i, item, optional=False, minLen=1, maxLen=None):
   if i < len(sys.argv):
     argstr = sys.argv[i]
@@ -163,7 +175,7 @@ def get_yyyymmdd(argstr, minLen=1, returnTimeStamp=False, returnDateTime=False):
   if argstr:
     if argstr[0] in ['+', '-']:
       today = datetime.date.today()
-      argstr = (datetime.datetime(today.year, today.month, today.day)+getDeltaDate(argstr)).strftime(YYYYMMDD_FORMAT)
+      argstr = (datetime.datetime(today.year, today.month, today.day)+get_delta_date(argstr)).strftime(YYYYMMDD_FORMAT)
     try:
       dateTime = datetime.datetime.strptime(argstr, YYYYMMDD_FORMAT)
       if returnTimeStamp:
@@ -256,7 +268,7 @@ def md5_matches_file(local_file, expected_md5, exitOnError):
 
 URL_SHORTENER_ENDPOINT = 'https://gam-shortn.appspot.com/create'
 
-def shorten_url(long_url, httpc=None):    
+def shorten_url(long_url, httpc=None):
     if not httpc:
         httpc = transport.create_http(timeout=10)
     headers = {'Content-Type': 'application/json', 'User-Agent': GAM_INFO}

src/var.py

@@ -6,13 +6,13 @@ import platform
 import re
 
 gam_author = 'Jay Lee <jay0lee@gmail.com>'
-gam_version = '5.02'
+gam_version = '5.05'
 gam_license = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
 GAM_INFO = (f'GAM {gam_version} - {GAM_URL} / {gam_author} / '
-           f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
-           f'{platform.platform()} {platform.machine()}')
+            f'Python {platform.python_version()} {sys.version_info.releaselevel} / '
+            f'{platform.platform()} {platform.machine()}')
 
 GAM_RELEASES = 'https://github.com/jay0lee/GAM/releases'
 GAM_WIKI = 'https://github.com/jay0lee/GAM/wiki'
@@ -725,6 +725,8 @@ GROUP_SETTINGS_BOOLEAN_ATTRIBUTES = set([
 GM_SYSEXITRC = 'sxrc'
 # Path to gam
 GM_GAM_PATH = 'gpth'
+# Python source, PyInstaller or StaticX?
+GM_GAM_TYPE = 'gtyp'
 # Are we on Windows?
 GM_WINDOWS = 'wndo'
 # Encodings
@@ -768,6 +770,7 @@ _FN_OAUTH2_TXT = 'oauth2.txt'
 GM_Globals = {
   GM_SYSEXITRC: 0,
   GM_GAM_PATH: None,
+  GM_GAM_TYPE: None,
   GM_WINDOWS: os.name == 'nt',
   GM_SYS_ENCODING: _DEFAULT_CHARSET,
   GM_EXTRA_ARGS_DICT:  {'prettyPrint': False},
@@ -840,6 +843,8 @@ GC_SHOW_GETTINGS = 'show_gettings'
 GC_SITE_DIR = 'site_dir'
 # CSV Columns GAM should show on CSV output
 GC_CSV_HEADER_FILTER = 'csv_header_filter'
+# CSV Columns GAM should not show on CSV output
+GC_CSV_HEADER_DROP_FILTER = 'csv_header_drop_filter'
 # CSV Rows GAM should filter
 GC_CSV_ROW_FILTER = 'csv_row_filter'
 # Minimum TLS Version required for HTTPS connections
@@ -875,6 +880,7 @@ GC_Defaults = {
   GC_SHOW_GETTINGS: True,
   GC_SITE_DIR: '',
   GC_CSV_HEADER_FILTER: '',
+  GC_CSV_HEADER_DROP_FILTER: '',
   GC_CSV_ROW_FILTER: '',
   GC_TLS_MIN_VERSION: tls_min,
   GC_TLS_MAX_VERSION: None,
@@ -922,6 +928,7 @@ GC_VAR_INFO = {
   GC_SHOW_GETTINGS: {GC_VAR_TYPE: GC_TYPE_BOOLEAN},
   GC_SITE_DIR: {GC_VAR_TYPE: GC_TYPE_DIRECTORY},
   GC_CSV_HEADER_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
+  GC_CSV_HEADER_DROP_FILTER: {GC_VAR_TYPE: GC_TYPE_HEADERFILTER},
   GC_CSV_ROW_FILTER: {GC_VAR_TYPE: GC_TYPE_ROWFILTER},
   GC_TLS_MIN_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},
   GC_TLS_MAX_VERSION: {GC_VAR_TYPE: GC_TYPE_STRING},