Update linux-arm64-before-install.sh

* Fix multiprocessing imports

* Cleanup multiprocessing imports

* Check for 32-bit GAM on 64-bit Windows

.github/workflows/pythonpackage.yml

@@ -22,3 +22,9 @@ jobs:
     - name: Run GAM
       run: |
         python src/gam.py version extended
+    - name: Get Env
+      run: |
+        set
+    - name: Test Install
+      run: |
+        if [[ $(uname -s) == "Linux" || $(uname -s) == "Darwin" ]]; then bash <(curl -s -S -L https://git.io/install-gam) -l; fi

.travis.yml

@@ -2,7 +2,7 @@ if: tag IS blank
 
 env:
   global:
-    - BUILD_PYTHON_VERSION=3.7.4
+    - BUILD_PYTHON_VERSION=3.8.0
     - BUILD_OPENSSL_VERSION=1.1.1d
     - PATCHELF_VERSION=0.9
     - PYINSTALLER_VERSION=3.5
@@ -32,9 +32,11 @@ cache:
     - $HOME/.cache/pip
     - $HOME/python
     - $HOME/ssl
-    - $HOME/pybuild
 
 matrix:
+  fast_finish: true
+  allow_failures:
+    - arch: arm64
   include:
     - os: linux
       name: "Linux 64-bit Bionic"
@@ -52,6 +54,34 @@ matrix:
         - GAMOS=linux
         - PLATFORM=x86_64
         - VMTYPE=build
+    - os: linux
+      dist: bionic
+      arch: arm64
+      name: "Linux ARM64 Bionic"
+      language: bash
+      filter_secrets: false
+      addons:
+        apt:
+          packages:
+            - ruby
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
+    - os: linux
+      dist: xenial
+      arch: arm64
+      name: "Linux ARM64 Xenial"
+      language: bash
+      filter_secrets: false
+      addons:
+        apt:
+          packages:
+            - ruby
+      env:
+        - GAMOS=linux
+        - PLATFORM=arm64
+        - VMTYPE=build
     - os: linux
       name: "Linux 64-bit Trusty"
       dist: trusty
@@ -89,11 +119,11 @@ matrix:
         - PLATFORM=x86_64
         - VMTYPE=test
     - os: linux
-      name: "Linux 64-bit - Python 3.8-dev Source Testing"
+      name: "Linux 64-bit - Python 3.7 Source Testing"
       dist: bionic
       language: python
       python:
-        - "3.8-dev"
+        - "3.7"
       env:
         - GAMOS=linux
         - PLATFORM=x86_64
@@ -127,7 +157,7 @@ matrix:
     - os: osx
       name: "MacOS 10.14"
       language: generic
-      osx_image: xcode11
+      osx_image: xcode11.1
       env:
         - GAMOS=macos
         - PLATFORM=x86_64
@@ -149,6 +179,9 @@ matrix:
         - PLATFORM=x86
         - VMTYPE=build
 
+before_cache:
+- echo "this is before cache"
+
 before_install:
 - source src/travis/$TRAVIS_OS_NAME-$PLATFORM-before-install.sh
 
@@ -246,6 +279,7 @@ script:
 - if [ "$e2e" = true ]; then $gam print users query "travis.jid=$jid" | $gam csv - gam delete user ~primaryEmail; fi
 - if [ "$e2e" = true ]; then $gam print mobile; fi
 - if [ "$e2e" = true ]; then $gam print cros allfields nolists; fi
+- if [ "$TRAVIS_OS_NAME" != "windows" ]; then bash <(curl -s -S -L https://git.io/install-gam) -l; fi
 
 before_deploy:
 - export TRAVIS_TAG="preview"

src/GamCommands.txt

@@ -140,6 +140,7 @@ If an item contains spaces, it should be surrounded by ".
 
 <AccessToken> ::= <String>
 <ACLScope> ::= [user:]<EmailAddress>|group:<EmailAddress>|domain[:<DomainName>]|default
+<APIScopeURL> ::= <String>
 <ASPID> ::= <String>
 <BuildingID> ::= <String>|id:<String>
 <CalendarACLRole> ::= editor|freebusy|freebusyreader|owner|reader|writer
@@ -552,6 +553,7 @@ Items, separated by spaces, with spaces, commas or single quotes in the items th
     "'it em' 'it,em' \"it'em\""
 
 <ACLList> ::= "<ACLScope>(,<ACLScope>)*"
+<APIScopeURLList> ::= "<APIScopeURL>(,<APIScopeURL>)*"
 <ASPIDList> ::= "<ASPID>(,<ASPID>)*"
 <CalendarList> ::= "<CalendarItem>(,<CalendarItem>)*"
 <ChatRoomList> ::= "<ChatRoom>(,<ChatRoom>)*"
@@ -839,7 +841,7 @@ gam oauth|oauth2 delete|revoke
 gam oauth|oauth2 info|verify [accesstoken <AccessToken>] [idtoken <IDToken>] [showsecret]
 gam oauth|oauth2 refresh
 
-gam <UserTypeEntity> check serviceaccount
+gam <UserTypeEntity> check serviceaccount [scope|scopes <APIScopeURLList>]
 
 gam whatis <EmailItem>
 

src/gam-install.sh

@@ -10,6 +10,7 @@ OPTIONS:
    -d      Directory where gam folder will be installed. Default is \$HOME/bin/
    -a      Architecture to install (i386, x86_64, x86_64_legacy, arm, arm64). Default is to detect your arch with "uname -m".
    -o      OS we are running (linux, macos). Default is to detect your OS with "uname -s".
+   -b      OS version. Default is to detect on MacOS and Linux.
    -l      Just upgrade GAM to latest version. Skips project creation and auth.
    -p      Profile update (true, false). Should script add gam command to environment. Default is true.
    -u      Admin user email address to use with GAM. Default is to prompt.
@@ -21,20 +22,23 @@ EOF
 target_dir="$HOME/bin"
 gamarch=$(uname -m)
 gamos=$(uname -s)
+osversion=""
 update_profile=true
 upgrade_only=false
 gamversion="latest"
 adminuser=""
 regularuser=""
-gam_glibc_vers="2.23 2.19 2.15"
+gam_glibc_vers="2.27 2.23 2.19 2.15"
+gam_macos_vers="10.14.4 10.13.6 10.12.6"
 
-while getopts "hd:a:o:lp:u:r:v:" OPTION
+while getopts "hd:a:o:b:lp:u:r:v:" OPTION
 do
      case $OPTION in
          h) usage; exit;;
          d) target_dir="$OPTARG";;
          a) gamarch="$OPTARG";;
          o) gamos="$OPTARG";;
+         b) osversion="$OPTARG";;
          l) upgrade_only=true;;
          p) update_profile="$OPTARG";;
          u) adminuser="$OPTARG";;
@@ -79,13 +83,26 @@ echo -e '\x1B[0m'
 
 version_gt()
 {
-test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+# MacOS < 10.13 doesn't support sort -V
+echo "" | sort -V > /dev/null 2>&1
+vsort_failed=$?
+if [ "${1}" = "${2}" ]; then
+  true
+elif (( $vsort_failed != 0 )); then
+  false
+else
+  test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"
+fi
 }
 
 case $gamos in
   [lL]inux)
     gamos="linux"
-    this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    if [ "$osversion" == "" ]; then
+      this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+    else
+      this_glibc_ver=$osversion
+    fi
     echo "This Linux distribution uses glibc $this_glibc_ver"
     useglibc="legacy"
     for gam_glibc_ver in $gam_glibc_vers; do
@@ -97,24 +114,33 @@ case $gamos in
     done
     case $gamarch in
       x86_64) gamfile="linux-x86_64-$useglibc.tar.xz";;
-      i?86) gamfile="linux-i686.tar.xz";;
-      arm|armv7l) gamfile="linux-armv7l.tar.xz";;
-      arm64|aarch64) gamfile="linux-aarch64.tar.xz";;
+      arm64|aarch64) gamfile="linux-arm64-$useglibc.tar.xz";;
       *)
-        echo_red "ERROR: this installer currently only supports i386, x86_64, arm and arm64 Linux. Looks like you're running on $gamarch. Exiting."
+        echo_red "ERROR: this installer currently only supports x86_64 and arm64 Linux. Looks like you're running on $gamarch. Exiting."
         exit
     esac
     ;;
   [Mm]ac[Oo][sS]|[Dd]arwin)
-    osver=$(sw_vers -productVersion | awk -F'.' '{print $2}')
-    if (( $osver < 13 )); then
-      echo_red "ERROR: GAM currently requires MacOS 10.13 or newer. You are running MacOS 10.$osver. Please upgrade." 
-      exit
-    else
-      echo_green "Good, you're running MacOS 10.$osver..."
-    fi
     gamos="macos"
-    gamfile="macos-x86_64.tar.xz"
+    if [ "$osversion" == "" ]; then
+      this_macos_ver=$(sw_vers -productVersion)
+    else
+      this_macos_ver=$osversion
+    fi
+    echo "You are running MacOS $this_macos_ver"
+    use_macos_ver=""
+    for gam_macos_ver in $gam_macos_vers; do
+      if version_gt $this_macos_ver $gam_macos_ver; then
+        use_macos_ver="MacOS$gam_macos_ver"
+        echo_green "Using GAM compiled on $use_macos_ver"
+        break
+      fi
+    done
+    if [ "$use_macos_ver" == "" ]; then
+      echo_red "Sorry, you need to be running at least MacOS $gam_macos_ver to run GAM"
+      exit
+    fi
+    gamfile="macos-x86_64-$use_macos_ver.tar.xz"
     ;;
   *)
     echo_red "Sorry, this installer currently only supports Linux and MacOS. Looks like you're runnning on $gamos. Exiting."
@@ -161,11 +187,16 @@ try:
 except KeyError:
   print('ERROR: assets value not found in JSON value of:\n\n%s' % release)"
 
-pycmd="python"
+pycmd="python3"
 $pycmd -V >/dev/null 2>&1
 rc=$?
 if (( $rc != 0 )); then
-  pycmd="python3"
+  pycmd="python"
+fi
+$pycmd -V >/dev/null 2>&1
+rc=$?
+if (( $rc != 0 )); then
+  pycmd="python2"
 fi
 $pycmd -V >/dev/null 2>&1
 rc=$?
@@ -205,7 +236,7 @@ fi
 
 if [ "$upgrade_only" = true ]; then
   echo_green "Here's information about your GAM upgrade:"
-  "$target_dir/gam/gam" version
+  "$target_dir/gam/gam" version extended
   rc=$?
   if (( $rc != 0 )); then
     echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."
@@ -328,7 +359,7 @@ while $project_created; do
 done
 
 echo_green "Here's information about your new GAM installation:"
-"$target_dir/gam/gam" version
+"$target_dir/gam/gam" version extended
 rc=$?
 if (( $rc != 0 )); then
   echo_red "ERROR: Failed running GAM for the first time with $rc. Please report this error to GAM mailing list. Exiting."

src/gam.py

@@ -52,10 +52,16 @@ import uuid
 import webbrowser
 import zipfile
 import http.client as http_client
-from multiprocessing import Pool
-from multiprocessing import freeze_support
+from multiprocessing import Pool as mp_pool
+from multiprocessing import freeze_support as mp_freeze_support
+from multiprocessing import set_start_method as mp_set_start_method
 from urllib.parse import urlencode, urlparse
+# workaround https://bitbucket.org/ecollins/passlib/issues/107/timeclock-has-gone
+# can be removed with passlib > 1.7.1
+if sys.platform == 'win32' and sys.version_info[1] >= 8:
+  time.clock = time.time
 from passlib.hash import sha512_crypt
+time.__dict__.pop('clock', None)
 import dateutil.parser
 
 import googleapiclient
@@ -614,46 +620,61 @@ def SetGlobalVariables():
     rowFilters = {}
     if not value:
       return rowFilters
-    try:
-      for column, filterStr in iter(json.loads(value.encode('unicode-escape').decode(UTF8)).items()):
-        mg = ROW_FILTER_COMP_PATTERN.match(filterStr)
-        if mg:
-          if mg.group(1) in ['date', 'time']:
-            if mg.group(1) == 'date':
-              valid, filterValue = getRowFilterDateOrDeltaFromNow(mg.group(3))
-            else:
-              valid, filterValue = getRowFilterTimeOrDeltaFromNow(mg.group(3))
-            if valid:
-              rowFilters[column] = (mg.group(1), mg.group(2), filterValue)
-              continue
-            systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Expected: {3}'.format(itemName, column, filterStr, filterValue))
-          else: #count
-            if mg.group(3).isdigit():
-              rowFilters[column] = (mg.group(1), mg.group(2), int(mg.group(3)))
-              continue
-            systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Expected: {3}'.format(itemName, column, filterStr, '<Number>'))
-        mg = ROW_FILTER_BOOL_PATTERN.match(filterStr)
-        if mg:
-          value = mg.group(2).lower()
-          if value in true_values:
-            filterValue = True
-          elif value in false_values:
-            filterValue = False
-          else:
-            systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Expected true|false'.format(itemName, column, filterStr))
-          rowFilters[column] = (mg.group(1), filterValue)
+    if value.startswith('{'):
+      try:
+        filterDict = json.loads(value.encode('unicode-escape').decode(UTF8))
+      except (TypeError, ValueError) as e:
+        systemErrorExit(3, 'Item: {0}, Value: "{1}", Failed to parse as JSON: {2}'.format(itemName, value, str(e)))
+    else:
+      filterDict = {}
+      status, filterList = shlexSplitListStatus(value)
+      if not status:
+        systemErrorExit(3, 'Item: {0}, Value: "{1}", Failed to parse as list'.format(itemName, value))
+      for filterVal in filterList:
+        if not filterVal:
           continue
-        mg = ROW_FILTER_RE_PATTERN.match(filterStr)
-        if mg:
-          try:
-            rowFilters[column] = (mg.group(1), re.compile(mg.group(2)))
+        try:
+          column, filterStr = filterVal.split(':', 1)
+        except ValueError:
+          systemErrorExit(3, 'Item: {0}, Value: "{1}", Expected column:filter'.format(itemName, filterVal))
+        filterDict[column] = filterStr
+    for column, filterStr in iter(filterDict.items()):
+      mg = ROW_FILTER_COMP_PATTERN.match(filterStr)
+      if mg:
+        if mg.group(1) in ['date', 'time']:
+          if mg.group(1) == 'date':
+            valid, filterValue = getRowFilterDateOrDeltaFromNow(mg.group(3))
+          else:
+            valid, filterValue = getRowFilterTimeOrDeltaFromNow(mg.group(3))
+          if valid:
+            rowFilters[column] = (mg.group(1), mg.group(2), filterValue)
             continue
-          except re.error as e:
-            systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Invalid RE: {3}'.format(itemName, column, filterStr, e))
-        systemErrorExit(3, 'Item: {0}, Value: "{1}": {2}, Expected: (date|time|count<Operator><Value>) or (boolean:true|false) or (regex|notregex:<RegularExpression>)'.format(itemName, column, filterStr))
-      return rowFilters
-    except (TypeError, ValueError) as e:
-      systemErrorExit(3, 'Item: {0}, Value: "{1}", Failed to parse as JSON: {2}'.format(itemName, value, str(e)))
+          systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Expected: {3}'.format(itemName, column, filterStr, filterValue))
+        else: #count
+          if mg.group(3).isdigit():
+            rowFilters[column] = (mg.group(1), mg.group(2), int(mg.group(3)))
+            continue
+          systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Expected: {3}'.format(itemName, column, filterStr, '<Number>'))
+      mg = ROW_FILTER_BOOL_PATTERN.match(filterStr)
+      if mg:
+        value = mg.group(2).lower()
+        if value in true_values:
+          filterValue = True
+        elif value in false_values:
+          filterValue = False
+        else:
+          systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Expected true|false'.format(itemName, column, filterStr))
+        rowFilters[column] = (mg.group(1), filterValue)
+        continue
+      mg = ROW_FILTER_RE_PATTERN.match(filterStr)
+      if mg:
+        try:
+          rowFilters[column] = (mg.group(1), re.compile(mg.group(2)))
+          continue
+        except re.error as e:
+          systemErrorExit(3, 'Item: {0}, Value: "{1}": "{2}", Invalid RE: {3}'.format(itemName, column, filterStr, e))
+      systemErrorExit(3, 'Item: {0}, Value: "{1}": {2}, Expected: (date|time|count<Operator><Value>) or (boolean:true|false) or (regex|notregex:<RegularExpression>)'.format(itemName, column, filterStr))
+    return rowFilters
 
   GC_Defaults[GC_CONFIG_DIR] = GM_Globals[GM_GAM_PATH]
   GC_Defaults[GC_CACHE_DIR] = os.path.join(GM_Globals[GM_GAM_PATH], 'gamcache')
@@ -850,6 +871,8 @@ def doGAMVersion(checkForArgs=True):
                             sys.version_info[1], sys.version_info[2], struct.calcsize('P')*8,
                             sys.version_info[3], googleapiclient.__version__,
                             getOSPlatform(), platform.machine(), GM_Globals[GM_GAM_PATH]))
+  if sys.platform.startswith('win') and str(struct.calcsize('P')*8).find('32') != -1 and platform.machine().find('64') != -1:
+    print(MESSAGE_UPDATE_GAM_TO_64BIT)
   if timeOffset:
     offset, nicetime = getLocalGoogleTimeOffset(testLocation)
     print(MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY % nicetime)
@@ -1546,6 +1569,15 @@ def printPassFail(description, result):
   print(' {0:74} {1}'.format(description, result))
 
 def doCheckServiceAccount(users):
+  i = 5
+  check_scopes = []
+  while i < len(sys.argv):
+    myarg = sys.argv[i].lower()
+    if myarg in ['scope', 'scopes']:
+      check_scopes = sys.argv[i+1].replace(',', ' ').split()
+      i += 2
+    else:
+      systemErrorExit(3, '%s is not a valid argument for "gam user <email> check serviceaccount"' % myarg)
   something_failed = False
   print('Computer clock status:')
   timeOffset, nicetime = getLocalGoogleTimeOffset()
@@ -1574,18 +1606,18 @@ def doCheckServiceAccount(users):
     something_failed = True
     auth_error = str(e.args[0])
   printPassFail('Authenticating...%s' % auth_error, sa_token_result)
-  all_scopes = []
-  for _, scopes in list(API_SCOPE_MAPPING.items()):
-    for scope in scopes:
-      if scope not in all_scopes:
-        all_scopes.append(scope)
-  all_scopes.sort()
+  if not check_scopes:
+    for _, scopes in list(API_SCOPE_MAPPING.items()):
+      for scope in scopes:
+        if scope not in check_scopes:
+          check_scopes.append(scope)
+  check_scopes.sort()
   for user in users:
     user = user.lower()
     all_scopes_pass = True
     oa2 = googleapiclient.discovery.build('oauth2', 'v1', _createHttpObj())
     print('Domain-Wide Delegation authentication as %s:' % (user))
-    for scope in all_scopes:
+    for scope in check_scopes:
       # try with and without email scope
       for scopes in [[scope, USERINFO_EMAIL_SCOPE], [scope]]:
         try:
@@ -1614,7 +1646,7 @@ def doCheckServiceAccount(users):
       return
     user_domain = user[user.find('@')+1:]
     # Tack on email scope for more accurate checking
-    all_scopes.append(USERINFO_EMAIL_SCOPE)
+    check_scopes.append(USERINFO_EMAIL_SCOPE)
     scopes_failed = '''Some scopes failed! Please go to:
 
 https://admin.google.com/%s/AdminHome?#OGX:ManageOauthClients
@@ -1625,7 +1657,7 @@ and grant Client name:
 
 Access to scopes:
 
-%s\n''' % (user_domain, service_account, ',\n'.join(all_scopes))
+%s\n''' % (user_domain, service_account, ',\n'.join(check_scopes))
     systemErrorExit(1, scopes_failed)
 
 # Batch processing request_id fields
@@ -7769,7 +7801,7 @@ def _run_oauth_flow(client_id, client_secret, scopes, access_type, login_hint=No
       }
     }
 
-  flow = ShortURLFlow.from_client_config(client_config, scopes)
+  flow = ShortURLFlow.from_client_config(client_config, scopes, autogenerate_code_verifier=True)
   kwargs = {'access_type': access_type}
   if login_hint:
     kwargs['login_hint'] = login_hint
@@ -13122,6 +13154,15 @@ def shlexSplitList(entity, dataDelimiter=' ,'):
   lexer.whitespace_split = True
   return list(lexer)
 
+def shlexSplitListStatus(entity, dataDelimiter=' ,'):
+  lexer = shlex.shlex(entity, posix=True)
+  lexer.whitespace = dataDelimiter
+  lexer.whitespace_split = True
+  try:
+    return (True, list(lexer))
+  except ValueError as e:
+    return (False, str(e))
+
 def getQueries(myarg, argstr):
   if myarg == 'query':
     return [argstr]
@@ -14035,7 +14076,7 @@ def run_batch(items):
   if not items:
     return
   num_worker_threads = min(len(items), GC_Values[GC_NUM_THREADS])
-  pool = Pool(num_worker_threads, init_gam_worker)
+  pool = mp_pool(num_worker_threads, init_gam_worker)
   sys.stderr.write('Using %s processes...\n' % num_worker_threads)
   try:
     results = []
@@ -14044,7 +14085,7 @@ def run_batch(items):
         sys.stderr.write('commit-batch - waiting for running processes to finish before proceeding\n')
         pool.close()
         pool.join()
-        pool = Pool(num_worker_threads, init_gam_worker)
+        pool = mp_pool(num_worker_threads, init_gam_worker)
         sys.stderr.write('commit-batch - running processes finished, proceeding\n')
         continue
       results.append(pool.apply_async(ProcessGAMCommandMulti, [item]))
@@ -14937,8 +14978,12 @@ def ProcessGAMCommand(args):
 
 # Run from command line
 if __name__ == "__main__":
-  if sys.platform.startswith('win'):
-    freeze_support()
+  mp_freeze_support()
+  if sys.platform == 'darwin':
+    # https://bugs.python.org/issue33725 in Python 3.8.0 seems
+    # to break parallel operations with errors about extra -b
+    # command line arguments
+    mp_set_start_method('fork')
   if sys.version_info[0] < 3 or sys.version_info[1] < 5:
     systemErrorExit(5, 'GAM requires Python 3.5 or newer. You are running %s.%s.%s. Please upgrade your Python version or use one of the binary GAM downloads.' % sys.version_info[:3])
   sys.exit(ProcessGAMCommand(sys.argv))

src/requirements.txt

@@ -4,6 +4,6 @@ filelock
 google-api-python-client>=1.7.10
 google-auth
 google-auth-httplib2
-google-auth-oauthlib==0.4.0
+google-auth-oauthlib>=0.4.1
 httplib2>=0.13.0
 passlib

src/travis/linux-arm64-before-install.sh

@@ -0,0 +1,73 @@
+export whereibelong=$(pwd)
+export dist=$(lsb_release --codename --short)
+echo "We are running on Ubuntu $dist"
+export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
+cpucount=$(nproc --all)
+echo "This device has $cpucount CPUs for compiling..."
+sudo apt-get -qq --yes update > /dev/null
+sudo apt-get -qq --yes install xz-utils > /dev/null
+SSLVER=$(~/ssl/bin/openssl version)
+SSLRESULT=$?
+PYVER=$(~/python/bin/python3 -V)
+PYRESULT=$?
+if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $PYTHON_BUILD_VERSION"* ]]; then
+  echo "RUNNING: apt dist-upgrade..."
+  sudo apt-get -qq --yes dist-upgrade > /dev/null
+  echo "Installing build tools..."
+  sudo apt-get -qq --yes install build-essential
+
+  echo "Installing deps for python3"
+  sudo cp -v /etc/apt/sources.list /tmp
+  sudo chmod a+rwx /tmp/sources.list
+  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+  sudo cp -v /tmp/sources.list /etc/apt
+  sudo apt-get -qq --yes update > /dev/null
+  sudo apt-get -qq --yes build-dep python3 > /dev/null
+  sudo apt-get -qq --yes install zlib1g-dev > /dev/null
+  sudo apt-get -qq --yes install libffi-dev > /dev/null
+
+  # Compile latest OpenSSL
+  echo "Downloading OpenSSL..."
+  wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  echo "Extracting OpenSSL..."
+  tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
+  cd openssl-$BUILD_OPENSSL_VERSION
+  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+  ./config shared --prefix=$HOME/ssl
+  echo "Running make for OpenSSL..."
+  make -j$cpucount -s
+  echo "Running make install for OpenSSL..."
+  make install > /dev/null
+  cd ~
+
+  # Compile latest Python
+  echo "Downloading Python $BUILD_PYTHON_VERSION..."
+  curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
+  echo "Extracting Python..."
+  tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+  cd Python-$BUILD_PYTHON_VERSION
+  echo "Compiling Python $BUILD_PYTHON_VERSION..."
+  safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+  unsafe_flags="--enable-optimizations --with-lto"  
+  echo "running configure with safe and unsafe"
+  ./configure $safe_flags $unsafe_flags > /dev/null
+  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+  echo "Installing Python..."
+  make install > /dev/null
+fi
+
+python=~/python/bin/python3
+pip=~/python/bin/pip3
+
+$python -V
+
+cd $whereibelong
+
+echo "Upgrading pip packages..."
+$pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
+$pip install --upgrade -r src/requirements.txt
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+
+mkdir ~/.ruby
+export GEM_HOME=~/.ruby
+export PATH=$PATH:~/.ruby/bin

src/travis/linux-arm64-install.sh

@@ -0,0 +1,33 @@
+cd src
+if [ "$VMTYPE" == "test" ]; then
+  export gam="$python gam.py"
+  export gampath=$(readlink -e .)
+else
+  $python -OO -m PyInstaller --clean --noupx --strip -F --distpath=gam $GAMOS-gam.spec
+  export gam="gam/gam"
+  export gampath=$(readlink -e gam)
+  export GAMVERSION=`$gam version simple`
+  cp LICENSE $gampath
+  cp whatsnew.txt $gampath
+  cp GamCommands.txt $gampath
+  this_glibc_ver=$(ldd --version | awk '/ldd/{print $NF}')
+  GAM_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-glibc$this_glibc_ver.tar.xz
+  rm $gampath/lastupdatecheck.txt
+  tar cfJ $GAM_ARCHIVE gam/
+  echo "PyInstaller GAM info:"
+  du -h gam/gam
+  time $gam version extended
+
+  if [[ "$dist" == "precise" ]]; then
+    GAM_LEGACY_ARCHIVE=gam-$GAMVERSION-$GAMOS-$PLATFORM-legacy.tar.xz
+    $python -OO -m staticx gam/gam gam/gam-staticx
+    strip gam/gam-staticx
+    rm gam/gam
+    mv gam/gam-staticx gam/gam
+    tar cfJ $GAM_LEGACY_ARCHIVE gam/
+    echo "Legacy StaticX GAM info:"
+    du -h gam/gam
+    time $gam version extended
+  fi
+
+fi

src/travis/linux-x86_64-before-install.sh

@@ -7,79 +7,90 @@ else
   export whereibelong=$(pwd)
   export dist=$(lsb_release --codename --short)
   echo "We are running on Ubuntu $dist"
-  echo "RUNNING: apt update..."
-  sudo apt-get -qq --yes update > /dev/null
-  echo "RUNNING: apt dist-upgrade..."
-#  sudo apt-get -qq --yes dist-upgrade > /dev/null
-  echo "Installing build tools..."
-  sudo apt-get -qq --yes install build-essential
-
-  echo "Installing deps for python3"
-  sudo cp -v /etc/apt/sources.list /tmp
-  sudo chmod a+rwx /tmp/sources.list
-  echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
-  sudo cp -v /tmp/sources.list /etc/apt
-  sudo apt-get -qq --yes update > /dev/null
-  sudo apt-get -qq --yes build-dep python3 > /dev/null
-
-  mypath=$HOME
-  echo "My Path is $mypath"
+  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   cpucount=$(nproc --all)
   echo "This device has $cpucount CPUs for compiling..."
+  SSLVER=$(~/ssl/bin/openssl version)
+  SSLRESULT=$?
+  PYVER=$(~/python/bin/python3 -V)
+  PYRESULT=$?
+  if [ $SSLRESULT -ne 0 ] || [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]] || [ $PYRESULT -ne 0 ] || [[ "$PYVER" != "Python $PYTHON_BUILD_VERSION"* ]]; then
+    echo "SSL Result: $SSLRESULT - SSL Ver: $SSLVER - Py Result: $PYRESULT - Py Ver: $PYVER"
+    if [ $SSLRESULT -ne 0 ]; then
+      echo "sslresult -ne 0"
+    fi
+    if [[ "$SSLVER" != "OpenSSL $BUILD_OPENSSL_VERSION "* ]]; then
+      echo "sslver not equal to..."
+    fi
+    if [ $PYRESULT -ne 0 ]; then
+      echo "pyresult -ne 0"
+    fi
+    if [[ "$PYVER" != "Python $PYTHON_BUILD_VERSION" ]]; then
+      echo "pyver not equal to..."
+    fi
+    cd ~
+    rm -rf ssl
+    rm -rf python
+    mkdir ssl
+    mkdir python
+    echo "RUNNING: apt update..."
+    sudo apt-get -qq --yes update > /dev/null
+    echo "RUNNING: apt dist-upgrade..."
+    sudo apt-get -qq --yes dist-upgrade > /dev/null
+    echo "Installing build tools..."
+    sudo apt-get -qq --yes install build-essential
+    echo "Installing deps for python3"
+    sudo cp -v /etc/apt/sources.list /tmp
+    sudo chmod a+rwx /tmp/sources.list
+    echo "deb-src http://archive.ubuntu.com/ubuntu/ $dist main" >> /tmp/sources.list
+    sudo cp -v /tmp/sources.list /etc/apt
+    sudo apt-get -qq --yes update > /dev/null
+    sudo apt-get -qq --yes build-dep python3 > /dev/null
 
-  cd ~/pybuild
-  # Compile latest OpenSSL
-  if [ ! -d openssl-$BUILD_OPENSSL_VERSION ]; then
+    # Compile latest OpenSSL
     wget --quiet https://www.openssl.org/source/openssl-$BUILD_OPENSSL_VERSION.tar.gz
     echo "Extracting OpenSSL..."
     tar xf openssl-$BUILD_OPENSSL_VERSION.tar.gz
-  fi
-  cd openssl-$BUILD_OPENSSL_VERSION
-  echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
-  ./config shared --prefix=$mypath/ssl
-  echo "Running make for OpenSSL..."
-  make -j$cpucount -s
-  echo "Running make install for OpenSSL..."
-  make install > /dev/null
-  export LD_LIBRARY_PATH=~/ssl/lib
-  cd ~
+    cd openssl-$BUILD_OPENSSL_VERSION
+    echo "Compiling OpenSSL $BUILD_OPENSSL_VERSION..."
+    ./config shared --prefix=$HOME/ssl
+    echo "Running make for OpenSSL..."
+    make -j$cpucount -s
+    echo "Running make install for OpenSSL..."
+    make install > /dev/null
+    cd ~
 
-  cd ~/pybuild
-  # Compile latest Python
-  if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
+    # Compile latest Python
     echo "Downloading Python $BUILD_PYTHON_VERSION..."
     curl -O https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/Python-$BUILD_PYTHON_VERSION.tar.xz
     echo "Extracting Python..."
     tar xf Python-$BUILD_PYTHON_VERSION.tar.xz
+    cd Python-$BUILD_PYTHON_VERSION
+    echo "Compiling Python $BUILD_PYTHON_VERSION..."
+    safe_flags="--with-openssl=$HOME/ssl --enable-shared --prefix=$HOME/python --with-ensurepip=upgrade"
+    unsafe_flags="--enable-optimizations --with-lto"
+    if [ ! -e Makefile ]; then
+      echo "running configure with safe and unsafe"
+      ./configure $safe_flags $unsafe_flags > /dev/null
+    fi
+    make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
+    RESULT=$?
+    echo "First make exited with $RESULT"
+    if [ $RESULT != 0 ]; then
+      echo "Trying Python compile again without unsafe flags..."
+      make clean
+      ./configure $safe_flags > /dev/null
+      make -j$cpucount -s
+      echo "Sticking with safe Python for now..."
+    fi
+    echo "Installing Python..."
+    make install > /dev/null
+    cd ~
   fi
-  cd Python-$BUILD_PYTHON_VERSION
-  echo "Compiling Python $BUILD_PYTHON_VERSION..."
-  safe_flags="--with-openssl=$mypath/ssl --enable-shared --prefix=$mypath/python --with-ensurepip=upgrade"
-  unsafe_flags="--enable-optimizations --with-lto"  
-  if [ ! -e Makefile ]; then
-    echo "running configure with safe and unsafe"
-    ./configure $safe_flags $unsafe_flags > /dev/null
-  fi
-  make -j$cpucount PROFILE_TASK="-m test.regrtest --pgo -j$(( $cpucount * 2 ))" -s
-  RESULT=$?
-  echo "First make exited with $RESULT"
-  if [ $RESULT != 0 ]; then
-    echo "Trying Python compile again without unsafe flags..."
-    make clean
-    ./configure $safe_flags > /dev/null
-    make -j$cpucount -s
-    echo "Sticking with safe Python for now..."
-  fi
-  echo "Installing Python..."
-  make install > /dev/null
-  cd ~
 
-  export LD_LIBRARY_PATH=~/ssl/lib:~/python/lib
   python=~/python/bin/python3
   pip=~/python/bin/pip3
 
-  $python -V
-
   if [[ "$dist" == "precise" ]]; then
     echo "Installing deps for StaticX..."
     sudo apt-get install --yes scons
@@ -94,10 +105,11 @@ else
     fi
     $pip install git+https://github.com/JonathonReinhart/staticx.git@master
   fi
+
   cd $whereibelong
 fi
 
 echo "Upgrading pip packages..."
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade pyinstaller
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz

src/travis/linux-x86_64-install.sh

@@ -29,5 +29,4 @@ else
     du -h gam/gam
     time $gam version extended
   fi
-
 fi

src/travis/osx-x86_64-before-install.sh

@@ -5,15 +5,15 @@ whereibelong=$(pwd)
 
 #brew upgrade
 
-cd ~/pybuild
+cd ~
 
-#if [ ! -f python-$BUILD_PYTHON_VERSION-macosx10.9.pkg ]; then
-#  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
-#fi
-#sudo installer -pkg python-$BUILD_PYTHON_VERSION-macosx10.9.pkg -target /
+if [ ! -f python-$BUILD_PYTHON_VERSION-macosx10.9.pkg ]; then
+  wget --quiet https://www.python.org/ftp/python/$BUILD_PYTHON_VERSION/python-$BUILD_PYTHON_VERSION-macosx10.9.pkg
+fi
+sudo installer -pkg python-$BUILD_PYTHON_VERSION-macosx10.9.pkg -target /
 
-brew install openssl@1.1
-brew upgrade python
+#brew install openssl@1.1
+#brew upgrade python
 
 export python=python3
 export pip=pip3
@@ -35,7 +35,7 @@ which $python
 #echo "Running make install for OpenSSL..."
 #make install > /dev/null
 #export LD_LIBRARY_PATH=~/ssl/lib
-#cd ~/pybuild
+#cd ~
 
 # Compile latest Python
 #if [ ! -d Python-$BUILD_PYTHON_VERSION ]; then
@@ -75,4 +75,4 @@ export PATH=/usr/local/opt/python/libexec/bin:$PATH
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
 $pip install --upgrade -r src/requirements.txt
-$pip install --upgrade pyinstaller
+$pip install --upgrade https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 

src/travis/windows-x86-before-install.sh

@@ -1,27 +1,13 @@
 echo "Installing Net-Framework-Core..."
 export mypath=$(pwd)
+cd ~
 until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~/pybuild
-export exefile=Win32OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
-if [ ! -e $exefile ]; then
-  echo "Downloading $exefile..."
-  wget --quiet https://slproweb.com/download/$exefile
-fi
-echo "Installing $exefile..."
-powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
 cinst -y --forcex86 python3
 until cinst -y wixtoolset; do echo "trying again..."; done
-echo "OpenSSL dlls..."
-ls -alRF /c/ssl
-echo "c drive"
-ls -al /c/
-echo "Python dlls..."
-ls -al /c/Python37/DLLs
-until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
-export PATH=$PATH:/c/Python37/scripts
+export PATH=$PATH:/c/Python38/scripts
 cd $mypath
-export python=/c/Python37/python.exe
-export pip=pip
+export python=/c/Python38/python.exe
+export pip=/c/Python38/scripts/pip.exe
 
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
@@ -33,9 +19,9 @@ $pip install --upgrade -r src/requirements.txt
 # lots of malware uses PyInstaller default bootloader
 # https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
 echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
-tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
-cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz 
+tar xf develop.tar.gz
+cd pyinstaller-develop/bootloader
 echo "bootloader before:"
 md5sum ../PyInstaller/bootloader/Windows-32bit/*
 $python ./waf all --target-arch=32bit

src/travis/windows-x86_64-before-install.sh

@@ -1,22 +1,22 @@
 echo "Installing Net-Framework-Core..."
 export mypath=$(pwd)
 until powershell Install-WindowsFeature Net-Framework-Core; do echo "trying again..."; done
-cd ~/pybuild
-export exefile=Win64OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
-if [ ! -e $exefile ]; then
-  echo "Downloading $exefile..."
-  wget --quiet https://slproweb.com/download/$exefile
-fi
-echo "Installing $exefile..."
-powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
+cd ~
+#export exefile=Win64OpenSSL_Light-${BUILD_OPENSSL_VERSION//./_}.exe
+#if [ ! -e $exefile ]; then
+#  echo "Downloading $exefile..."
+#  wget --quiet https://slproweb.com/download/$exefile
+#fi
+#echo "Installing $exefile..."
+#powershell ".\\${exefile} /silent /sp- /suppressmsgboxes /DIR=C:\\ssl"
 cinst -y python3
 until cinst -y wixtoolset; do echo "trying again..."; done
-until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/Python37/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
-until cp -v /c/ssl/libssl-1_1-x64.dll /c/Python37/DLLs/libssl-1_1.dll; do echo "trying again..."; done
-export PATH=$PATH:/c/Python37/scripts
+#until cp -v /c/ssl/libcrypto-1_1-x64.dll /c/Python37/DLLs/libcrypto-1_1.dll; do echo "trying again..."; done
+#until cp -v /c/ssl/libssl-1_1-x64.dll /c/Python37/DLLs/libssl-1_1.dll; do echo "trying again..."; done
+export PATH=$PATH:/c/Python38/scripts
 cd $mypath
-export python=/c/Python37/python.exe
-export pip=pip
+export python=/c/Python38/python.exe
+export pip=/c/Python38/scripts/pip.exe
 
 $pip install --upgrade pip
 $pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 $pip install -U
@@ -27,9 +27,12 @@ $pip install --upgrade -r src/requirements.txt
 # lots of malware uses PyInstaller default bootloader
 # https://stackoverflow.com/questions/53584395/how-to-recompile-the-bootloader-of-pyinstaller
 echo "Downloading PyInstaller..."
-wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
-tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
-cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+#wget --quiet https://github.com/pyinstaller/pyinstaller/releases/download/v$PYINSTALLER_VERSION/PyInstaller-$PYINSTALLER_VERSION.tar.gz
+wget --quiet https://github.com/pyinstaller/pyinstaller/archive/develop.tar.gz
+#tar xf PyInstaller-$PYINSTALLER_VERSION.tar.gz
+tar xf develop.tar.gz
+#cd PyInstaller-$PYINSTALLER_VERSION/bootloader
+cd pyinstaller-develop/bootloader
 echo "bootloader before:"
 md5sum ../PyInstaller/bootloader/Windows-64bit/*
 $python ./waf all --target-arch=64bit
@@ -39,5 +42,5 @@ echo "PATH: $PATH"
 cd ..
 $python setup.py install
 echo "cd to $mypath..."
-until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
+#until cp -v /c/ssl/*.dll /c/Python37/DLLs; do echo "trying again..."; done
 cd $mypath

src/travis/windows-x86_64-install.sh

@@ -1,7 +1,9 @@
 cd src
+echo "compiling GAM with pyinstaller..."
 pyinstaller --clean --noupx -F --distpath=gam $GAMOS-gam.spec
 export gam="gam/gam"
 export gampath=$(readlink -e gam)
+echo "running compiled GAM..."
 $gam version
 export GAMVERSION=`$gam version simple`
 rm gam/lastupdatecheck.txt

src/var.py

@@ -6,7 +6,7 @@ import platform
 import re
 
 gam_author = 'Jay Lee <jay0lee@gmail.com>'
-gam_version = '4.95'
+gam_version = '4.96'
 gam_license = 'Apache License 2.0 (http://www.apache.org/licenses/LICENSE-2.0)'
 
 GAM_URL = 'https://git.io/gam'
@@ -949,6 +949,7 @@ MESSAGE_NO_TRANSFER_LACK_OF_DISK_SPACE = 'Cowardly refusing to perform migration
 MESSAGE_RESULTS_TOO_LARGE_FOR_GOOGLE_SPREADSHEET = 'Results are too large for Google Spreadsheets. Uploading as a regular CSV file.'
 MESSAGE_SERVICE_NOT_APPLICABLE = 'Service not applicable for this address: {0}. Please make sure service is enabled for user and run\n\ngam user <user> check serviceaccount\n\nfor further instructions'
 MESSAGE_INSTRUCTIONS_OAUTH2SERVICE_JSON = 'Please run\n\ngam create project\ngam user <user> check serviceaccount\n\nto create and configure a service account.'
+MESSAGE_UPDATE_GAM_TO_64BIT = "You're running a 32-bit version of GAM on a 64-bit version of Windows, upgrade to a windows-x86_64 version of GAM"
 MESSAGE_YOUR_SYSTEM_TIME_DIFFERS_FROM_GOOGLE_BY = 'Your system time differs from Google by %s'
 # oauth errors
 OAUTH2_TOKEN_ERRORS = [