diff --git a/docs/Authorization.md b/docs/Authorization.md index d82a2f29..e93642c7 100644 --- a/docs/Authorization.md +++ b/docs/Authorization.md @@ -896,6 +896,118 @@ gam update serviceaccount (scope|scopes )* * `` - Typically `user `, a non-Google Workspace administrator. * `scopes ` - Verify/enable service account access for a set of specific scopes rather than selecting the scopes. +``` +gam user user@domain.com update serviceaccount + +[*] 0) AlertCenter API +[*] 1) Analytics API - read only +[*] 2) Analytics Admin API - read only +[*] 3) Calendar API (supports readonly) +[*] 4) Chat API - Memberships (supports readonly) +[*] 5) Chat API - Messages (supports readonly) +[*] 6) Chat API - Spaces (supports readonly) +[*] 7) Chat API - Spaces Delete +[*] 8) Classroom API - Course Announcements (supports readonly) +[*] 9) Classroom API - Course Topics (supports readonly) +[*] 10) Classroom API - Course Work/Materials (supports readonly) +[*] 11) Classroom API - Course Work/Submissions (supports readonly) +[*] 12) Classroom API - Profile Emails +[*] 13) Classroom API - Profile Photos +[*] 14) Classroom API - Rosters (supports readonly) +[*] 15) Cloud Identity Devices API (supports readonly) +[*] 16) Cloud Resource Manager API v3 +[*] 17) Docs API (supports readonly) +[*] 18) Drive API (supports readonly) +[*] 19) Drive API - todrive +[*] 20) Drive Activity API v2 - must pair with Drive API +[*] 21) Drive Labels API v2beta - Admin (supports readonly) +[*] 22) Drive Labels API v2beta - User (supports readonly) +[*] 23) Forms API +[*] 24) Gmail API - Basic Settings (Filters,IMAP, Language, POP, Vacation) - read/write, Sharing Settings (Delegates, Forwarding, SendAs) - read +[*] 25) Gmail API - Full Access (Labels, Messages) +[*] 26) Gmail API - Full Access (Labels, Messages) except delete message +[ ] 27) Gmail API - Full Access - read only +[ ] 28) Gmail API - Send Messages - including todrive +[*] 29) Gmail API - Sharing Settings (Delegates, Forwarding, SendAs) - write +[*] 30) Identity and Access Management API +[*] 31) Keep API (supports readonly) +[*] 32) Looker Studio API (supports readonly) +[*] 33) OAuth2 API +[*] 34) People API (supports readonly) +[*] 35) People API - Other Contacts - read only +[*] 36) People Directory API - read only +[*] 37) Sheets API (supports readonly) +[*] 38) Sheets API - todrive +[*] 39) Sites API +[*] 40) Tasks API (supports readonly) +[ ] 41) Youtube API - read only + +Select an unselected scope [ ] by entering a number; yields [*] +For scopes that support readonly, enter a number and an 'r' to grant read-only access; yields [R] +For scopes that support action, enter a number and an 'a' to grant action-only access; yields [A] +Clear read-only access [R] or action-only access [A] from a scope by entering a number; yields [*] +Unselect a selected scope [*] by entering a number; yields [ ] +Select all default scopes by entering an 's'; yields [*] for default scopes, [ ] for others +Unselect all scopes by entering a 'u'; yields [ ] for all scopes +Exit without changes/authorization by entering an 'e' +Continue to authorization by entering a 'c' + +Please enter 0-41[a|r] or s|u|e|c: c + +System time status + Your system time differs from admin.googleapis.com by less than 1 second PASS +Service Account Private Key Authentication + Authentication PASS +Service Account Private Key age; Google recommends rotating keys on a routine basis + Service Account Private Key age: 364 days WARN +Domain-wide Delegation authentication:, User: user@domain.com, Scopes: 34 + https://mail.google.com/ PASS (1/34) + https://sites.google.com/feeds PASS (2/34) + https://www.googleapis.com/auth/analytics.readonly PASS (3/34) + https://www.googleapis.com/auth/apps.alerts PASS (4/34) + https://www.googleapis.com/auth/calendar PASS (5/34) + https://www.googleapis.com/auth/chat.delete PASS (6/34) + https://www.googleapis.com/auth/chat.memberships PASS (7/34) + https://www.googleapis.com/auth/chat.messages PASS (8/34) + https://www.googleapis.com/auth/chat.spaces PASS (9/34) + https://www.googleapis.com/auth/classroom.announcements PASS (10/34) + https://www.googleapis.com/auth/classroom.coursework.students PASS (11/34) + https://www.googleapis.com/auth/classroom.courseworkmaterials PASS (12/34) + https://www.googleapis.com/auth/classroom.profile.emails PASS (13/34) + https://www.googleapis.com/auth/classroom.profile.photos PASS (14/34) + https://www.googleapis.com/auth/classroom.rosters PASS (15/34) + https://www.googleapis.com/auth/classroom.topics PASS (16/34) + https://www.googleapis.com/auth/cloud-identity PASS (17/34) + https://www.googleapis.com/auth/cloud-platform PASS (18/34) + https://www.googleapis.com/auth/contacts PASS (19/34) + https://www.googleapis.com/auth/contacts.other.readonly PASS (20/34) + https://www.googleapis.com/auth/datastudio PASS (21/34) + https://www.googleapis.com/auth/directory.readonly PASS (22/34) + https://www.googleapis.com/auth/documents PASS (23/34) + https://www.googleapis.com/auth/drive PASS (24/34) + https://www.googleapis.com/auth/drive.activity PASS (25/34) + https://www.googleapis.com/auth/drive.admin.labels FAIL (26/34) + https://www.googleapis.com/auth/drive.labels FAIL (27/34) + https://www.googleapis.com/auth/gmail.modify PASS (28/34) + https://www.googleapis.com/auth/gmail.settings.basic PASS (29/34) + https://www.googleapis.com/auth/gmail.settings.sharing PASS (30/34) + https://www.googleapis.com/auth/keep PASS (31/34) + https://www.googleapis.com/auth/spreadsheets PASS (32/34) + https://www.googleapis.com/auth/tasks PASS (33/34) + https://www.googleapis.com/auth/userinfo.profile PASS (34/34) +Some scopes Failed! +To authorize them, please go to the following link in your browser: + + https://admin.google.com/ac/owl/domainwidedelegation?clientScopeToAdd=https://mail.google.com/,... + +You will be directed to the Google Workspace admin console Security > API Controls > Domain-wide Delegation page +The "Add a new Client ID" box will open +Make sure that "Overwrite existing client ID" is checked +Click AUTHORIZE +When the box closes you're done +After authorizing it may take some time for this test to pass so wait a few moments and then try this command again. +``` + ## Configure Limited access You can configure GAM to allow users limited access to your domain via GAM. You can limit both client and service account access. diff --git a/docs/GamUpdates.md b/docs/GamUpdates.md index e3461b52..e5659446 100644 --- a/docs/GamUpdates.md +++ b/docs/GamUpdates.md @@ -10,6 +10,61 @@ Add the `-s` option to the end of the above commands to suppress creating the `g See [Downloads](https://github.com/taers232c/GAMADV-XTD3/wiki/Downloads) for Windows or other options, including manual installation +### 6.71.05 + +Fixed a bug introduced in 6.71.00 that caused a trap in `gam print filelist`. + +Added option `tdfrom ` to `` that causes GAM to use `` as the from address +in all emails sent. By default, the from address is the Google Workspace Admin in `gam oauth info`. + +### 6.71.04 + +Updated `gam create|update cseidentity` to accept either of the following key pair options: +* `primarykeypairid ` - The configuration of a CSE identity that uses the same key pair for signing and encryption. +* `signingkeypairid encryptionkeypairid ` - The configuration of a CSE identity that uses different key pairs for signing and encryption. + +Updated CSV output row sorting to avoid a trap that occurred when a row was missing one of the sort fields. + +### 6.71.03 + +Added option `tdalert ` to ``. When a todrive file is created or updated, +GAM will send notification emails to all `tdalert ` users if `tdnotify` is true. +`` must be valid within your Google Workspace. + +### 6.71.02 + +Added additional error handling to Gmail Client Side Encryption commands. + +### 6.71.01 + +Fixed bug in `gam audit monitor create` that caused a trap. + +### 6.71.00 + +Added `csv_output_sort_headers` string list variable to `gam.cfg` that causes GAM to sort CSV output +rows by the column headers specified in the variable. The column headers are case insensitive and +if column header does not appear in the CSV output, it is ignored. + +Added `sortheaders ` to `redirect csv ` that has the same effect as above. + +The sort keys specified in `redirect csv ... sortheaders ` take precedence over the values from `gam.cfg`. + +Added option `tdsubject ` to `` that causes GAM to use `` as the subject +in all emails sent. In ``, `#file#` will, be replaced by the file title and `#sheet#` will be replaced +by the sheet/tab title. By default, the subject is the file title. + +### 6.70.09 + +Added additional error handling to Gmail Client Side Encryption commands. + +Added options `showpem` and `showkaclsdata` to all Gmail CSE commands that process/display +CSE key pairs. By default, the `pem` and `kaclsdata` fields will not be displayed unless +the corresponding `show` option is specified. + +### 6.70.08 + +Fixed bug in `gam create cseidentity ` that caused an error. + ### 6.70.07 Updated user instructions in `gam oauth create` and `gam update serviceaccount` diff --git a/docs/How-to-Upgrade-from-Standard-GAM.md b/docs/How-to-Upgrade-from-Standard-GAM.md index 8514444c..3b04303a 100644 --- a/docs/How-to-Upgrade-from-Standard-GAM.md +++ b/docs/How-to-Upgrade-from-Standard-GAM.md @@ -334,7 +334,7 @@ writes the credentials into the file oauth2.txt. admin@server:/Users/admin/bin/gamadv-xtd3$ rm -f /Users/admin/GAMConfig/oauth2.txt admin@server:/Users/admin/bin/gamadv-xtd3$ ./gam version WARNING: Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, Item: oauth2_txt, Value: /Users/admin/GAMConfig/oauth2.txt, Not Found -GAMADV-XTD3 6.70.07 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.71.05 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 @@ -1006,7 +1006,7 @@ writes the credentials into the file oauth2.txt. C:\GAMADV-XTD3>del C:\GAMConfig\oauth2.txt C:\GAMADV-XTD3>gam version WARNING: Config File: C:\GAMConfig\gam.cfg, Section: DEFAULT, Item: oauth2_txt, Value: C:\GAMConfig\oauth2.txt, Not Found -GAMADV-XTD3 6.70.07 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.71.05 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.2 64-bit final Windows-10-10.0.17134 AMD64 diff --git a/docs/Meta-Commands-and-File-Redirection.md b/docs/Meta-Commands-and-File-Redirection.md index 0a88475e..7c7a5896 100644 --- a/docs/Meta-Commands-and-File-Redirection.md +++ b/docs/Meta-Commands-and-File-Redirection.md @@ -56,6 +56,7 @@ The only `` recognized in this `
` are: * `csv_output_row_drop_filter` * `csv_output_row_drop_filter_mode` * `csv_output_row_limit` +* `csv_output_sort_headers` ### Select input filter section Select an input filter section from gam.cfg and process a GAM command using values from that section. @@ -113,7 +114,7 @@ You can redirect stdout and stderr to null and stderr can be redirected to stdou ::= redirect csv [multiprocess] [append] [noheader] [charset ] [columndelimiter ] [noescapechar ] [quotechar ] - [timestampcolumn ] + [sortheaders ] [timestampcolumn ] [todrive *] | redirect stdout [multiprocess] [append] | redirect stdout null [multiprocess] | @@ -151,6 +152,9 @@ The `quotechar ` subargument sets the character used to quote fields that contaim special charactere; the default value is the value of `csv_output_quote_char` in `gam.cfg` which defaults to double quote. +The `sortheaders ` argument causes GAM to sort CSV output rows by the column headers specified in ``. +The column headers are case insensitive and if column header does not appear in the CSV output, it is ignored. + The `timestampcolumn ` adds a column named `` to the CSV file; the value is the timestamp of when the GAM command started. diff --git a/docs/Todrive.md b/docs/Todrive.md index 520018f2..ccb9b0d7 100644 --- a/docs/Todrive.md +++ b/docs/Todrive.md @@ -174,11 +174,15 @@ direct the uploaded file to a particular user and location and add a timestamp t ``` ::= (tdaddsheet [])| + (tdalert )*| (tdbackupsheet (id:)|)| + (tdcellnumberformat text|number)| + (tdcellwrap clip|overflow|wrap)| (tdclearfilter [])| (tdcopysheet (id:)|)| (tddescription )| (tdfileid )| + (tdfrom )| (tdlocalcopy [])| (tdlocale )| (tdnobrowser [])| @@ -191,13 +195,12 @@ direct the uploaded file to a particular user and location and add a timestamp t (tdsheet (id:)|)| (tdsheettimestamp [] [tdsheettimeformat ]) (tdsheettitle )| - ([tdsheetdaysoffset ] [tdsheethoursoffset ] [tdtimeformat ])| - ([tddaysoffset ] [tdhoursoffset )| + ([tdsheetdaysoffset ] [tdsheethoursoffset ])| + (tdtimestamp [] [tdtimeformat ] + ([tddaysoffset ] [tdhoursoffset ])| (tdtimezone )| (tdtitle )| - (tdcellwrap clip|overflow|wrap)| - (tdcellnumberformat text|plain)| (tdupdatesheet [])| (tduploadnodata [])| (tduser ) @@ -227,6 +230,7 @@ If `tdfileid ` is not specified, a new file is created. * `tdtimeformat` - Format of the timestamp added to the title of the uploaded file; if not specified, the `todrive_timeformat` value from gam.cfg is used, that value defaults to '' which selects an ISO format timestamp. * See: https://docs.python.org/3/library/datetime.html#strftime-strptime-behavior * `tddaysoffset` and `tdhoursoffset` - Values that subtract time from the timestamp, they default to 0. A possible use for these values is as documentation to reflect the end of the time period that the uploaded report covers. +* `tdsubject ` - Use `` as the subject in all emails sent. In ``, `#file#` will, be replaced by the file title and `#sheet#` will be replaced by the sheet/tab title. By default, the subject is the file title. ## Spreadsheet settings * `tdlocale ` - The Spreadsheet settings Locale value. @@ -235,9 +239,10 @@ If `tdfileid ` is not specified, a new file is created. * `tdcellnumberformat text|number` - The Spreadsheet number format. ## Open browser and send email -* `tdnobrowser` - If False, a browser is opened to view the file uploaded to Google Drive; if not specified, the `todrive_nobrowser` value from gam.cfg is used. -* `tdnoemail` - If False, an email is sent to `tduser` informing them of name and URL of the uploaded file; if not specified, the `todrive_noemail` value from gam.cfg is used. -* `tdnotify` - If True, an email is sent to all `tdshare ` users informing them of name and URL of the uploaded/updated file. +* `tdnobrowser` - If False, a browser is opened to view the file uploaded to Google Drive; if not specified, the `todrive_nobrowser` value from gam.cfg is used. If True, no browser is opened. +* `tdnoemail` - If False, an email is sent to `tduser` informing them of name and URL of the uploaded file; if not specified, the `todrive_noemail` value from gam.cfg is used. If True, no email is sent to `tduser`. +* `tdnotify` - If True, an email is sent to all `tdshare ` and `tdalert ` users informing them of name and URL of the uploaded/updated file. If False, no emails are sent. +* `tdfrom ` - Emails will be sent with `` as the from address. By default, the from address is the Google Workspace Admin in `gam oauth info`. ## Escape character * `tdnoescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `todrive_no_escape_char` from `gam.cfg` will be used diff --git a/docs/Users-Calendars-Events.md b/docs/Users-Calendars-Events.md index 627f0b25..2bf75f40 100644 --- a/docs/Users-Calendars-Events.md +++ b/docs/Users-Calendars-Events.md @@ -592,6 +592,13 @@ To empty the calendar trash a temporary calendar is created, the deleted events gam empty calendartrash ``` +## Move calendar events to another calendar +Generally you won't move all events from one calendar to another; typically, you'll move events created by the event creator +using `matchfield creatoremail ` in conjunction with other `` and `` options. +``` +gam move events [] destination|to [] +``` + ## Display calendar events ``` gam info events [] [maxinstances ] diff --git a/docs/Users-Gmail-CSE.md b/docs/Users-Gmail-CSE.md index 4f01fcd7..be3aeff0 100644 --- a/docs/Users-Gmail-CSE.md +++ b/docs/Users-Gmail-CSE.md @@ -49,9 +49,15 @@ Creates and configures a client-side encryption identity that's authorized to se Google publishes the S/MIME certificate to a shared domain-wide directory so that people within a Google Workspace organization can encrypt and send mail to the identity. ``` -gam create cseidentity [kpemail ] +gam create cseidentity + (primarykeypairid ) | (signingkeypairid encryptionkeypairid ) + [kpemail ] [formatjson] ``` +One of the following is required: +* `primarykeypairid ` - The configuration of a CSE identity that uses the same key pair for signing and encryption. +* `signingkeypairid encryptionkeypairid ` - The configuration of a CSE identity that uses different key pairs for signing and encryption. + If `kpemail ` is not specified, the user's primary email address is used for the identity. By default, Gam displays the identity as an indented list of keys and values; the following option causes the output to be in JSON format: @@ -60,10 +66,16 @@ By default, Gam displays the identity as an indented list of keys and values; th ## Update Gmail CSE Identity Associates a different key pair with an existing client-side encryption identity. The updated key pair must validate against Google's S/MIME certificate profiles. ``` -gam update cseidentity [kpemail ] +gam update cseidentity + (primarykeypairid ) | (signingkeypairid encryptionkeypairid ) + [kpemail ] [formatjson] ``` -If `kpemail ` is not specified, the key pair for the user's primary email address is identity updated. +One of the following is required: +* `primarykeypairid ` - The configuration of a CSE identity that uses the same key pair for signing and encryption. +* `signingkeypairid encryptionkeypairid ` - The configuration of a CSE identity that uses different key pairs for signing and encryption. + +bIf `kpemail ` is not specified, the key pair for the user's primary email address is identity updated. By default, Gam displays the identity as an indented list of keys and values; the following option causes the output to be in JSON format: * `formatjson` - Display the fields in JSON format. @@ -112,7 +124,7 @@ Create a CSE Key Pair for the primary address of a user. gam create csekeypair [incertdir ] [inkeydir ] [addidentity []] [kpemail ] - [formatjson|returnidonly] + [showpem] [showkaclsdata] [formatjson|returnidonly] ``` * The S/MIME certificate files for the users are in the `incertdir ` folder/directory. * If this option is not specified, the directory is taken from `gam.cfg/gmail_cse_incert_dir`. @@ -126,6 +138,8 @@ gam create csekeypair * `kacls_url` - The URI of the key access control list service that manages the private key. * `wrapped_private_key` - Opaque data generated and used by the key access control list service. +By default, the `pem` and `kaclsdata` fields will not be displayed unless the corresponding `showpem` and `showkaclsdata` option is specified. + By default, Gam displays the new key pair as an indented list of keys and values; the following options cause the output to be displayed in alternate forms. * `formatjson` - Display the fields in JSON format. * `returnidonly` - Display just the new ``. @@ -139,11 +153,14 @@ By default, Gam displays the identity as an indented list of keys and values; th ## Action Gmail CSE Key Pairs +### Display pem and kaclsdata fields +By default, the `pem` and `kaclsdata` fields will not be displayed unless the corresponding `showpem` and `showkaclsdata` option is specified. + ### Disable Turns off a client-side encryption key pair. The authenticated user can no longer use the key pair to decrypt incoming CSE message texts or sign outgoing CSE mail. ``` gam disable csekeypair - [formatjson] + [showpem] [showkaclsdata] [formatjson] ``` By default, Gam displays the disabled key pair as an indented list of keys and values; the following option causes the output to be displayed in alternate forms. * `formatjson` - Display the fields in JSON format. @@ -152,7 +169,7 @@ By default, Gam displays the disabled key pair as an indented list of keys and v Turn on a client-side encryption key pair that was turned off. The key pair becomes active again for any associated client-side encryption identities. ``` gam ensable csekeypair - [formatjson] + [showpem] [showkaclsdata] [formatjson] ``` By default, Gam displays the enabled key pair as an indented list of keys and values; the following option causes the output to be displayed in alternate forms. * `formatjson` - Display the fields in JSON format. @@ -167,10 +184,13 @@ gam obliterate csekeypair Gmail can't restore or decrypt any messages that were encrypted by an obliterated key. Authenticated users and Google Workspace administrators lose access to reading the encrypted messages. ## Display Gmail CSE Key Pairs +### Display pem and kaclsdata fields +By default, the `pem` and `kaclsdata` fields will not be displayed unless the corresponding `showpem` and `showkaclsdata` option is specified. + ### Display an existing client-side encryption key pair. ``` gam info csekeypair - [formatjson] + [showpem] [showkaclsdata] [formatjson] ``` By default, Gam displays the key pairs as an indented list of keys and values; the following option causes the output to be in JSON format: * `formatjson` - Display the fields in JSON format. @@ -179,14 +199,14 @@ By default, Gam displays the key pairs as an indented list of keys and values; t ### Display all client-side encryption key pairs for an authenticated user. ``` gam show csekeypairs - [formatjson] + [showpem] [showkaclsdata] [formatjson] ``` By default, Gam displays the key pairs as an indented list of keys and values; the following option causes the output to be in JSON format: * `formatjson` - Display the fields in JSON format. ``` gam print csekeypairs [todrive *] - [formatjson [quotechar ]] + [showpem] [showkaclsdata] [formatjson [quotechar ]] ``` By default, Gam displays the key pairs as columns of fields; the following option causes the output to be in JSON format: * `formatjson` - Display the fields in JSON format. diff --git a/docs/Version-and-Help.md b/docs/Version-and-Help.md index 0d0d9c5a..12ec8a99 100644 --- a/docs/Version-and-Help.md +++ b/docs/Version-and-Help.md @@ -3,7 +3,7 @@ Print the current version of Gam with details ``` gam version -GAMADV-XTD3 6.70.07 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.71.05 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 @@ -15,7 +15,7 @@ Time: 2023-06-02T21:10:00-07:00 Print the current version of Gam with details and time offset information ``` gam version timeoffset -GAMADV-XTD3 6.70.07 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.71.05 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 @@ -27,7 +27,7 @@ Your system time differs from www.googleapis.com by less than 1 second Print the current version of Gam with extended details and SSL information ``` gam version extended -GAMADV-XTD3 6.70.07 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.71.05 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 @@ -64,7 +64,7 @@ MacOS High Sierra 10.13.6 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 Version Check: Current: 5.35.08 - Latest: 6.70.07 + Latest: 6.71.05 echo $? 1 ``` @@ -72,7 +72,7 @@ echo $? Print the current version number without details ``` gam version simple -6.70.07 +6.71.05 ``` In Linux/MacOS you can do: ``` @@ -82,7 +82,7 @@ echo $VER Print the current version of Gam and address of this Wiki ``` gam help -GAM 6.70.07 - https://github.com/taers232c/GAMADV-XTD3 +GAM 6.71.05 - https://github.com/taers232c/GAMADV-XTD3 Ross Scroggs Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 diff --git a/docs/gam.cfg.md b/docs/gam.cfg.md index 2c8b205f..0cbe4dee 100644 --- a/docs/gam.cfg.md +++ b/docs/gam.cfg.md @@ -255,6 +255,10 @@ csv_output_row_filter_mode csv_output_row_limit A limit on the number of rows to write to a CSV file; a value of 0 sets no limit. Default: 0 +csv_output_sort_headers + A list of column headers that causes GAM to sort CSV output rows by those headers. + The column headers are case insensitive and if column header does not appear in the CSV output, it is ignored. + Default: Blank csv_output_subfield_delimiter Character used to delimit fields and subfields in headers when writing CSV files; this must be a single character diff --git a/src/GamCommands.txt b/src/GamCommands.txt index d942f9dd..54442785 100644 --- a/src/GamCommands.txt +++ b/src/GamCommands.txt @@ -600,6 +600,7 @@ If an item contains spaces, it should be surrounded by ". ::= <String> <ToDriveAttribute> ::= (tdaddsheet [<Boolean>])| + (tdalert <EmailAddress>)*| (tdbackupsheet (id:<Number>)|<String>)| (tdcellnumberformat text|number)| (tdcellwrap clip|overflow|wrap)| @@ -607,20 +608,23 @@ If an item contains spaces, it should be surrounded by ". (tdcopysheet (id:<Number>)|<String>)| (tddescription <String>)| (tdfileid <DriveFileID>)| + (tdfrom <EmailAddress>)| (tdlocalcopy [<Boolean>])| (tdlocale <Locale>)| (tdnobrowser [<Boolean>])| (tdnoemail [<Boolean>])| (tdnoescapechar [<Boolean>])| + (tdnotify [<Boolean>])| (tdparent (id:<DriveFolderID>)|<DriveFolderName>)| (tdretaintitle [<Boolean>])| (tdshare <EmailAddress> commenter|reader|writer)*| (tdsheet (id:<Number>)|<String>)| (tdsheettimestamp [<Boolean>] [tdsheettimeformat <String>]) (tdsheettitle <String>)| + (tdsubject <String>)| ([tdsheetdaysoffset <Number>] [tdsheethoursoffset <Number>])| (tdtimestamp [<Boolean>] [tdtimeformat <String>] - [tddaysoffset <Number>] [tdhoursoffset <Number>])| + ([tddaysoffset <Number>] [tdhoursoffset <Number>])| (tdtimezone <TimeZone>)| (tdtitle <String>)| (tdupdatesheet [<Boolean>])| @@ -1247,7 +1251,7 @@ For redirect csv, the optional arguments must appear in the order shown. <Redirect> ::= redirect csv <FileName> [multiprocess] [append] [noheader] [charset <Charset>] [columndelimiter <Character>] [noescapechar <Boolean>] [quotechar <Character>] - [timestampcolumn <String>] + [sortheaders <StringList>] [timestampcolumn <String>] [todrive <ToDriveAttribute>*] | redirect stdout <FileName> [multiprocess] [append] | redirect stdout null [multiprocess] | @@ -7173,9 +7177,13 @@ gam <UserTypeEntity> print smimes [todrive <ToDriveAttribute>*] # Users - Gmail Client Side Encryption -gam <UserTypeEntity> create cseidentity <KeyPairID> [kpemail <EmailAddress>] +gam <UserTypeEntity> create cseidentity + (primarykeypairid <KeyPairID>) | (signingkeypairid <KeyPairID> encryptionkeypairid <KeyPairID>) + [kpemail <EmailAddress>] [formatjson] -gam <UserTypeEntity> update cseidentity <KeyPairID> [kpemail <EmailAddress>] +gam <UserTypeEntity> update cseidentity + (primarykeypairid <KeyPairID>) | (signingkeypairid <KeyPairID> encryptionkeypairid <KeyPairID>) + [kpemail <EmailAddress>] [formatjson] gam <UserTypeEntity> delete cseidentity [kpemail <EmailAddress>] @@ -7189,19 +7197,19 @@ gam <UserTypeEntity> print cseidentities [todrive <ToDriveAttribute>*] gam <UserTypeEntity> create csekeypair [incertdir <FilePath>] [inkeydir <FilePath>] [addidentity [<Boolean>]] [kpemail <EmailAddress>] - [formatjson|returnidonly] + [showpem] [showkaclsdata] [formatjson|returnidonly] gam <UserTypeEntity> disable csekeypair <KeyPairID> - [formatjson] + [showpem] [showkaclsdata] [formatjson] gam <UserTypeEntity> enable csekeypair <KeyPairID> - [formatjson] + [showpem] [showkaclsdata] [formatjson] gam <UserTypeEntity> obliterate csekeypair <KeyPairID> gam <UserTypeEntity> info csekeypair <KeyPairID> - [formatjson] + [showpem] [showkaclsdata] [formatjson] gam <UserTypeEntity> show csekeypairs - [formatjson] + [showpem] [showkaclsdata] [formatjson] gam <UserTypeEntity> print csekeypairs [todrive <ToDriveAttribute>*] - [formatjson [quotechar <Character>]] + [showpem] [showkaclsdata] [formatjson [quotechar <Character>]] # Users - Gmail - Settings diff --git a/src/GamUpdate.txt b/src/GamUpdate.txt index 63dd2166..4ed0797c 100644 --- a/src/GamUpdate.txt +++ b/src/GamUpdate.txt @@ -2,6 +2,61 @@ Merged GAM-Team version +6.71.05 + +Fixed a bug introduced in 6.71.00 that caused a trap in `gam <UserTypeEntity> print filelist`. + +Added option `tdfrom <EmailAddress>` to `<ToDriveAttribute>` that causes GAM to use `<EmailAddress>` as the from address +in all emails sent. By default, the from address is the Google Workspace Admin in `gam oauth info`.o + +6.71.04 + +Updated `gam <UserTypeEntity> create|update cseidentity` to accept either of the following key pair options: +* `primarykeypairid <KeyPairID>` - The configuration of a CSE identity that uses the same key pair for signing and encryption. +* `signingkeypairid <KeyPairID> encryptionkeypairid <KeyPairID>` - The configuration of a CSE identity that uses different key pairs for signing and encryption. + +Updated CSV output row sorting to avoid a trap that occurred when a row was missing one of the sort fields. + +6.71.03 + +Added option `tdalert <EmailAddress>` to `<ToDriveAttribute>`. When a todrive file is created or updated, +GAM will send notification emails to all `tdalert <EmailAddress>` users if `tdnotify` is true. +`<EmailAddress>` must be valid within your Google Workspace. + +6.71.02 + +Added additional error handling to Gmail Client Side Encryption commands. + +6.71.01 + +Fixed bug in `gam audit monitor create` that caused a trap. + +6.71.00 + +Added `csv_output_sort_headers` string list variable to `gam.cfg` that causes GAM to sort CSV output +rows by the column headers specified in the variable. The column headers are case insensitive and +if column header does not appear in the CSV output, it is ignored. + +Added `sortheaders <StringList>` to `redirect csv <FileName>` that has the same effect as above. + +The sort keys specified in `redirect csv ... sortheaders <StringList>` take precedence over the values from `gam.cfg`. + +Added option `tdsubject <String>` to `<ToDriveAttribute>` that causes GAM to use `<String>` as the subject +in all emails sent. In `<String>`, `#file#` will, be replaced by the file title and `#sheet#` will be replaced +by the sheet/tab title. By default, the subject is the file title. + +6.70.09 + +Added additional error handling to Gmail Client Side Encryption commands. + +Added options `showpem` and `showkaclsdata` to all Gmail CSE commands that process/display +CSE key pairs. By default, the `pem` and `kaclsdata` fields will not be displayed unless +the corresponding `show` option is specified. + +6.70.08 + +Fixed bug in `gam <UserTypeEntity> create cseidentity <KeyPairID>` that caused an error. + 6.70.07 Updated user instructions in `gam oauth create` and `gam <UserTypeEntity> update serviceaccount` diff --git a/src/gam/__init__.py b/src/gam/__init__.py index 506aad3d..d55eb13c 100755 --- a/src/gam/__init__.py +++ b/src/gam/__init__.py @@ -3408,16 +3408,6 @@ def SetGlobalVariables(): _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {filters}') return headerFilters - def _getCfgHeaderForce(sectionName, itemName): - value = GM.Globals[GM.PARSER].get(sectionName, itemName) - headerForce = [] - if not value or (len(value) == 2 and _stringInQuotes(value)): - return headerForce - splitStatus, headerForce = shlexSplitListStatus(value) - if not splitStatus: - _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {headerForce}') - return headerForce - def _getCfgHeaderFilterFromForce(sectionName, itemName): headerFilters = [] for filterStr in GC.Values[itemName]: @@ -3613,6 +3603,16 @@ def SetGlobalVariables(): _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {integerLimits(minLen, maxLen, Msg.STRING_LENGTH)}') return '' + def _getCfgStringList(sectionName, itemName): + value = GM.Globals[GM.PARSER].get(sectionName, itemName) + stringlist = [] + if not value or (len(value) == 2 and _stringInQuotes(value)): + return stringlist + splitStatus, stringlist = shlexSplitListStatus(value) + if not splitStatus: + _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.INVALID_LIST}: {stringlist}') + return stringlist + def _getCfgTimezone(sectionName, itemName): value = _stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName).lower()) if value == 'utc': @@ -3946,7 +3946,7 @@ def SetGlobalVariables(): value = bytes(value[2:-1], UTF8) elif varType == GC.TYPE_TIMEZONE: value = getString(Cmd.OB_STRING, checkBlank=True) - else: + else: # GC.TYPE_STRING, GC.TYPE_STRINGLIST minLen, maxLen = itemEntry.get(GC.VAR_LIMITS, (0, None)) value = _quoteStringIfLeadingTrailingBlanks(getString(Cmd.OB_STRING, minLen=minLen, maxLen=maxLen)) GM.Globals[GM.PARSER].set(sectionName, itemName, value) @@ -3973,14 +3973,14 @@ def SetGlobalVariables(): GC.Values[itemName] = _getCfgNumber(sectionName, itemName) elif varType == GC.TYPE_HEADERFILTER: GC.Values[itemName] = _getCfgHeaderFilter(sectionName, itemName) - elif varType == GC.TYPE_HEADERFORCE: - GC.Values[itemName] = _getCfgHeaderForce(sectionName, itemName) elif varType == GC.TYPE_LOCALE: GC.Values[itemName] = _getCfgLocale(sectionName, itemName) elif varType == GC.TYPE_PASSWORD: GC.Values[itemName] = _getCfgPassword(sectionName, itemName) elif varType == GC.TYPE_STRING: GC.Values[itemName] = _getCfgString(sectionName, itemName) + elif varType in {GC.TYPE_STRINGLIST, GC.TYPE_HEADERFORCE}: + GC.Values[itemName] = _getCfgStringList(sectionName, itemName) elif varType == GC.TYPE_FILE: GC.Values[itemName] = _getCfgFile(sectionName, itemName) # Row filters @@ -3996,7 +3996,7 @@ def SetGlobalVariables(): GC.Values[GC.CSV_INPUT_ROW_DROP_FILTER_MODE] = _getCfgChoice(inputFilterSectionName, GC.CSV_INPUT_ROW_DROP_FILTER_MODE) GC.Values[GC.CSV_INPUT_ROW_LIMIT] = _getCfgNumber(inputFilterSectionName, GC.CSV_INPUT_ROW_LIMIT) if outputFilterSectionName: - GC.Values[GC.CSV_OUTPUT_HEADER_FORCE] = _getCfgHeaderForce(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) + GC.Values[GC.CSV_OUTPUT_HEADER_FORCE] = _getCfgStringList(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) if GC.Values[GC.CSV_OUTPUT_HEADER_FORCE]: GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilterFromForce(outputFilterSectionName, GC.CSV_OUTPUT_HEADER_FORCE) else: @@ -4007,6 +4007,7 @@ def SetGlobalVariables(): GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER] = _getCfgRowFilter(outputFilterSectionName, GC.CSV_OUTPUT_ROW_DROP_FILTER) GC.Values[GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE] = _getCfgChoice(outputFilterSectionName, GC.CSV_OUTPUT_ROW_DROP_FILTER_MODE) GC.Values[GC.CSV_OUTPUT_ROW_LIMIT] = _getCfgNumber(outputFilterSectionName, GC.CSV_OUTPUT_ROW_LIMIT) + GC.Values[GC.CSV_OUTPUT_SORT_HEADERS] = _getCfgStringList(outputFilterSectionName, GC.CSV_OUTPUT_SORT_HEADERS) elif GC.Values[GC.CSV_OUTPUT_HEADER_FORCE]: GC.Values[GC.CSV_OUTPUT_HEADER_FILTER] = _getCfgHeaderFilterFromForce(sectionName, GC.CSV_OUTPUT_HEADER_FORCE) if status['errors']: @@ -4049,7 +4050,7 @@ def SetGlobalVariables(): _setMultiprocessExit() # redirect csv <FileName> [multiprocess] [append] [noheader] [charset <CharSet>] # [columndelimiter <Character>] [noescapechar <Boolean>] [quotechar <Character>]] -# [timestampcolumn <String>] +# [sortheaders <StringList>] [timestampcolumn <String>] # [todrive <ToDriveAttribute>*] # redirect stdout <FileName> [multiprocess] [append] # redirect stdout null @@ -4070,6 +4071,8 @@ def SetGlobalVariables(): GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR] = GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR] = getCharacter() if checkArgumentPresent('noescapechar'): GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR] = getBoolean() + if checkArgumentPresent('sortheaders'): + GM.Globals[GM.CSV_OUTPUT_SORT_HEADERS] = GC.Values[GC.CSV_OUTPUT_SORT_HEADERS] = getString(Cmd.OB_STRING_LIST, minLen=0).replace(',', ' ').split() if checkArgumentPresent('timestampcolumn'): GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN] = getString(Cmd.OB_STRING, minLen=0) _setCSVFile(filename, mode, encoding, writeHeader, multi) @@ -5134,12 +5137,12 @@ def checkGAPIError(e, softErrors=False, retryOnHttpError=False, mapNotFound=True elif http_status == 400: if '@attachmentnotvisible' in lmessage: error = makeErrorDict(http_status, GAPI.BAD_REQUEST, message) - elif 'does not match' in lmessage or 'invalid' in lmessage: - error = makeErrorDict(http_status, GAPI.INVALID, message) - elif status == 'FAILED_PRECONDITION' or 'precondition check failed' in lmessage: - error = makeErrorDict(http_status, GAPI.FAILED_PRECONDITION, message) elif status == 'INVALID_ARGUMENT': error = makeErrorDict(http_status, GAPI.INVALID_ARGUMENT, message) + elif status == 'FAILED_PRECONDITION' or 'precondition check failed' in lmessage: + error = makeErrorDict(http_status, GAPI.FAILED_PRECONDITION, message) + elif 'does not match' in lmessage or 'invalid' in lmessage: + error = makeErrorDict(http_status, GAPI.INVALID, message) elif http_status == 401: if 'active session is invalid' in lmessage and reason == 'authError': message += ' Drive SDK API access disabled' @@ -7618,6 +7621,7 @@ class CSVPrintFile(): self.titlesList = [] self.JSONtitlesSet = set() self.JSONtitlesList = [] + self.sortHeaders = [] self.SetHeaderForce(GC.Values[GC.CSV_OUTPUT_HEADER_FORCE]) if not self.headerForce and titles is not None: self.SetTitles(titles) @@ -7631,6 +7635,9 @@ class CSVPrintFile(): GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = GC.Values.get(GC.CSV_OUTPUT_NO_ESCAPE_CHAR, False) self.SetNoEscapeChar(GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR]) self.SetQuoteChar(GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR]) + if GM.Globals.get(GM.CSV_OUTPUT_SORT_HEADERS) is None: + GM.Globals[GM.CSV_OUTPUT_SORT_HEADERS] = GC.Values.get(GC.CSV_OUTPUT_SORT_HEADERS, []) + self.SetSortHeaders(GM.Globals[GM.CSV_OUTPUT_SORT_HEADERS]) if GM.Globals.get(GM.CSV_OUTPUT_TIMESTAMP_COLUMN) is None: GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = GC.Values.get(GC.CSV_OUTPUT_TIMESTAMP_COLUMN, '') self.SetTimestampColumn(GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN]) @@ -7846,7 +7853,7 @@ class CSVPrintFile(): 'fileId': None, 'parentId': None, 'parent': GC.Values[GC.TODRIVE_PARENT], 'retaintitle': False, 'localcopy': GC.Values[GC.TODRIVE_LOCALCOPY], 'uploadnodata': GC.Values[GC.TODRIVE_UPLOAD_NODATA], 'nobrowser': GC.Values[GC.TODRIVE_NOBROWSER], 'noemail': GC.Values[GC.TODRIVE_NOEMAIL], - 'share': [], 'notify': False} + 'alert': [], 'share': [], 'notify': False, 'subject': None, 'from': None} while Cmd.ArgumentsRemaining(): myarg = getArgument() if myarg == 'tduser': @@ -7918,12 +7925,18 @@ class CSVPrintFile(): self.todrive['noemail'] = getBoolean() elif myarg == 'tdnoescapechar': self.todrive['noescapechar'] = getBoolean() + elif myarg == 'tdalert': + self.todrive['alert'].append({'emailAddress': normalizeEmailAddressOrUID(getString(Cmd.OB_EMAIL_ADDRESS))}) elif myarg == 'tdshare': self.todrive['share'].append({'emailAddress': normalizeEmailAddressOrUID(getString(Cmd.OB_EMAIL_ADDRESS)), 'type': 'user', 'role': getChoice(self.TDSHARE_ACL_ROLES_MAP, mapChoice=True)}) elif myarg == 'tdnotify': self.todrive['notify'] = getBoolean() + elif myarg == 'tdsubject': + self.todrive['subject'] = getString(Cmd.OB_STRING, minLen=0) + elif myarg == 'tdfrom': + self.todrive['from'] = getString(Cmd.OB_EMAIL_ADDRESS) else: Cmd.Backup() break @@ -8164,6 +8177,9 @@ class CSVPrintFile(): else: self.todaysTime = todaysTime().strftime(GC.Values[GC.OUTPUT_TIMEFORMAT]) + def SetSortHeaders(self, sortHeaders): + self.sortHeaders = sortHeaders + def SetFormatJSON(self, formatJSON): self.formatJSON = formatJSON @@ -8340,11 +8356,26 @@ class CSVPrintFile(): Ent.FormatEntityValueList(entityValueList)+[Act.NotPerformed(), errMsg], currentCountNL(0, 0))) + @staticmethod + def itemgetter(*items): + if len(items) == 1: + item = items[0] + def g(obj): + return obj.get(item, '') + else: + def g(obj): + return tuple(obj.get(item, '') for item in items) + return g + def writeCSVData(writer): try: if GM.Globals[GM.CSVFILE][GM.REDIRECT_WRITE_HEADER]: writer.writerow(dict((item, item) for item in writer.fieldnames)) - writer.writerows(self.rows) + if not self.sortHeaders: + writer.writerows(self.rows) + else: + for row in sorted(self.rows, key=itemgetter(*self.sortHeaders)): + writer.writerow(row) return True except IOError as e: stderrErrorMsg(e) @@ -8362,6 +8393,13 @@ class CSVPrintFile(): 'strict': False} return writerDialect + def normalizeSortHeaders(): + if self.sortHeaders: + writerKeyMap = {} + for k in titlesList: + writerKeyMap[k.lower()] = k + self.sortHeaders = [writerKeyMap[k.lower()] for k in self.sortHeaders if k.lower() in writerKeyMap] + def writeCSVToStdout(): csvFile = StringIOobject() writerDialect = setDialect('\n', self.noEscapeChar) @@ -8652,12 +8690,16 @@ class CSVPrintFile(): file_url = result['webViewLink'] msg_txt = f'{Msg.DATA_UPLOADED_TO_DRIVE_FILE}:\n{file_url}' printKeyValueList([msg_txt]) + if not self.todrive['subject']: + subject = title + else: + subject = self.todrive['subject'].replace('#file#', title).replace('#sheet#', sheetTitle) if not self.todrive['noemail']: - send_email(title, msg_txt, user, clientAccess=GC.Values[GC.TODRIVE_CLIENTACCESS]) + send_email(subject, msg_txt, user, clientAccess=GC.Values[GC.TODRIVE_CLIENTACCESS], msgFrom=self.todrive['from']) if self.todrive['notify']: - for share in self.todrive['share']: - if share['emailAddress'] != user: - send_email(title, msg_txt, share['emailAddress'], clientAccess=GC.Values[GC.TODRIVE_CLIENTACCESS]) + for recipient in self.todrive['share']+self.todrive['alert']: + if recipient['emailAddress'] != user: + send_email(subject, msg_txt, recipient['emailAddress'], clientAccess=GC.Values[GC.TODRIVE_CLIENTACCESS], msgFrom=self.todrive['from']) if not self.todrive['nobrowser']: webbrowser.open(file_url) except (GAPI.forbidden, GAPI.insufficientPermissions): @@ -8679,7 +8721,7 @@ class CSVPrintFile(): (self.titlesList, self.sortTitlesList, self.indexedTitles, self.formatJSON, self.JSONtitlesList, self.columnDelimiter, self.noEscapeChar, self.quoteChar, - self.timestampColumn, + self.sortHeaders, self.timestampColumn, self.mapDrive3Titles, self.fixPaths, self.mapNodataFields, @@ -8732,6 +8774,7 @@ class CSVPrintFile(): else: self.AddJSONTitle(self.timestampColumn) titlesList = self.JSONtitlesList + normalizeSortHeaders() if (not self.todrive) or self.todrive['localcopy']: if GM.Globals[GM.CSVFILE][GM.REDIRECT_NAME] == '-': if GM.Globals[GM.STDOUT][GM.REDIRECT_MULTI_FD]: @@ -9285,6 +9328,7 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo csvPF.SetColumnDelimiter(GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER]) csvPF.SetNoEscapeChar(GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR]) csvPF.SetQuoteChar(GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR]) + csvPF.SetSortHeaders(GC.Values[GC.CSV_OUTPUT_SORT_HEADERS]) csvPF.SetTimestampColumn(GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN]) csvPF.SetHeaderFilter(GC.Values[GC.CSV_OUTPUT_HEADER_FILTER]) csvPF.SetHeaderDropFilter(GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER]) @@ -9307,12 +9351,13 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo csvPF.SetColumnDelimiter(dataItem[5]) csvPF.SetNoEscapeChar(dataItem[6]) csvPF.SetQuoteChar(dataItem[7]) - csvPF.SetTimestampColumn(dataItem[8]) - csvPF.SetMapDrive3Titles(dataItem[9]) - csvPF.SetFixPaths(dataItem[10]) - csvPF.SetNodataFields(dataItem[11], dataItem[12], dataItem[13], dataItem[14], dataItem[15]) - csvPF.SetShowPermissionsLast(dataItem[16]) - csvPF.SetZeroBlankMimeTypeCounts(dataItem[17]) + csvPF.SetSortHeaders(dataItem[8]) + csvPF.SetTimestampColumn(dataItem[9]) + csvPF.SetMapDrive3Titles(dataItem[10]) + csvPF.SetFixPaths(dataItem[11]) + csvPF.SetNodataFields(dataItem[12], dataItem[13], dataItem[14], dataItem[15], dataItem[16]) + csvPF.SetShowPermissionsLast(dataItem[17]) + csvPF.SetZeroBlankMimeTypeCounts(dataItem[18]) elif dataType == GM.REDIRECT_QUEUE_DATA: csvPF.rows.extend(dataItem) elif dataType == GM.REDIRECT_QUEUE_ARGS: @@ -9327,6 +9372,7 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo csvPF.SetColumnDelimiter(GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER]) csvPF.SetNoEscapeChar(GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR]) csvPF.SetQuoteChar(GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR]) + csvPF.SetSortHeaders(GC.Values[GC.CSV_OUTPUT_SORT_HEADERS]) csvPF.SetTimestampColumn(GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN]) csvPF.SetHeaderFilter(GC.Values[GC.CSV_OUTPUT_HEADER_FILTER]) csvPF.SetHeaderDropFilter(GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER]) @@ -9470,7 +9516,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, printCrosOUs, printCrosOUsAndChildren, output_dateformat, output_timeformat, csvColumnDelimiter, csvNoEscapeChar, csvQuoteChar, - csvTimestampColumn, + csvSortHeaders, csvTimestampColumn, csvHeaderFilter, csvHeaderDropFilter, csvHeaderForce, csvRowFilter, csvRowFilterMode, csvRowDropFilter, csvRowDropFilterMode, @@ -9501,6 +9547,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, GM.Globals[GM.CSV_OUTPUT_ROW_FILTER] = csvRowFilter[:] GM.Globals[GM.CSV_OUTPUT_ROW_FILTER_MODE] = csvRowFilterMode GM.Globals[GM.CSV_OUTPUT_ROW_LIMIT] = csvRowLimit + GM.Globals[GM.CSV_OUTPUT_SORT_HEADERS] = csvSortHeaders GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = csvTimestampColumn GM.Globals[GM.CSV_TODRIVE] = todrive.copy() GM.Globals[GM.DEBUG_LEVEL] = debugLevel @@ -9708,6 +9755,7 @@ def MultiprocessGAMCommands(items, showCmds): GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER], GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR], GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR], + GC.Values[GC.CSV_OUTPUT_SORT_HEADERS], GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN], GC.Values[GC.CSV_OUTPUT_HEADER_FILTER], GC.Values[GC.CSV_OUTPUT_HEADER_DROP_FILTER], @@ -12543,8 +12591,8 @@ def _showMailboxMonitorRequestStatus(request, i=0, count=0): printKeyValueList(['End', request['endDate']]) printKeyValueList(['Monitor Incoming', request['outgoingEmailMonitorLevel']]) printKeyValueList(['Monitor Outgoing', request['incomingEmailMonitorLevel']]) - printKeyValueList(['Monitor Chats', request['chatMonitorLevel']]) - printKeyValueList(['Monitor Drafts', request['draftMonitorLevel']]) + printKeyValueList(['Monitor Chats', request.get('chatMonitorLevel', 'NONE')]) + printKeyValueList(['Monitor Drafts', request.get('draftMonitorLevel', 'NONE')]) Ind.Decrement() # gam audit monitor create <EmailAddress> <DestEmailAddress> [begin <DateTime>] [end <DateTime>] [incoming_headers] [outgoing_headers] [nochats] [nodrafts] [chat_headers] [draft_headers] @@ -69908,7 +69956,8 @@ def printShowSmimes(users): def _showCSEItem(result, entityType, keyField, timeObjects, i, count, FJQC): if FJQC.formatJSON: - printLine(json.dumps(cleanJSON(result, timeObjects=timeObjects), ensure_ascii=False, sort_keys=True)) + printLine(json.dumps(cleanJSON(result, timeObjects=timeObjects), + ensure_ascii=False, sort_keys=True)) return Ind.Increment() printEntity([entityType, result[keyField]], i, count) @@ -69917,16 +69966,39 @@ def _showCSEItem(result, entityType, keyField, timeObjects, i, count, FJQC): Ind.Decrement() Ind.Decrement() +def _initCSEKeyPairSkipObjects(): + return {'pem', 'kaclsData'} + +def _resetCSEKeyPairSkipObjects(myarg, skipObjects): + if myarg == 'showpem': + skipObjects.discard('pem') + elif myarg == 'showkaclsdata': + skipObjects.discard('kaclsData') + else: + return False + return True + +def _stripCSEKeyPairSkipObjects(result, skipObjects): + if 'pem' in skipObjects: + result.pop('pem', None) + if 'kaclsData' in skipObjects: + for privateKeyMetadata in result.get('privateKeyMetadata', []): + if 'kaclsKeyMetadata' in privateKeyMetadata: + privateKeyMetadata['kaclsKeyMetadata'].pop('kaclsData', None) + CSE_IDENTITY_TIME_OBJECTS = {} CSE_KEYPAIR_TIME_OBJECTS = {'disableTime'} def _printShowCSEItems(users, entityType, keyField, timeObjects): csvPF = CSVPrintFile(['User', keyField]) if Act.csvFormat() else None FJQC = FormatJSONQuoteChar(csvPF) + skipObjects = _initCSEKeyPairSkipObjects() if entityType == Ent.CSE_KEYPAIR else set() while Cmd.ArgumentsRemaining(): myarg = getArgument() if csvPF and myarg == 'todrive': csvPF.GetTodriveParameters() + elif entityType == Ent.CSE_KEYPAIR and _resetCSEKeyPairSkipObjects(myarg, skipObjects): + pass else: FJQC.GetFormatJSONQuoteChar(myarg, True) i, count, users = getEntityArgument(users) @@ -69960,6 +70032,8 @@ def _printShowCSEItems(users, entityType, keyField, timeObjects): j = 0 for result in results: j += 1 + if entityType == Ent.CSE_KEYPAIR: + _stripCSEKeyPairSkipObjects(result, skipObjects) _showCSEItem(result, entityType, keyField, timeObjects, j, jcount, FJQC) else: for result in results: @@ -69968,7 +70042,8 @@ def _printShowCSEItems(users, entityType, keyField, timeObjects): csvPF.WriteRowTitles(row) elif csvPF.CheckRowTitles(row): csvPF.WriteRowNoFilter({'User': user, keyField: result[keyField], - 'JSON': json.dumps(cleanJSON(result, timeObjects=timeObjects), ensure_ascii=False, sort_keys=True)}) + 'JSON': json.dumps(cleanJSON(result, skipObjects=skipObjects, timeObjects=timeObjects), + ensure_ascii=False, sort_keys=True)}) if csvPF: csvPF.writeCSVfile(Ent.Plural(entityType)) @@ -69979,16 +70054,72 @@ CSE_IDENTITY_ACTION_FUNCTION_MAP = { Act.INFO: 'get', } -# gam <UserTypeEntity> create cseidentity <KeyPairID> [kpemail <EmailAddress>] +# gam <UserTypeEntity> create cseidentity +# (primarykeypairid <KeyPairID>) | (signingkeypairid <KeyPairID> encryptionkeypairid <KeyPairID>) +# [kpemail <EmailAddress>] # [formatjson] -# gam <UserTypeEntity> update cseidentity <KeyPairID> [kpemail <EmailAddress>] +# gam <UserTypeEntity> update cseidentity +# (primarykeypairid <KeyPairID>) | (signingkeypairid <KeyPairID> encryptionkeypairid <KeyPairID>) +# [kpemail <EmailAddress>] # [formatjson] +def createUpdateCSEIdentity(users): + function = CSE_IDENTITY_ACTION_FUNCTION_MAP[Act.Get()] + primaryKeyPairId = signingKeyPairId = encryptionKeyPairId = None + FJQC = FormatJSONQuoteChar() + kpEmail = None + while Cmd.ArgumentsRemaining(): + myarg = getArgument() + if myarg == 'primarykeypairid': + primaryKeyPairId = getString(Cmd.OB_CSE_KEYPAIR_ID) + elif myarg == 'signingkeypairid': + signingKeyPairId = getString(Cmd.OB_CSE_KEYPAIR_ID) + elif myarg == 'encryptionkeypairid': + encryptionKeyPairId = getString(Cmd.OB_CSE_KEYPAIR_ID) + elif myarg == 'kpemail': + kpEmail = getEmailAddress(noUid=True) + else: + FJQC.GetFormatJSON(myarg) + if primaryKeyPairId: + if signingKeyPairId or encryptionKeyPairId: + usageErrorExit(Msg.ARE_MUTUALLY_EXCLUSIVE.format('primarykeypairid', 'signingkeypairid/encryptionkeypairid')) + identity = {'primaryKeyPairId': primaryKeyPairId, 'emailAddress': None} + keyPairId = primaryKeyPairId + elif signingKeyPairId or encryptionKeyPairId: + if not signingKeyPairId or not encryptionKeyPairId: + usageErrorExit(Msg.ARE_BOTH_REQUIRED.format('signingkeypairid', 'encryptionkeypairid')) + identity = {'signAndEncryptKeyPairs': {'signingKeyPairId': signingKeyPairId, 'encryptionKeyPairId': encryptionKeyPairId}, + 'emailAddress': None} + keyPairId = f'{signingKeyPairId}/{encryptionKeyPairId}' + else: + missingArgumentExit('primarykeypairid|(signingkeypairid & encryptionkeypairid)') + i, count, users = getEntityArgument(users) + for user in users: + i += 1 + user, gmail = buildGAPIServiceObject(API.GMAIL, user, i, count) + if not gmail: + continue + identity['emailAddress'] = user if not kpEmail else kpEmail + kwargs = {'body': identity} + if function == 'patch': + kwargs['emailAddress'] = identity['emailAddress'] + kvList = [Ent.USER, user, Ent.CSE_IDENTITY, identity['emailAddress'], Ent.CSE_KEYPAIR, keyPairId] + try: + result = callGAPI(gmail.users().settings().cse().identities(), function, + throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED, GAPI.INVALID_ARGUMENT, GAPI.NOT_FOUND, GAPI.ALREADY_EXISTS], + userId='me', **kwargs) + if not FJQC.formatJSON: + entityActionPerformed(kvList, i, count) + _showCSEItem(result, Ent.CSE_IDENTITY, 'emailAddress', CSE_IDENTITY_TIME_OBJECTS, i, count, FJQC) + except (GAPI.permissionDenied, GAPI.invalidArgument, GAPI.notFound, GAPI.alreadyExists) as e: + entityActionFailedWarning(kvList, str(e), i, count) + except (GAPI.serviceNotAvailable, GAPI.badRequest): + entityServiceNotApplicableWarning(Ent.USER, user, i, count) + # gam <UserTypeEntity> delete cseidentity [kpemail <EmailAddress>] # gam <UserTypeEntity> info cseidentity [kpemail <EmailAddress>] # [formatjson] def processCSEIdentity(users): function = CSE_IDENTITY_ACTION_FUNCTION_MAP[Act.Get()] - keyPairId = getString(Cmd.OB_CSE_KEYPAIR_ID) if function in {'create', 'patch'} else None FJQC = FormatJSONQuoteChar() kpEmail = None while Cmd.ArgumentsRemaining(): @@ -70003,24 +70134,17 @@ def processCSEIdentity(users): user, gmail = buildGAPIServiceObject(API.GMAIL, user, i, count) if not gmail: continue - if keyPairId: - identity = {'keyPairId': keyPairId, 'emailAddress': user if not kpEmail else kpEmail} - kwargs = {'body': identity} - if function == 'patch': - kwargs['emailAddress'] = identity['emailAddress'] - kvList = [Ent.USER, user, Ent.CSE_IDENTITY, identity['emailAddress'], Ent.CSE_KEYPAIR, keyPairId] - else: - kwargs = {'cseEmailAddress': user if not kpEmail else kpEmail} - kvList = [Ent.USER, user, Ent.CSE_IDENTITY, kwargs['cseEmailAddress']] + kwargs = {'cseEmailAddress': user if not kpEmail else kpEmail} + kvList = [Ent.USER, user, Ent.CSE_IDENTITY, kwargs['cseEmailAddress']] try: result = callGAPI(gmail.users().settings().cse().identities(), function, - throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED], + throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED, GAPI.NOT_FOUND], userId='me', **kwargs) - if not FJQC.formatJSON: + if not FJQC.formatJSON: entityActionPerformed(kvList, i, count) if function != 'delete': _showCSEItem(result, Ent.CSE_IDENTITY, 'emailAddress', CSE_IDENTITY_TIME_OBJECTS, i, count, FJQC) - except GAPI.permissionDenied as e: + except (GAPI.permissionDenied, GAPI.notFound) as e: entityActionFailedWarning(kvList, str(e), i, count) except (GAPI.serviceNotAvailable, GAPI.badRequest): entityServiceNotApplicableWarning(Ent.USER, user, i, count) @@ -70034,7 +70158,7 @@ def printShowCSEIdentities(users): # gam <UserTypeEntity> create csekeypair [incertdir <FilePath>] [inkeydir <FilePath>] # [addidentity [<Boolean>]] [kpemail <EmailAddress>] -# [formatjson|returnidonly] +# [showpem] [showkaclsdata] [formatjson|returnidonly] def createCSEKeyPair(users): def _getFolderPath(myarg): filepath = os.path.expanduser(getString(Cmd.OB_FILE_PATH)) @@ -70047,6 +70171,7 @@ def createCSEKeyPair(users): inkeydir = GC.Values[GC.GMAIL_CSE_INKEY_DIR] addIdentity = returnIdOnly = False kpEmail = None + skipObjects = _initCSEKeyPairSkipObjects() while Cmd.ArgumentsRemaining(): myarg = getArgument() if myarg == 'incertdir': @@ -70059,6 +70184,8 @@ def createCSEKeyPair(users): kpEmail = getEmailAddress(noUid=True) elif myarg == 'returnidonly': returnIdOnly = True + elif _resetCSEKeyPairSkipObjects(myarg, skipObjects): + pass else: FJQC.GetFormatJSON(myarg) if not incertdir: @@ -70071,18 +70198,17 @@ def createCSEKeyPair(users): user, gmail = buildGAPIServiceObject(API.GMAIL, user, i, count) if not gmail: continue + kvList = [Ent.USER, user, Ent.CSE_KEYPAIR, None] smimeFilename = os.path.join(incertdir, user+'.p7pem') if not os.path.isfile(smimeFilename): - entityActionNotPerformedWarning([Ent.USER, user, Ent.CSE_KEYPAIR, None], - Msg.FILE_NOT_FOUND.format(smimeFilename), i, count) + entityActionNotPerformedWarning(kvList, Msg.FILE_NOT_FOUND.format(smimeFilename), i, count) continue smimeData = readFile(smimeFilename, mode='rb', continueOnError=True) if smimeData is None: continue kaclFilename = os.path.join(inkeydir, user+'.wrap') if not os.path.isfile(kaclFilename): - entityActionNotPerformedWarning([Ent.USER, user, Ent.CSE_KEYPAIR, None], - Msg.FILE_NOT_FOUND.format(kaclFilename), i, count) + entityActionNotPerformedWarning(kvList, Msg.FILE_NOT_FOUND.format(kaclFilename), i, count) continue jsonData = readFile(kaclFilename, mode='r', encoding=UTF8, continueOnError=True) if jsonData is None: @@ -70098,17 +70224,14 @@ def createCSEKeyPair(users): 'privateKeyMetadata': [{'kaclsKeyMetadata': {'kaclsUri': kaclsUri, 'kaclsData': kaclsData}}] } except KeyError: - entityActionNotPerformedWarning([Ent.USER, user, Ent.CSE_KEYPAIR, None], - Msg.JSON_KEY_NOT_FOUND.format(key, kaclFilename), i, count) + entityActionNotPerformedWarning(kvList, Msg.JSON_KEY_NOT_FOUND.format(key, kaclFilename), i, count) continue except (IndexError, SyntaxError, TypeError, ValueError) as e: - entityActionNotPerformedWarning([Ent.USER, user, Ent.CSE_KEYPAIR, None], - Msg.JSON_ERROR.format(str(e), kaclFilename) , i, count) + entityActionNotPerformedWarning(kvList, Msg.JSON_ERROR.format(str(e), kaclFilename) , i, count) continue - kvList = [Ent.USER, user, Ent.CSE_KEYPAIR, None] try: result = callGAPI(gmail.users().settings().cse().keypairs(), 'create', - throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED], + throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED, GAPI.ALREADY_EXISTS], userId='me', body=cseKeyPair) keyPairId = result['keyPairId'] @@ -70116,14 +70239,15 @@ def createCSEKeyPair(users): kvList[-1] = keyPairId if not FJQC.formatJSON: entityActionPerformed(kvList, i, count) + _stripCSEKeyPairSkipObjects(result, skipObjects) _showCSEItem(result, Ent.CSE_KEYPAIR, 'keyPairId', CSE_KEYPAIR_TIME_OBJECTS, i, count, FJQC) elif not addIdentity: writeStdout(f'{keyPairId}\n') if addIdentity: identity = {'keyPairId': keyPairId, 'emailAddress': user if not kpEmail else kpEmail} - kvList = [Ent.USER, user, Ent.CSE_IDENTITY, identity['emailAddress'], Ent.CSE_KEYPAIR, keyPairId] + kvList = [Ent.USER, user, Ent.CSE_KEYPAIR, keyPairId, Ent.CSE_IDENTITY, identity['emailAddress']] result = callGAPI(gmail.users().settings().cse().identities(), 'create', - throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED], + throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED, GAPI.ALREADY_EXISTS], userId='me', body=identity) if not returnIdOnly: if not FJQC.formatJSON: @@ -70131,7 +70255,7 @@ def createCSEKeyPair(users): _showCSEItem(result, Ent.CSE_IDENTITY, 'emailAddress', CSE_IDENTITY_TIME_OBJECTS, i, count, FJQC) else: writeStdout(f'{keyPairId}-{user}\n') - except GAPI.permissionDenied as e: + except (GAPI.permissionDenied, GAPI.alreadyExists) as e: entityActionFailedWarning(kvList, str(e), i, count) except (GAPI.serviceNotAvailable, GAPI.badRequest): entityServiceNotApplicableWarning(Ent.USER, user, i, count) @@ -70144,16 +70268,23 @@ CSE_KEYPAIR_ACTION_FUNCTION_MAP = { } # gam <UserTypeEntity> disable csekeypair <KeyPairID> -# [formatjson] +# [showpem] [showkaclsdata] [formatjson] # gam <UserTypeEntity> enable csekeypair <KeyPairID> -# [formatjson] +# [showpem] [showkaclsdata] [formatjson] # gam <UserTypeEntity> obliterate csekeypair <KeyPairID> -# gam <UserTypeEntity> info csekeypair <KeyPairID> -# [formatjson] +# gam <UserTypeEntity> info csekey3pair <KeyPairID> +# [showpem] [showkaclsdata] [formatjson] def processCSEKeyPair(users): function = CSE_KEYPAIR_ACTION_FUNCTION_MAP[Act.Get()] keyPairId = getString(Cmd.OB_CSE_KEYPAIR_ID) - FJQC = FormatJSONQuoteChar(formatJSONOnly=True) + FJQC = FormatJSONQuoteChar() + skipObjects = _initCSEKeyPairSkipObjects() + while Cmd.ArgumentsRemaining(): + myarg = getArgument() + if _resetCSEKeyPairSkipObjects(myarg, skipObjects): + pass + else: + FJQC.GetFormatJSON(myarg) i, count, users = getEntityArgument(users) for user in users: i += 1 @@ -70163,23 +70294,25 @@ def processCSEKeyPair(users): kvList = [Ent.USER, user, Ent.CSE_KEYPAIR, keyPairId] try: result = callGAPI(gmail.users().settings().cse().keypairs(), function, - throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED], + throwReasons=GAPI.GMAIL_THROW_REASONS+[GAPI.PERMISSION_DENIED, GAPI.INVALID_ARGUMENT, + GAPI.FAILED_PRECONDITION, GAPI.ALREADY_EXISTS], userId='me', keyPairId=keyPairId) if function != 'obliterate': if not FJQC.formatJSON: entityActionPerformed(kvList, i, count) + _stripCSEKeyPairSkipObjects(result, skipObjects) _showCSEItem(result, Ent.CSE_KEYPAIR, 'keyPairId', CSE_KEYPAIR_TIME_OBJECTS, i, count, FJQC) else: entityActionPerformed(kvList, i, count) - except GAPI.permissionDenied as e: + except (GAPI.permissionDenied, GAPI.invalidArgument, GAPI.failedPrecondition, GAPI.alreadyExists) as e: entityActionFailedWarning(kvList, str(e), i, count) except (GAPI.serviceNotAvailable, GAPI.badRequest): entityServiceNotApplicableWarning(Ent.USER, user, i, count) # gam <UserTypeEntity> show csekeypairs -# [formatjson] +# [showpem] [showkaclsdata] [formatjson] # gam <UserTypeEntity> print csekeypairs [todrive <ToDriveAttribute>*] -# [formatjson [quotechar <Character>]] +# [showpem] [showkaclsdata] [formatjson [quotechar <Character>]] def printShowCSEKeyPairs(users): _printShowCSEItems(users, Ent.CSE_KEYPAIR, 'keyPairId', CSE_KEYPAIR_TIME_OBJECTS) @@ -72996,7 +73129,7 @@ USER_ADD_CREATE_FUNCTIONS = { Cmd.ARG_CHATSPACE: createChatSpace, Cmd.ARG_CLASSROOMINVITATION: createClassroomInvitations, Cmd.ARG_CONTACTDELEGATE: processContactDelegates, - Cmd.ARG_CSEIDENTITY: processCSEIdentity, + Cmd.ARG_CSEIDENTITY: createUpdateCSEIdentity, Cmd.ARG_CSEKEYPAIR: createCSEKeyPair, Cmd.ARG_LOOKERSTUDIOPERMISSION: processLookerStudioPermissions, Cmd.ARG_DELEGATE: processDelegates, @@ -73510,7 +73643,7 @@ USER_COMMANDS_WITH_OBJECTS = { Cmd.ARG_CHATMEMBER: deleteUpdateChatMember, Cmd.ARG_CHATMESSAGE: updateChatMessage, Cmd.ARG_CHATSPACE: updateChatSpace, - Cmd.ARG_CSEIDENTITY: processCSEIdentity, + Cmd.ARG_CSEIDENTITY: createUpdateCSEIdentity, Cmd.ARG_LOOKERSTUDIOPERMISSION: processLookerStudioPermissions, Cmd.ARG_DELEGATE: updateDelegates, Cmd.ARG_DRIVEFILE: updateDriveFile,