diff --git a/docs/Bulk-Processing.md b/docs/Bulk-Processing.md index 89a79cd8..5336a19a 100644 --- a/docs/Bulk-Processing.md +++ b/docs/Bulk-Processing.md @@ -69,13 +69,13 @@ gam redirect stdout ./NewStudents.out redirect stderr ./NewStudents.err tbatch N ## CSV files ``` gam csv |-|(gsheet )|(gdoc ) [charset ] [warnifnodata] - [columndelimiter ] [quotechar ] [fields ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] [maxrows ] gam gam loop |-|(gsheet )|(gdoc ) [charset ] [warnifnodata] - [columndelimiter ] [quotechar ] [fields ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] [maxrows ] gam @@ -87,6 +87,7 @@ gam loop |-|(gsheet )|(gdoc ) [charset * `gsheet ` - A Google Sheet and the one or more columns that contain data * `gdoc ` - A Google Doc and the one or more columns that contain data * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `fields ` - The column headings of a CSV file that does not contain column headings. * `(matchfield|skipfield )*` - The criteria to select rows from the CSV file; can be used multiple times; if not specified, all rows are selected diff --git a/docs/CSV-Special-Characters.md b/docs/CSV-Special-Characters.md new file mode 100644 index 00000000..72dc9c08 --- /dev/null +++ b/docs/CSV-Special-Characters.md @@ -0,0 +1,94 @@ +# CSV Special Characters +- [Python CSV documentation](https://docs.python.org/3/library/csv.html#dialects-and-formatting-parameters) + +## Python variables that control CSV file reading/writing: +``` +Dialect.delimiter + A one-character string used to separate fields. + It defaults to ','. + +Dialect.doublequote + Controls how instances of quotechar appearing inside a field should themselves be quoted. + When True, the character is doubled. When False, the escapechar is used as a prefix to the quotechar. + It defaults to True. + +Dialect.escapechar + A one-character string used by the writer to escape the delimiter if quoting is set to QUOTE_NONE and the quotechar if doublequote is False. + On reading, the escapechar removes any special meaning from the following character. + It defaults to None, which disables escaping. + +Dialect.lineterminator + The string used to terminate lines produced by the writer. + It defaults to '\r\n'. + + The reader is hard-coded to recognise either '\r' or '\n' as end-of-line, and ignores lineterminator. + +Dialect.quotechar + A one-character string used to quote fields containing special characters, such as the delimiter or quotechar, or which contain new-line characters. + It defaults to '"'. + +Dialect.quoting + Controls when quotes should be generated by the writer and recognised by the reader. It can take on any of the QUOTE_* constants (see section Module Contents). + It defaults to QUOTE_MINIMAL. +``` + +## GAM variables that control CSV file reading/writing: +``` +csv_input_column_delimiter = , - Dialect.delimiter +csv_input_no_escape_char = true - Dialect.escapechar is set to None if true, '\' if false +csv_input_quote_char = " - Dialect.quotechar +csv_output_column_delimiter = , - Dialect.delimiter +csv_output_no_escape_char = false - Dialect.escapechar is set to None if true, '\' if false +csv_output_line_terminator = lf - Dialect.lineterminator +csv_output_quote_char = " - Dialect.quotechar +todrive_no_escape_char = true - Dialect.escapechar is set to None if true, '\' if false +``` + +GAM sets Dialect.doublequote to true and Dialect.quoting to QUOTE_MINIMAL; there are no variables to change these values. + +## Examples + +### Local file, default settings +With these settings, here are examples of how field values are mapped on output to a local file: +``` +csv_output_column_delimiter = , +csv_output_no_escape_char = false +csv_output_quote_char = " +``` + +| Input | Output | +|-------|--------| +| abc def | abc def | +| abc,def | "abc,def" | +| abc"def | "abc""def" | +| abc\def | abc\\\\def | + +### Local file, modified settings +With these settings, here are examples of how field values are mapped on output to a local file: +``` +csv_output_column_delimiter = , +csv_output_no_escape_char = true +csv_output_quote_char = " +``` + +| Input | Output | +|-------|--------| +| abc def | abc def | +| abc,def | "abc,def" | +| abc"def | "abc""def" | +| abc\def | abc\def | + +### todrive, default settings +With these settings, here are examples of how field values are mapped on output to todrive +``` +csv_output_column_delimiter = , +todrive_no_escape_char = true +csv_output_quote_char = " +``` + +| Input | Output | +|-------|--------| +| abc def | abc def | +| abc,def | "abc,def" | +| abc"def | "abc""def" | +| abc\def | abc\def | diff --git a/docs/Cloud-Identity-Devices.md b/docs/Cloud-Identity-Devices.md index d395d1ca..13f50b6b 100644 --- a/docs/Cloud-Identity-Devices.md +++ b/docs/Cloud-Identity-Devices.md @@ -36,8 +36,9 @@ See: https://support.google.com/a/answer/7549103 ::= "(,)*" ::= devices/ + ::= "(,)*" ::= - | + | devicesn | (query:)|(query ) ::= android|chrome_os|google_sync|linux|mac_os|windows ::= devices//deviceUsers/ diff --git a/docs/Collections-of-ChromeOS-Devices.md b/docs/Collections-of-ChromeOS-Devices.md index f3e714ef..c476917b 100644 --- a/docs/Collections-of-ChromeOS-Devices.md +++ b/docs/Collections-of-ChromeOS-Devices.md @@ -90,7 +90,7 @@ (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -100,7 +100,7 @@ (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -117,7 +117,7 @@ (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -128,7 +128,7 @@ (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ] [quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -263,7 +263,7 @@ croscsvfile (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -276,6 +276,7 @@ croscsvfile * `gcsdoc(:)+ ` - A Google Cloud Storage Bucket Object and the one or more columns that contain ChromeOS deviceIds * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings @@ -290,7 +291,7 @@ croscsvfile_sn (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -303,6 +304,7 @@ croscsvfile_sn * `gcsdoc(:)+ ` - A Google Cloud Storage Bucket Object and the one or more columns that contain ChromeOS serial numbers * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings @@ -334,7 +336,7 @@ csvdatafile (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -348,6 +350,7 @@ csvdatafile * `gcsdoc(:)+ ` - A Google Cloud Storage Bucket Object and the one or more columns that contain ChromeOS deviceIds * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings @@ -363,7 +366,7 @@ csvkmd (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ] [quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -376,6 +379,7 @@ csvkmd * `gdoc ` - A Google Doc containing rows with columns of the type of item specified * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings diff --git a/docs/Collections-of-Items.md b/docs/Collections-of-Items.md index 37cd7830..2c97e8bd 100644 --- a/docs/Collections-of-Items.md +++ b/docs/Collections-of-Items.md @@ -63,7 +63,7 @@ A CSV file with one or more columns per row that contain Items. (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -75,6 +75,7 @@ A CSV file with one or more columns per row that contain Items. * `gcsdoc(:)+ ` - A Google Cloud Storage Bucket Object and the one or more columns that contain Items * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings @@ -90,7 +91,7 @@ A CSV file with a key column that contains an Item and optional subkey and data (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ] [quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -102,6 +103,7 @@ A CSV file with a key column that contains an Item and optional subkey and data * `gcscsv ` - A Google Cloud Storage Bucket Object containing rows with columns of items * `gcsdoc ` - A Google Cloud Storage Bucket Object containing rows with columns of items * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings diff --git a/docs/Collections-of-Users.md b/docs/Collections-of-Users.md index 16bc1429..3a18f898 100644 --- a/docs/Collections-of-Users.md +++ b/docs/Collections-of-Users.md @@ -127,7 +127,7 @@ (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ][quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -148,7 +148,7 @@ (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ][quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -161,7 +161,7 @@ (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ][quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -360,7 +360,7 @@ csvfile (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ][quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -373,7 +373,8 @@ csvfile * `gcsdoc(:)+ ` - A Google Cloud Storage Bucket Object and the one or more columns that contain Users * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used -* `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used +* `quotechar ` - The column quote character is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings * `(matchfield|skipfield )*` - The criteria to select rows from the CSV file; can be used multiple times; if not specified, all rows are selected @@ -408,7 +409,7 @@ csvdatafile (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ][quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -422,7 +423,8 @@ csvdatafile * `gcsdoc(:)+ ` - A Google Cloud Storage Bucket Object and the one or more columns contain the type of item specified * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used -* `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used +* `quotechar ` - The column quote character is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings * `(matchfield|skipfield )*` - The criteria to select rows from the CSV file; can be used multiple times; if not specified, all rows are selected @@ -439,7 +441,7 @@ csvkmd (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ][quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -454,7 +456,8 @@ csvkmd * `gcsdoc ` - A Google Cloud Storage Bucket Object with columns of the type of item specified * `warnifnodata` - Issue message 'No CSV file data found' and exit with return code 60 if there is no data selected from the file * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used -* `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used +* `quotechar ` - The column quote character is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `endcsv` - Use this option to signal the end of the csvfile parameters in the case that the next argument on the command line is `fields` but is specifying the output field list for the command not column headings * `fields ` - The column headings of a CSV file that does not contain column headings * `(keyfield [keypattern ] [keyvalue ] [delimiter ])+` diff --git a/docs/GamUpdates.md b/docs/GamUpdates.md index 0a9a0314..c807c1e4 100644 --- a/docs/GamUpdates.md +++ b/docs/GamUpdates.md @@ -10,6 +10,29 @@ Add the `-s` option to the end of the above commands to suppress creating the `g See [Downloads](https://github.com/taers232c/GAMADV-XTD3/wiki/Downloads) for Windows or other options, including manual installation +### 6.66.02 + +Updated device commmands to handle the following error caused by an invalid query. +``` +ERROR: 400: invalidArgument - Request contains an invalid argument. +``` + +Added fields `deviceid` and `hostname` to ``. + +### 6.66.01 + +Added the following variables to gam.cfg that allow control over whether `\` is used as an escape character +when reading/writing CSV files. +``` +csv_input_no_escape_char - default value True +csv_output_no_escape_char - default value False +todrive_no_escape_char - default value True +``` +When the value is True, `\` is ignored as an escape character; when the value is False, +`\\` on input is converted to `\`, `\` on output is converted to `\\`. + +* See: https://github.com/taers232c/GAMADV-XTD3/wiki/CSV-Special-Characters + ### 6.66.00 Added support for `Focus Time` and `Out of Office` status events in user's primary calendars. diff --git a/docs/How-to-Upgrade-from-Standard-GAM.md b/docs/How-to-Upgrade-from-Standard-GAM.md index ec41c774..93a9d86a 100644 --- a/docs/How-to-Upgrade-from-Standard-GAM.md +++ b/docs/How-to-Upgrade-from-Standard-GAM.md @@ -334,7 +334,7 @@ writes the credentials into the file oauth2.txt. admin@server:/Users/admin/bin/gamadv-xtd3$ rm -f /Users/admin/GAMConfig/oauth2.txt admin@server:/Users/admin/bin/gamadv-xtd3$ ./gam version WARNING: Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, Item: oauth2_txt, Value: /Users/admin/GAMConfig/oauth2.txt, Not Found -GAMADV-XTD3 6.66.00 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.66.02 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.10.8 64-bit final MacOS High Sierra 10.13.6 x86_64 @@ -1002,7 +1002,7 @@ writes the credentials into the file oauth2.txt. C:\GAMADV-XTD3>del C:\GAMConfig\oauth2.txt C:\GAMADV-XTD3>gam version WARNING: Config File: C:\GAMConfig\gam.cfg, Section: DEFAULT, Item: oauth2_txt, Value: C:\GAMConfig\oauth2.txt, Not Found -GAMADV-XTD3 6.66.00 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.66.02 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.0 64-bit final Windows-10-10.0.17134 AMD64 diff --git a/docs/Meta-Commands-and-File-Redirection.md b/docs/Meta-Commands-and-File-Redirection.md index 123dfe7e..0a88475e 100644 --- a/docs/Meta-Commands-and-File-Redirection.md +++ b/docs/Meta-Commands-and-File-Redirection.md @@ -112,7 +112,7 @@ You can redirect stdout and stderr to null and stderr can be redirected to stdou ``` ::= redirect csv [multiprocess] [append] [noheader] [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [timestampcolumn ] [todrive *] | redirect stdout [multiprocess] [append] | @@ -141,8 +141,11 @@ subsequent GAM commands specify `append noheader`. The `charset ` subargument sets the character set of the CSV file; the default is the value of `charset` in `gam.cfg` which defaults to UTF-8. -The `columndelimiter ` sets the intercolumn delimiter of the CSV file; the default value -is the value of csv_output_column_delimiter` in `gam.cfg` which defaults to comma. +The `columndelimiter ` subargument sets the intercolumn delimiter of the CSV file; the default value +is the value of `csv_output_column_delimiter` in `gam.cfg` which defaults to comma. + +The `noescapechar ` subargument controls whether `\` is used as an escape character when writing the CSV file; the default value +is the value of `csv_output_no_escape_char` in `gam.cfg` which defaults to False. The `quotechar ` subargument sets the character used to quote fields in the CSV file that contaim special charactere; the default value is the value of `csv_output_quote_char` in `gam.cfg` diff --git a/docs/Tag-Replace.md b/docs/Tag-Replace.md index 76206373..fa87b9db 100644 --- a/docs/Tag-Replace.md +++ b/docs/Tag-Replace.md @@ -167,7 +167,7 @@ gam create user * gam update user [updateprimaryemail ] [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] [clearschema ] [clearschema .] [createifnotfound] [notify ] [subject ] diff --git a/docs/Todrive.md b/docs/Todrive.md index 9c24fa56..1ef95688 100644 --- a/docs/Todrive.md +++ b/docs/Todrive.md @@ -132,6 +132,9 @@ todrive_nobrowser todrive_noemail Enable/disable sending an email when todrive is specified Default: True +todrive_no_escape_char + When writing a CSV file to Google Drive, should `\` be ignored as an escape character. + Default: True todrive_parent Parent folder for CSV files when todrive is specified; can be id: or @@ -180,6 +183,7 @@ direct the uploaded file to a particular user and location and add a timestamp t (tdlocale )| (tdnobrowser [])| (tdnoemail [])| + (tdnoescapechar [])| (tdparent (id:)|)| (tdretaintitle [])| (tdshare commenter|reader|writer)| @@ -233,6 +237,9 @@ If `tdfileid ` is not specified, a new file is created. * `tdnobrowser` - If False, a browser is opened to view the file uploaded to Google Drive; if not specified, the `todrive_nobrowser` value from gam.cfg is used. * `tdnoemail` - If False, an email is sent to `tduser` informing them of name and URL of the uploaded file; if not specified, the `todrive_noemail` value from gam.cfg is used. +## Escape character +* `tdnoescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `todrive_no_escape_char` from `gam.cfg` will be used + ## Local copy * `tdlocalcopy` - Should a local copy of the CSV file be saved in addition to the file uploaded to Google Drive; if not specified, the `todrive_localcopy` value from gam.cfg is used. @@ -265,7 +272,7 @@ If `tdfileid ` is not specified, a new file is created. You can specify `todrive` options in conjunction with `redirect csv`. ``` redirect csv [multiprocess] [append] [noheader] [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [todrive *] ``` If you are doing `redirect csv multiprocess`, it is more efficient to specify `todrive *` as part of @@ -274,6 +281,8 @@ the redirect as verification of the `todrive` settings, which can invole several `columndelimiter ` and `quotechar ` will not generally be used with `todrive` as Google Sheets only recognizes `,` as the column delimiter and `"` as the quote character. +`noescapechar true` will generally be used with `todrive` as Google Sheets does not recognize `\\` as an escaped `\`. + ## Examples Generate a list of user IDs and names, title the file "User IDs and Names", upload it to the "GAM Reports" folder of usermgr@domain.com, add a timestamp to the title. ``` diff --git a/docs/Users-Calendars-Events.md b/docs/Users-Calendars-Events.md index 698d4e59..551da5ef 100644 --- a/docs/Users-Calendars-Events.md +++ b/docs/Users-Calendars-Events.md @@ -48,7 +48,7 @@ (gdoc )| (gcscsv )| (gcsdoc )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] ::= @@ -57,7 +57,7 @@ (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] diff --git a/docs/Users-Drive-Activity-Settings.md b/docs/Users-Drive-Activity-Settings.md index c40cef10..463c1af1 100644 --- a/docs/Users-Drive-Activity-Settings.md +++ b/docs/Users-Drive-Activity-Settings.md @@ -78,7 +78,7 @@ gam print|show driveactivity [v2] [todrive * yesterday|today|thismonth|(previousmonths )] [action|actions [not] ] [consolidationstrategy legacy|none] - [idmapfile |(gsheet ) [charset ] [columndelimiter ] [quotechar ]] + [idmapfile |(gsheet ) [charset ] [columndelimiter ] [noescapechar ] [quotechar ]] [formatjson [quotechar ]] ``` By default, Drive Activity API v2 is used; the `v2` option is ignored. diff --git a/docs/Users.md b/docs/Users.md index 303cb680..657ebebe 100644 --- a/docs/Users.md +++ b/docs/Users.md @@ -598,7 +598,7 @@ gam update user [ignorenullpassword] * [verifynotinvitable|alwaysevict] [noactionifalias] [updateprimaryemail ] [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] [clearschema ] [clearschema .] [createifnotfound] [notfoundpassword random|] @@ -619,7 +619,7 @@ gam update users [ignorenullpassword] * [verifynotinvitable|alwaysevict] [noactionifalias] [updateprimaryemail ] [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] [clearschema ] [clearschema .] [createifnotfound] [notfoundpassword random|] @@ -640,7 +640,7 @@ gam update users [ignorenullpassword] * [verifynotinvitable|alwaysevict] [noactionifalias] [updateprimaryemail ] [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] [clearschema ] [clearschema .] [createifnotfound] [notfoundpassword random|] @@ -814,11 +814,12 @@ No update is performed if a user does not belong to any group in the CSV file or ``` [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] ``` * `` - A CSV file containing rows with columns of items * `columndelimiter ` - Columns are separated by ``; if not specified, the value of `csv_input_column_delimiter` from `gam.cfg` will be used +* `noescapechar ` - Should `\` be ignored as an escape character; if not specified, the value of `csv_input_no_escape_char` from `gam.cfg` will be used * `quotechar ` - The column quote characer is ``; if not specified, the value of `csv_input_quote_char` from `gam.cfg` will be used * `fields ` - The column headings of a CSV file that does not contain column headings * `keyfield ` - The column heading of the group column; the default is Group diff --git a/docs/Version-and-Help.md b/docs/Version-and-Help.md index a99493d1..98ad3aaf 100644 --- a/docs/Version-and-Help.md +++ b/docs/Version-and-Help.md @@ -4,7 +4,7 @@ Print the current version of Gam with details ``` gam version -GAMADV-XTD3 6.66.00 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.66.02 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.0 64-bit final MacOS Monterey 12.7 x86_64 @@ -16,7 +16,7 @@ Time: 2023-06-02T21:10:00-07:00 Print the current version of Gam with details and time offset information ``` gam version timeoffset -GAMADV-XTD3 6.66.00 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.66.02 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.0 64-bit final MacOS Monterey 12.7 x86_64 @@ -28,7 +28,7 @@ Your system time differs from www.googleapis.com by less than 1 second Print the current version of Gam with extended details and SSL information ``` gam version extended -GAMADV-XTD3 6.66.00 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.66.02 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs Python 3.12.0 64-bit final MacOS Monterey 12.7 x86_64 @@ -65,7 +65,7 @@ MacOS High Sierra 10.13.6 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 Version Check: Current: 5.35.08 - Latest: 6.66.00 + Latest: 6.66.02 echo $? 1 ``` @@ -73,7 +73,7 @@ echo $? Print the current version number without details ``` gam version simple -6.66.00 +6.66.02 ``` In Linux/MacOS you can do: ``` @@ -83,7 +83,7 @@ echo $VER Print the current version of Gam and address of this Wiki ``` gam help -GAM 6.66.00 - https://github.com/taers232c/GAMADV-XTD3 +GAM 6.66.02 - https://github.com/taers232c/GAMADV-XTD3 Ross Scroggs Python 3.12.0 64-bit final MacOS Monterey 12.7 x86_64 diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md index f2ab4713..bc08ed4f 100644 --- a/docs/_Sidebar.md +++ b/docs/_Sidebar.md @@ -43,6 +43,7 @@ Command Processing * [Command Line Parsing](Command-Line-Parsing) * [Command Logging and Progress](Command-Logging-Progress) * [Command data from Google Docs/Sheets/Storage](Command-Data-From-Google-Docs-Sheets-Storage) +* [CSV Special Characters](CSV-Special-Characters) * [CSV Input Filtering](CSV-Input-Filtering) * [CSV Output Filtering](CSV-Output-Filtering) * [Meta Commands and File Redirection](Meta-Commands-and-File-Redirection) diff --git a/docs/gam.cfg.md b/docs/gam.cfg.md index e4f1b6c5..d3f9ef21 100644 --- a/docs/gam.cfg.md +++ b/docs/gam.cfg.md @@ -148,6 +148,10 @@ csv_input_column_delimiter All places where an input CSV file can be specified have an argument columndelimiter that can override this value. Default: ',' +csv_input_no_escape_char + When reading a CSV file, should `\` be ignored as an escape character. + Set this to False if the input file data was written using `\` as an escape character. + Default: True csv_input_quote_char A one-character string used to quote fields containing special characters, such as the csv_input_column_delimiter or csv_input_quote_char, or newline characters. @@ -209,11 +213,15 @@ csv_output_header_force for inclusion in the CSV file written by a gam print command Default: '' csv_output_line_terminator - Allowed values: cr, lf, crlf +p Allowed values: cr, lf, crlf Designates character(s) used to terminate the lines of a CSV file. For Linux and Mac OS, this would typically be lf. For Windows, this would typically be crlf. Default: lf +csv_output_no_escape_char + When writing a CSV file, should `\` be ignored as an escape character. + Set this to True if the output file data is to be read by a non-Python program. + Default: False csv_output_quote_char A one-character string used to quote fields containing special characters, such as the csv_output_column_delimiter or csv_output_quote_char @@ -527,6 +535,9 @@ todrive_nobrowser todrive_noemail Enable/disable sending an email when todrive is specified Default: True +todrive_no_escape_char + When writing a CSV file to Google Drive, should `\` be ignored as an escape character. + Default: True todrive_parent Parent folder for CSV files when todrive is specified; can be id: or @@ -605,6 +616,7 @@ Section: DEFAULT config_dir = /Users/admin/.gam contact_max_results = 100 csv_input_column_delimiter = , + csv_input_no_escape_char = true csv_input_quote_char = '"' csv_input_row_drop_filter = '' csv_input_row_drop_filter = '' @@ -619,6 +631,7 @@ Section: DEFAULT csv_output_header_filter = '' csv_output_header_force = '' csv_output_line_terminator = lf + csv_output_no_escape_char = false csv_output_quote_char = '"' csv_output_row_drop_filter = '' csv_output_row_drop_filter_mode = anymatch @@ -689,6 +702,7 @@ Section: DEFAULT todrive_locale = '' todrive_nobrowser = false todrive_noemail = true + todrive_no_escape_char = true todrive_parent = root todrive_sheet_timeformat = '' todrive_sheet_timestamp = false @@ -797,6 +811,7 @@ clock_skew_in_seconds = 10 config_dir = /Users/admin/.gam contact_max_results = 100 csv_input_column_delimiter = , +csv_input_no_escape_char = true csv_input_quote_char = '"' csv_input_row_drop_filter = '' csv_input_row_filter = '' @@ -807,6 +822,7 @@ csv_output_header_drop_filter = '' csv_output_header_filter = '' csv_output_header_force = '' csv_output_line_terminator = lf +csv_output_no_escape_char = false csv_output_quote_char = '"' csv_output_row_drop_filter = csv_output_row_filter = '' @@ -873,6 +889,7 @@ todrive_localcopy = false todrive_locale = '' todrive_nobrowser = false todrive_noemail = true +todrive_no_escape_char = true todrive_parent = root todrive_sheet_timeformat = '' todrive_sheet_timestamp = false diff --git a/src/GamCommands.txt b/src/GamCommands.txt index 167a5f98..a4f25cc1 100644 --- a/src/GamCommands.txt +++ b/src/GamCommands.txt @@ -609,6 +609,7 @@ If an item contains spaces, it should be surrounded by ". (tdlocale )| (tdnobrowser [])| (tdnoemail [])| + (tdnoescapechar [])| (tdparent (id:)|)| (tdretaintitle [])| (tdshare commenter|reader|writer)| @@ -788,7 +789,7 @@ Specify a collection of ChromeOS devices by directly specifying them or by speci (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -798,7 +799,7 @@ Specify a collection of ChromeOS devices by directly specifying them or by speci (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -815,7 +816,7 @@ Specify a collection of ChromeOS devices by directly specifying them or by speci (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -826,7 +827,7 @@ Specify a collection of ChromeOS devices by directly specifying them or by speci (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ] [quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -874,7 +875,7 @@ Specify a collection of Users by directly specifying them or by specifiying item (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -895,7 +896,7 @@ Specify a collection of Users by directly specifying them or by specifiying item (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ])| @@ -908,7 +909,7 @@ Specify a collection of Users by directly specifying them or by specifiying item (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ] [quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -925,7 +926,7 @@ Specify a collection of items by directly specifying them; the item type is dete (gdoc )| (gcscsv )| (gcsdoc )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] ::= @@ -941,7 +942,7 @@ Specify a collection of items by directly specifying them; the item type is dete (gdoc(:)+ )| (gcscsv(:)+ )| (gcsdoc(:)+ )) - [warnifnodata] [columndelimiter ] [quotechar ] + [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [endcsv|(fields )] (matchfield|skipfield )* [delimiter ] @@ -956,7 +957,7 @@ Specify a collection of items by directly specifying them; the item type is dete (gdoc )| (gcscsv )| (gcsdoc )) - [charset ] [columndelimiter ] [quotechar ] [fields ]) + [charset ] [columndelimiter ] [noescapechar ] [quotechar ] [fields ]) keyfield [keypattern ] [keyvalue ] [delimiter ] subkeyfield [keypattern ] [keyvalue ] [delimiter ] (matchfield|skipfield )* @@ -1237,7 +1238,7 @@ If the pattern {{Section}} appears in , it will be replaced with the n For redirect csv, the optional arguments must appear in the order shown. ::= redirect csv [multiprocess] [append] [noheader] [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [timestampcolumn ] [todrive *] | redirect stdout [multiprocess] [append] | @@ -1284,13 +1285,13 @@ gam tbatch [showcmds []] [charset ] [delimiter ]) gam csv [warnifnodata] - [columndelimiter ] [quotechar ] [fields ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] [maxrows ] gam gam loop [warnifnodata] - [columndelimiter ] [quotechar ] [fields ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] [maxrows ] gam @@ -3766,9 +3767,12 @@ gam show cigroup-members # Cloud Identity Devices ::= devices/ - ::= devices//deviceUsers/ + ::= "(,)*" + ::= + | devicesn | + (query:)|(query ) ::= android|chrome_os|google_sync|ios|linux|mac_os|windows - + ::= devices//deviceUsers/ ::= androidspecificattributes| assettag| @@ -3778,11 +3782,13 @@ gam show cigroup-members buildnumber| compromisedstate| createtime| + deviceid| devicetype| enableddeveloperoptions| enabledusbdebugging| endpointverificationspecificattributes| encryptionstate| + hostname| imei| kernelversion| lastsynctime| @@ -5203,7 +5209,7 @@ gam update user [ignorenullpassword] * [verifynotinvitable|alwaysevict] [noactionifalias] [updateprimaryemail ] [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] [immutableous ]| [clearschema ] [clearschema .] @@ -5239,7 +5245,7 @@ gam update users [ignorenullpassword] * [verifynotinvitable] [noactionifalias] [updateprimaryemail ] [updateoufromgroup [charset ] - [columndelimiter ] [quotechar ] + [columndelimiter ] [noescapechar ] [quotechar ] [fields ] [keyfield ] [datafield ]] [clearschema ] [clearschema .] [createifnotfound] [notfoundpassword (random [])|blocklogin|] diff --git a/src/GamUpdate.txt b/src/GamUpdate.txt index 4dce897b..3821abd4 100644 --- a/src/GamUpdate.txt +++ b/src/GamUpdate.txt @@ -2,6 +2,29 @@ Merged GAM-Team version +6.66.02 + +Updated device commmands to handle the following error caused by an invalid query. +``` +ERROR: 400: invalidArgument - Request contains an invalid argument. +``` + +Added fields `deviceid` and `hostname` to ``. + +6.66.01 + +Added the following variables to gam.cfg that allow control over whether `\` is used as an escape character +when reading/writing CSV files. +``` +csv_input_no_escape_char - default value True +csv_output_no_escape_char - default value False +todrive_no_escape_char - default value True +``` +When the value is True, `\` is ignored as an escape character; when the value is False, +`\\` on input is converted to `\`, `\` on output is converted to `\\`. + +* See: https://github.com/taers232c/GAMADV-XTD3/wiki/CSV-Special-Characters + 6.66.00 Added support for `Focus Time` and `Out of Office` status events in user's primary calendars. diff --git a/src/gam/__init__.py b/src/gam/__init__.py index 23d2bb9a..fbb126ac 100755 --- a/src/gam/__init__.py +++ b/src/gam/__init__.py @@ -3173,6 +3173,10 @@ def openCSVFileReader(filename, fieldnames=None): columnDelimiter = getCharacter() else: columnDelimiter = GC.Values[GC.CSV_INPUT_COLUMN_DELIMITER] + if checkArgumentPresent('noescapechar'): + noEscapeChar = getBoolean() + else: + noEscapeChar = GC.Values[GC.CSV_INPUT_NO_ESCAPE_CHAR] if checkArgumentPresent('quotechar'): quotechar = getCharacter() else: @@ -3180,7 +3184,10 @@ def openCSVFileReader(filename, fieldnames=None): if not checkArgumentPresent('endcsv') and checkArgumentPresent('fields'): fieldnames = shlexSplitList(getString(Cmd.OB_FIELD_NAME_LIST)) try: - csvFile = csv.DictReader(f, fieldnames=fieldnames, delimiter=columnDelimiter, quotechar=quotechar) + csvFile = csv.DictReader(f, fieldnames=fieldnames, + delimiter=columnDelimiter, + escapechar='\\' if not noEscapeChar else None, + quotechar=quotechar) return (f, csvFile, csvFile.fieldnames if csvFile.fieldnames is not None else []) except (csv.Error, UnicodeDecodeError, UnicodeError) as e: systemErrorExit(FILE_ERROR_RC, e) @@ -3343,6 +3350,8 @@ def SetGlobalVariables(): value = codecs.escape_decode(bytes(_stripStringQuotes(GM.Globals[GM.PARSER].get(sectionName, itemName)), UTF8))[0].decode(UTF8) if not value and (itemName == 'csv_output_field_delimiter'): return ' ' + if not value and (itemName in {'csv_input_escape_char', 'csv_output_escape_char'}): + return None if len(value) == 1: return value _printValueError(sectionName, itemName, f'"{value}"', f'{Msg.EXPECTED}: {integerLimits(1, 1, Msg.STRING_LENGTH)}') @@ -4033,7 +4042,7 @@ def SetGlobalVariables(): if checkArgumentPresent(Cmd.MULTIPROCESSEXIT_CMD): _setMultiprocessExit() # redirect csv [multiprocess] [append] [noheader] [charset ] -# [columndelimiter ] [quotechar ]] +# [columndelimiter ] [noescapechar ] [quotechar ]] # [timestampcolumn ] # [todrive *] # redirect stdout [multiprocess] [append] @@ -4053,6 +4062,8 @@ def SetGlobalVariables(): GM.Globals[GM.CSV_OUTPUT_COLUMN_DELIMITER] = GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER] = getCharacter() if checkArgumentPresent('quotechar'): GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR] = GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR] = getCharacter() + if checkArgumentPresent('noescapechar'): + GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR] = getBoolean() if checkArgumentPresent('timestampcolumn'): GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN] = getString(Cmd.OB_STRING, minLen=0) _setCSVFile(filename, mode, encoding, writeHeader, multi) @@ -7601,7 +7612,9 @@ class CSVPrintFile(): self.SetColumnDelimiter(GM.Globals[GM.CSV_OUTPUT_COLUMN_DELIMITER]) if GM.Globals.get(GM.CSV_OUTPUT_QUOTE_CHAR) is None: GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR] = GC.Values.get(GC.CSV_OUTPUT_QUOTE_CHAR, '"') - self.SetEscapeChar('\\') + if GM.Globals.get(GM.CSV_OUTPUT_NO_ESCAPE_CHAR) is None: + GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = GC.Values.get(GC.CSV_OUTPUT_NO_ESCAPE_CHAR, False) + self.SetNoEscapeChar(GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR]) self.SetQuoteChar(GM.Globals[GM.CSV_OUTPUT_QUOTE_CHAR]) if GM.Globals.get(GM.CSV_OUTPUT_TIMESTAMP_COLUMN) is None: GM.Globals[GM.CSV_OUTPUT_TIMESTAMP_COLUMN] = GC.Values.get(GC.CSV_OUTPUT_TIMESTAMP_COLUMN, '') @@ -7812,6 +7825,7 @@ class CSVPrintFile(): 'backupSheetEntity': None, 'copySheetEntity': None, 'locale': GC.Values[GC.TODRIVE_LOCALE], 'timeZone': GC.Values[GC.TODRIVE_TIMEZONE], 'timestamp': GC.Values[GC.TODRIVE_TIMESTAMP], 'timeformat': GC.Values[GC.TODRIVE_TIMEFORMAT], + 'noescapechar': GC.Values[GC.TODRIVE_NO_ESCAPE_CHAR], 'daysoffset': None, 'hoursoffset': None, 'sheettimestamp': GC.Values[GC.TODRIVE_SHEET_TIMESTAMP], 'sheettimeformat': GC.Values[GC.TODRIVE_SHEET_TIMEFORMAT], 'sheetdaysoffset': None, 'sheethoursoffset': None, @@ -7888,6 +7902,8 @@ class CSVPrintFile(): self.todrive['nobrowser'] = getBoolean() elif myarg == 'tdnoemail': self.todrive['noemail'] = getBoolean() + elif myarg == 'tdnoescapechar': + self.todrive['noescapechar'] = getBoolean() elif myarg == 'tdshare': self.todrive['share']['emailAddress'] = normalizeEmailAddressOrUID(getString(Cmd.OB_EMAIL_ADDRESS)) self.todrive['share']['type'] = 'user' @@ -8119,8 +8135,8 @@ class CSVPrintFile(): def SetColumnDelimiter(self, columnDelimiter): self.columnDelimiter = columnDelimiter - def SetEscapeChar(self, escapeChar): - self.escapeChar = escapeChar + def SetNoEscapeChar(self, noEscapeChar): + self.noEscapeChar = noEscapeChar def SetQuoteChar(self, quoteChar): self.quoteChar = quoteChar @@ -8318,11 +8334,11 @@ class CSVPrintFile(): stderrErrorMsg(e) return False - def setDialect(lineterminator): + def setDialect(lineterminator, noEscapeChar): writerDialect = { 'delimiter': self.columnDelimiter, 'doublequote': True, - 'escapechar': self.escapeChar, + 'escapechar': '\\' if not noEscapeChar else None, 'lineterminator': lineterminator, 'quotechar': self.quoteChar, 'quoting': csv.QUOTE_MINIMAL, @@ -8332,7 +8348,7 @@ class CSVPrintFile(): def writeCSVToStdout(): csvFile = StringIOobject() - writerDialect = setDialect('\n') + writerDialect = setDialect('\n', self.noEscapeChar) writer = csv.DictWriter(csvFile, titlesList, extrasaction=extrasaction, **writerDialect) if writeCSVData(writer): try: @@ -8347,7 +8363,7 @@ class CSVPrintFile(): encoding=GM.Globals[GM.CSVFILE][GM.REDIRECT_ENCODING], errors='backslashreplace', continueOnError=True) if csvFile: - writerDialect = setDialect(str(GC.Values[GC.CSV_OUTPUT_LINE_TERMINATOR])) + writerDialect = setDialect(str(GC.Values[GC.CSV_OUTPUT_LINE_TERMINATOR]), self.noEscapeChar) writer = csv.DictWriter(csvFile, titlesList, extrasaction=extrasaction, **writerDialect) writeCSVData(writer) closeFile(csvFile) @@ -8363,7 +8379,7 @@ class CSVPrintFile(): csvFile = TemporaryFile(mode='w+', encoding=UTF8) else: csvFile = StringIOobject() - writerDialect = setDialect('\n') + writerDialect = setDialect('\n', self.todrive['noescapechar']) writer = csv.DictWriter(csvFile, titlesList, extrasaction=extrasaction, **writerDialect) if writeCSVData(writer): if ((self.todrive['title'] is None) or @@ -8639,7 +8655,7 @@ class CSVPrintFile(): GM.Globals[GM.CSVFILE][GM.REDIRECT_QUEUE].put((GM.REDIRECT_QUEUE_CSVPF, (self.titlesList, self.sortTitlesList, self.indexedTitles, self.formatJSON, self.JSONtitlesList, - self.columnDelimiter, self.quoteChar, + self.columnDelimiter, self.noEscapeChar, self.quoteChar, self.timestampColumn, self.mapDrive3Titles, self.fixPaths, @@ -9244,6 +9260,7 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo Cmd = glclargs.GamCLArgs() else: csvPF.SetColumnDelimiter(GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER]) + csvPF.SetNoEscapeChar(GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR]) csvPF.SetQuoteChar(GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR]) csvPF.SetTimestampColumn(GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN]) csvPF.SetHeaderFilter(GC.Values[GC.CSV_OUTPUT_HEADER_FILTER]) @@ -9265,13 +9282,14 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo csvPF.SetFormatJSON(dataItem[3]) csvPF.AddJSONTitles(dataItem[4]) csvPF.SetColumnDelimiter(dataItem[5]) - csvPF.SetQuoteChar(dataItem[6]) - csvPF.SetTimestampColumn(dataItem[7]) - csvPF.SetMapDrive3Titles(dataItem[8]) - csvPF.SetFixPaths(dataItem[9]) - csvPF.SetNodataFields(dataItem[10], dataItem[11], dataItem[12], dataItem[13], dataItem[14]) - csvPF.SetShowPermissionsLast(dataItem[15]) - csvPF.SetZeroBlankMimeTypeCounts(dataItem[16]) + csvPF.SetNoEscapeChar(dataItem[6]) + csvPF.SetQuoteChar(dataItem[7]) + csvPF.SetTimestampColumn(dataItem[8]) + csvPF.SetMapDrive3Titles(dataItem[9]) + csvPF.SetFixPaths(dataItem[10]) + csvPF.SetNodataFields(dataItem[11], dataItem[12], dataItem[13], dataItem[14], dataItem[15]) + csvPF.SetShowPermissionsLast(dataItem[16]) + csvPF.SetZeroBlankMimeTypeCounts(dataItem[17]) elif dataType == GM.REDIRECT_QUEUE_DATA: csvPF.rows.extend(dataItem) elif dataType == GM.REDIRECT_QUEUE_ARGS: @@ -9284,6 +9302,7 @@ def CSVFileQueueHandler(mpQueue, mpQueueStdout, mpQueueStderr, csvPF, datetimeNo elif dataType == GM.REDIRECT_QUEUE_VALUES: GC.Values = dataItem csvPF.SetColumnDelimiter(GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER]) + csvPF.SetNoEscapeChar(GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR]) csvPF.SetQuoteChar(GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR]) csvPF.SetTimestampColumn(GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN]) csvPF.SetHeaderFilter(GC.Values[GC.CSV_OUTPUT_HEADER_FILTER]) @@ -9427,7 +9446,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, debugLevel, todrive, printAguDomains, printCrosOUs, printCrosOUsAndChildren, output_dateformat, output_timeformat, - csvColumnDelimiter, csvQuoteChar, + csvColumnDelimiter, csvNoEscapeChar, csvQuoteChar, csvTimestampColumn, csvHeaderFilter, csvHeaderDropFilter, csvHeaderForce, @@ -9449,6 +9468,7 @@ def ProcessGAMCommandMulti(pid, numItems, logCmd, mpQueueCSVFile, mpQueueStdout, GM.Globals[GM.CSV_SUBKEY_FIELD] = None GM.Globals[GM.CSV_DATA_FIELD] = None GM.Globals[GM.CSV_OUTPUT_COLUMN_DELIMITER] = csvColumnDelimiter + GM.Globals[GM.CSV_OUTPUT_NO_ESCAPE_CHAR] = csvNoEscapeChar GM.Globals[GM.CSV_OUTPUT_HEADER_DROP_FILTER] = csvHeaderDropFilter[:] GM.Globals[GM.CSV_OUTPUT_HEADER_FILTER] = csvHeaderFilter[:] GM.Globals[GM.CSV_OUTPUT_HEADER_FORCE] = csvHeaderForce[:] @@ -9658,6 +9678,7 @@ def MultiprocessGAMCommands(items, showCmds): GC.Values[GC.PRINT_CROS_OUS], GC.Values[GC.PRINT_CROS_OUS_AND_CHILDREN], GC.Values[GC.OUTPUT_DATEFORMAT], GC.Values[GC.OUTPUT_TIMEFORMAT], GC.Values[GC.CSV_OUTPUT_COLUMN_DELIMITER], + GC.Values[GC.CSV_OUTPUT_NO_ESCAPE_CHAR], GC.Values[GC.CSV_OUTPUT_QUOTE_CHAR], GC.Values[GC.CSV_OUTPUT_TIMESTAMP_COLUMN], GC.Values[GC.CSV_OUTPUT_HEADER_FILTER], @@ -22261,7 +22282,7 @@ def printShowUserPeopleContactGroups(users): entityTypeName = Ent.Singular(entityType) csvPF = CSVPrintFile([entityTypeName, 'resourceName'], 'sortall') if Act.csvFormat() else None if csvPF: - csvPF.SetEscapeChar(None) + csvPF.SetNoEscapeChar(True) FJQC = FormatJSONQuoteChar(csvPF) fieldsList = [] parameters = _initPersonMetadataParameters() @@ -26376,7 +26397,7 @@ def doPrintShowChromePolicies(): customer = _getCustomersCustomerIdWithC() csvPF = CSVPrintFile(CHROME_POLICY_SORT_TITLES, indexedTitles=CHROME_POLICY_INDEXED_TITLES) if Act.csvFormat() else None if csvPF: - csvPF.SetEscapeChar(None) + csvPF.SetNoEscapeChar(True) FJQC = FormatJSONQuoteChar(csvPF) appId = orgUnit = policySchemaFilter = printerId = None showPolicies = CHROME_POLICY_SHOW_ALL @@ -26783,13 +26804,13 @@ def getCIDeviceEntity(): pageMessage = getPageMessage() try: devices = callGAPIpages(ci.devices(), 'list', 'devices', - throwReasons=[GAPI.INVALID, GAPI.PERMISSION_DENIED], + throwReasons=[GAPI.INVALID, GAPI.INVALID_ARGUMENT, GAPI.PERMISSION_DENIED], retryReasons=GAPI.SERVICE_NOT_AVAILABLE_RETRY_REASONS, pageMessage=pageMessage, customer=customer, filter=query, fields='nextPageToken,devices(name)', pageSize=100) return (devices, ci, customer, False) - except GAPI.invalid: + except (GAPI.invalid, GAPI.invalidArgument): Cmd.Backup() usageErrorExit(Msg.INVALID_QUERY) except GAPI.permissionDenied as e: @@ -27121,11 +27142,13 @@ DEVICE_FIELDS_CHOICE_MAP = { 'buildnumber': 'buildNumber', 'compromisedstate': 'compromisedState', 'createtime': 'createTime', + 'deviceid': 'deviceId', 'devicetype': 'deviceType', 'enableddeveloperoptions': 'enabledDeveloperOptions', 'enabledusbdebugging': 'enabledUsbDebugging', 'encryptionstate': 'encryptionState', 'endpointverificationspecificattributes': 'endpointVerificationSpecificAttributes', + 'hostname': 'hostname', 'imei': 'imei', 'kernelversion': 'kernelVersion', 'lastsynctime': 'lastSyncTime', @@ -69278,7 +69301,7 @@ NOTES_ROLE_CHOICE_MAP = { def printShowNotes(users): csvPF = CSVPrintFile(['User', 'name', 'title', 'owner', 'ownedByMe']) if Act.csvFormat() else None if csvPF: - csvPF.SetEscapeChar(None) + csvPF.SetNoEscapeChar(True) FJQC = FormatJSONQuoteChar(csvPF) compact = False fieldsList = [] @@ -69860,7 +69883,7 @@ def printShowTasks(users): csvPF = CSVPrintFile(['User', 'tasklistId', 'id', 'taskId', 'title', 'status', 'due', 'updated', 'completed'], 'sortall') if Act.csvFormat() else None if csvPF: - csvPF.SetEscapeChar(None) + csvPF.SetNoEscapeChar(True) CSVTitle = 'Tasks' FJQC = FormatJSONQuoteChar(csvPF) tasklistEntity = None @@ -70092,7 +70115,7 @@ def processTasklists(users): def printShowTasklists(users): csvPF = CSVPrintFile(['User', 'id', 'title']) if Act.csvFormat() else None if csvPF: - csvPF.SetEscapeChar(None) + csvPF.SetNoEscapeChar(True) CSVTitle = 'TaskLists' FJQC = FormatJSONQuoteChar(csvPF) countsOnly = False