From adbee45073accfa8fe54ee080ca520ee50777d16 Mon Sep 17 00:00:00 2001 From: Ross Scroggs Date: Fri, 9 Feb 2024 10:00:33 -0800 Subject: [PATCH] Added option `skiprows ` to `gam csv|loop` --- docs/Bulk-Processing.md | 9 ++++--- docs/GamUpdates.md | 6 +++++ docs/How-to-Upgrade-from-Standard-GAM.md | 12 ++++----- docs/Version-and-Help.md | 30 ++++++++++----------- src/GamCommands.txt | 4 +-- src/GamUpdate.txt | 6 +++++ src/gam/__init__.py | 33 ++++++++++++++++++++---- 7 files changed, 69 insertions(+), 31 deletions(-) diff --git a/docs/Bulk-Processing.md b/docs/Bulk-Processing.md index 97a29b99..9a19d819 100644 --- a/docs/Bulk-Processing.md +++ b/docs/Bulk-Processing.md @@ -72,13 +72,13 @@ gam redirect stdout ./NewStudents.out redirect stderr ./NewStudents.err tbatch N gam csv |-|(gsheet )|(gdoc ) [charset ] [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] - [maxrows ] + [skiprows ] [maxrows ] gam gam loop |-|(gsheet )|(gdoc ) [charset ] [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] - [maxrows ] + [skiprows ] [maxrows ] gam ``` * `gam csv` - Use parallel processing @@ -93,7 +93,10 @@ gam loop |-|(gsheet )|(gdoc ) [charset * `fields ` - The column headings of a CSV file that does not contain column headings. * `(matchfield|skipfield )*` - The criteria to select rows from the CSV file; can be used multiple times; if not specified, all rows are selected * `showcmds` - Write `timestamp,command number/number of commands,command` to stderr when each command starts; write `timestamp, command number/numberof commands,complete` to stderr when command completes -* `maxrows ` - Limit the number of filtered rows processed from the CSV file/Google Sheet. +* `skiprows ` - Skip filtered rows from the CSV file/Google Sheet. + * `skiprows 0` - All rows are processed, this is the default + * `skiprows N` - The first N filtered rows are skipped +* `maxrows ` - Limit the number of filtered rows processed from the CSV file/Google Sheet after any skipped rows. * `maxrows 0` - All rows are processed, this is the default * `maxrows N` - N filtered rows are processed diff --git a/docs/GamUpdates.md b/docs/GamUpdates.md index 90545ab3..38bcd028 100644 --- a/docs/GamUpdates.md +++ b/docs/GamUpdates.md @@ -10,6 +10,12 @@ Add the `-s` option to the end of the above commands to suppress creating the `g See [Downloads](https://github.com/taers232c/GAMADV-XTD3/wiki/Downloads) for Windows or other options, including manual installation +### 6.68.04 + +Added option `skiprows ` to `gam csv|loop` that causes GAM to skip processing the first `` filtered rows. + +* See: https://github.com/taers232c/GAMADV-XTD3/wiki/Bulk-Processing#csv-files + ### 6.68.03 Fixed bug in `gam create drivefileacl` that caused a trap. diff --git a/docs/How-to-Upgrade-from-Standard-GAM.md b/docs/How-to-Upgrade-from-Standard-GAM.md index a85cf95f..2fa40cf7 100644 --- a/docs/How-to-Upgrade-from-Standard-GAM.md +++ b/docs/How-to-Upgrade-from-Standard-GAM.md @@ -334,12 +334,12 @@ writes the credentials into the file oauth2.txt. admin@server:/Users/admin/bin/gamadv-xtd3$ rm -f /Users/admin/GAMConfig/oauth2.txt admin@server:/Users/admin/bin/gamadv-xtd3$ ./gam version WARNING: Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, Item: oauth2_txt, Value: /Users/admin/GAMConfig/oauth2.txt, Not Found -GAMADV-XTD3 6.68.03 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.68.04 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs -Python 3.12.1 64-bit final +Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 Path: /Users/admin/bin/gamadv-xtd3 -Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain.com +Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain: domain.com admin@server:/Users/admin/bin/gamadv-xtd3$ ./gam oauth create @@ -1002,12 +1002,12 @@ writes the credentials into the file oauth2.txt. C:\GAMADV-XTD3>del C:\GAMConfig\oauth2.txt C:\GAMADV-XTD3>gam version WARNING: Config File: C:\GAMConfig\gam.cfg, Section: DEFAULT, Item: oauth2_txt, Value: C:\GAMConfig\oauth2.txt, Not Found -GAMADV-XTD3 6.68.03 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.68.04 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs -Python 3.12.1 64-bit final +Python 3.12.2 64-bit final Windows-10-10.0.17134 AMD64 Path: C:\GAMADV-XTD3 -Config File: C:\GAMConfig\gam.cfg, Section: DEFAULT, customer_id: my_customer, domain.com +Config File: C:\GAMConfig\gam.cfg, Section: DEFAULT, customer_id: my_customer, domain: domain.com C:\GAMADV-XTD3>gam oauth create diff --git a/docs/Version-and-Help.md b/docs/Version-and-Help.md index 9c34e270..9b53bb3e 100644 --- a/docs/Version-and-Help.md +++ b/docs/Version-and-Help.md @@ -3,41 +3,41 @@ Print the current version of Gam with details ``` gam version -GAMADV-XTD3 6.68.03 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.68.04 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs -Python 3.12.1 64-bit final +Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 -Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain.com +Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain: domain.com Time: 2023-06-02T21:10:00-07:00 ``` Print the current version of Gam with details and time offset information ``` gam version timeoffset -GAMADV-XTD3 6.68.03 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.68.04 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs -Python 3.12.1 64-bit final +Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 -Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain.com +Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain: domain.com Your system time differs from www.googleapis.com by less than 1 second ``` Print the current version of Gam with extended details and SSL information ``` gam version extended -GAMADV-XTD3 6.68.03 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource +GAMADV-XTD3 6.68.04 - https://github.com/taers232c/GAMADV-XTD3 - pythonsource Ross Scroggs -Python 3.12.1 64-bit final +Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 -Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain.com +Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain: domain.com Time: 2023-06-02T21:10:00-07:00 Your system time differs from admin.googleapis.com by less than 1 second OpenSSL 3.1.1 30 May 2023 cryptography 41.0.1 -filelock 3.12.1 +filelock 3.12.2 google-api-python-client 2.88.0 google-auth-httplib2 0.1.0 google-auth-oauthlib 1.0.0 @@ -64,7 +64,7 @@ MacOS High Sierra 10.13.6 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 Version Check: Current: 5.35.08 - Latest: 6.68.03 + Latest: 6.68.04 echo $? 1 ``` @@ -72,7 +72,7 @@ echo $? Print the current version number without details ``` gam version simple -6.68.03 +6.68.04 ``` In Linux/MacOS you can do: ``` @@ -82,12 +82,12 @@ echo $VER Print the current version of Gam and address of this Wiki ``` gam help -GAM 6.68.03 - https://github.com/taers232c/GAMADV-XTD3 +GAM 6.68.04 - https://github.com/taers232c/GAMADV-XTD3 Ross Scroggs -Python 3.12.1 64-bit final +Python 3.12.2 64-bit final MacOS Sonoma 14.2.1 x86_64 Path: /Users/Admin/bin/gamadv-xtd3 -Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain.com +Config File: /Users/admin/GAMConfig/gam.cfg, Section: DEFAULT, customer_id: my_customer, domain: domain.com Time: 2023-06-02T21:10:00-07:00 Help: Syntax in file /Users/Admin/bin/gamadv-xtd3/GamCommands.txt Help: Documentation is at https://github.com/taers232c/GAMADV-XTD3/wiki diff --git a/src/GamCommands.txt b/src/GamCommands.txt index 3e913e5f..ccc730c1 100644 --- a/src/GamCommands.txt +++ b/src/GamCommands.txt @@ -1288,13 +1288,13 @@ gam tbatch [showcmds []] gam csv [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] - [maxrows ] + [skiprows ] [maxrows ] gam gam loop [warnifnodata] [columndelimiter ] [noescapechar ] [quotechar ] [fields ] (matchfield|skipfield )* [showcmds []] - [maxrows ] + [skiprows ] [maxrows ] gam You can make substitutions in with values from the CSV file. diff --git a/src/GamUpdate.txt b/src/GamUpdate.txt index 58aea96b..56decc3a 100644 --- a/src/GamUpdate.txt +++ b/src/GamUpdate.txt @@ -2,6 +2,12 @@ Merged GAM-Team version +6.68.04 + +Added option `skiprows ` to `gam csv|loop` that causes GAM to skip processing the first `` filtered rows. + +* See: https://github.com/taers232c/GAMADV-XTD3/wiki/Bulk-Processing#csv-files + 6.68.03 Fixed bug in `gam create drivefileacl` that caused a trap. diff --git a/src/gam/__init__.py b/src/gam/__init__.py index e0391f2a..4eae9d3e 100755 --- a/src/gam/__init__.py +++ b/src/gam/__init__.py @@ -9859,6 +9859,12 @@ def _getShowCommands(): return getBoolean() return GC.Values[GC.SHOW_COMMANDS] +def _getSkipRows(): + if checkArgumentPresent('skiprows'): + return getInteger(minVal=0) +# return GC.Values[GC.CSV_INPUT_ROW_SKIP] + return 0 + def _getMaxRows(): if checkArgumentPresent('maxrows'): return getInteger(minVal=0) @@ -10041,7 +10047,7 @@ def processSubFields(GAM_argv, row, subFields): # gam csv [warnifnodata] # [columndelimiter ] [quotechar ] [fields ] # (matchfield|skipfield )* [showcmds []] -# [maxrows ] +# [skiprows ] [maxrows ] # gam def doCSV(testMode=False): filename = getString(Cmd.OB_FILE_NAME) @@ -10051,6 +10057,7 @@ def doCSV(testMode=False): f, csvFile, fieldnames = openCSVFileReader(filename) matchFields, skipFields = getMatchSkipFields(fieldnames) showCmds = _getShowCommands() + skipRows = _getSkipRows() maxRows = _getMaxRows() checkArgumentPresent(Cmd.GAM_CMD, required=True) if not Cmd.ArgumentsRemaining(): @@ -10065,8 +10072,13 @@ def doCSV(testMode=False): i = 0 for row in csvFile: if checkMatchSkipFields(row, fieldnames, matchFields, skipFields): - items.append(processSubFields(GAM_argv, row, subFields)) i += 1 + if skipRows: + if i <= skipRows: + continue + i = 1 + skipRows = 0 + items.append(processSubFields(GAM_argv, row, subFields)) if maxRows and i >= maxRows: break closeFile(f) @@ -10091,7 +10103,7 @@ def doCSVTest(): # gam loop [warnifnodata] # [columndelimiter ] [quotechar ] [fields ] # (matchfield|skipfield )* [showcmds []] -# [maxrows ] +# [skiprows ] [maxrows ] # gam def doLoop(loopCmd): filename = getString(Cmd.OB_FILE_NAME) @@ -10101,6 +10113,7 @@ def doLoop(loopCmd): f, csvFile, fieldnames = openCSVFileReader(filename) matchFields, skipFields = getMatchSkipFields(fieldnames) showCmds = _getShowCommands() + skipRows = _getSkipRows() maxRows = _getMaxRows() checkArgumentPresent(Cmd.GAM_CMD, required=True) if not Cmd.ArgumentsRemaining(): @@ -10136,9 +10149,14 @@ def doLoop(loopCmd): i = 0 for row in csvFile: if checkMatchSkipFields(row, fieldnames, matchFields, skipFields): + i += 1 + if skipRows: + if i <= skipRows: + continue + i = 1 + skipRows = 0 item = processSubFields(GAM_argv, row, subFields) logCmd = Cmd.QuotedArgumentList(item) - i += 1 if i % 100 == 0: batchWriteStderr(Msg.PROCESSING_ITEM_N.format(currentISOformatTimeStamp(), i)) sysRC = ProcessGAMCommand(item, processGamCfg=processGamCfg, inLoop=True) @@ -10154,8 +10172,13 @@ def doLoop(loopCmd): i = 0 for row in csvFile: if checkMatchSkipFields(row, fieldnames, matchFields, skipFields): - items.append(processSubFields(GAM_argv, row, subFields)) i += 1 + if skipRows: + if i <= skipRows: + continue + i = 1 + skipRows = 0 + items.append(processSubFields(GAM_argv, row, subFields)) if maxRows and i >= maxRows: break closeFile(f)