diff --git a/DataAgent/CAPABILITIES.md b/DataAgent/CAPABILITIES.md index b015b47..806f4c6 100644 --- a/DataAgent/CAPABILITIES.md +++ b/DataAgent/CAPABILITIES.md @@ -69,6 +69,8 @@ x12 Control is a template: omit it for the existing hour-of-year scheme, use `1{ invoice groups keep their first-seen order, and each invoice's details keep their input order. the source owns any sorting. pdi no longer sorts invoices by key or moves D records before I records. grouping still puts an invoice's rows together if the source interleaves invoice keys. +pdi and x12-210 allow one header per key. rows under one key must agree on every header field: pdi's `Head` fields, and for x12-210 every column not matched by `Detail`, used by a template or not. a key with two headers fails the run before the file is written, naming the key and the fields that differ, so nothing is sent. before 0.12.1 that input wrote the invoice once per header, each copy with every line. fix the query, or list a varying line-level column in `Detail`. keys match without regard to case. + invoice substitutions without a format preserve PowerShell's string interpolation, including dates and numbers. alignment pads that string. an explicit `:format` instead formats the original typed value using .NET's current-culture formatting. set `adapter` to a `.ps1` path, relative to the job directory or absolute. no registration or core edit. each script receives `-Data` and `-Options`: diff --git a/DataAgent/DataAgent.psd1 b/DataAgent/DataAgent.psd1 index 8bdb8e4..a30fc09 100644 --- a/DataAgent/DataAgent.psd1 +++ b/DataAgent/DataAgent.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'DataAgent.psm1' - ModuleVersion = '0.12.0' + ModuleVersion = '0.12.1' GUID = '32662c0b-0a5d-49ac-8c62-8f7ae05f79c0' Author = 'Roy Ashbrook' Copyright = '(c) 2026 Roy Ashbrook. MIT License.' diff --git a/DataAgent/fmt/invoice/template.ps1 b/DataAgent/fmt/invoice/template.ps1 index 804bf7c..fb22966 100644 --- a/DataAgent/fmt/invoice/template.ps1 +++ b/DataAgent/fmt/invoice/template.ps1 @@ -13,6 +13,21 @@ function Get-InvoiceRows($Rows) { else { $Rows } } +# The flat layouts take one header per distinct header row and pull details by key alone, so two +# headers under one key would each be written with every line of that invoice. Call this before +# the output file is opened: a failed run then leaves no partial file and nothing to send. +# It asks the detail lookup's own question (Where-Object -EQ), not Group-Object's: the two compare +# strings differently in some cultures, and the guard must agree with what the lookup will pull. +function Assert-InvoiceHeads($Heads, [string] $Key) { + foreach ($head in $Heads) { + $same = @($Heads | Where-Object -Property $Key -EQ -Value $head.$Key) + if ($same.Count -lt 2) { continue } + $a, $b = $same[0, 1] + $fields = @($a.PSObject.Properties.Name | Where-Object { "$($a.$_)" -cne "$($b.$_)" }) + throw "invoice key '$($head.$Key)' has $($same.Count) different headers, differing in: $($fields -join ', '). Rows under one key must agree on every header field." + } +} + function Expand-InvoiceLine([string] $Template, $Row, [hashtable] $Scope = @{}) { $out = [Text.StringBuilder]::new() $at = 0 diff --git a/DataAgent/fmt/pdi.ps1 b/DataAgent/fmt/pdi.ps1 index 5bffafb..d66ba31 100644 --- a/DataAgent/fmt/pdi.ps1 +++ b/DataAgent/fmt/pdi.ps1 @@ -2,8 +2,10 @@ param($Data, [hashtable] $Options) . "$PSScriptRoot/invoice/template.ps1" $Options = Get-InvoiceOptions $Options pdi $rows = @(Get-InvoiceRows $Data) +$heads = @($rows | Select-Object -Property $Options.Head -Unique) +Assert-InvoiceHeads $heads $Options.Key & { - foreach ($head in ($rows | Select-Object -Property $Options.Head -Unique)) { + foreach ($head in $heads) { if ($Options.H) { Expand-InvoiceLine $Options.H $head $Options.Values } foreach ($row in ($rows | Where-Object -Property $Options.Key -EQ -Value $head.($Options.Key))) { if (!$Options.Type -or $row.($Options.Type) -eq 'd') { Expand-InvoiceLine $Options.D $row $Options.Values } diff --git a/DataAgent/fmt/x12-210.ps1 b/DataAgent/fmt/x12-210.ps1 index 64c1385..08cea3b 100644 --- a/DataAgent/fmt/x12-210.ps1 +++ b/DataAgent/fmt/x12-210.ps1 @@ -30,10 +30,12 @@ function Expand-X12Segments($Templates, $Row) { } } } +$invoices = @($rows | Select-Object -Property * -ExcludeProperty $Options.Detail -Unique) +Assert-InvoiceHeads $invoices $Options.Key & { Expand-X12Segments $Options.Head $null $st = 0 - foreach ($invoice in ($rows | Select-Object -Property * -ExcludeProperty $Options.Detail -Unique)) { + foreach ($invoice in $invoices) { $scope.st = ++$st $segments = [Collections.Generic.List[string]]::new() foreach ($segment in (Expand-X12Segments $Options.Invoice $invoice)) { $segments.Add($segment) } diff --git a/README.md b/README.md index cb1ead8..8aabd3c 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,13 @@ the repeated part of a feed job: get data, format it, send it. use the tools you already use. ```powershell -Install-Module DataAgent -RequiredVersion 0.12.0 -Scope CurrentUser +Install-Module DataAgent -RequiredVersion 0.12.1 -Scope CurrentUser ``` ## the job ```powershell -Import-Module DataAgent -RequiredVersion 0.12.0 +Import-Module DataAgent -RequiredVersion 0.12.1 Invoke-DataAgent "$PSScriptRoot/settings.json" ``` diff --git a/examples/job.ps1 b/examples/job.ps1 index 62c4119..9f35a7f 100644 --- a/examples/job.ps1 +++ b/examples/job.ps1 @@ -1,2 +1,2 @@ -Import-Module DataAgent -RequiredVersion 0.12.0 -ErrorAction Stop +Import-Module DataAgent -RequiredVersion 0.12.1 -ErrorAction Stop Invoke-DataAgent "$PSScriptRoot/settings.json" diff --git a/testing/DataAgent.Test/DataAgent.Test.psd1 b/testing/DataAgent.Test/DataAgent.Test.psd1 index 93bc6a1..5a7db8b 100644 --- a/testing/DataAgent.Test/DataAgent.Test.psd1 +++ b/testing/DataAgent.Test/DataAgent.Test.psd1 @@ -1,6 +1,6 @@ @{ RootModule = 'DataAgent.Test.psm1' - ModuleVersion = '0.12.0' + ModuleVersion = '0.12.1' GUID = '3d16fcb0-a991-4c65-a6e4-3bfcd360688d' Author = 'Roy Ashbrook' Copyright = '(c) 2026 Roy Ashbrook. MIT License.' @@ -10,7 +10,7 @@ CmdletsToExport = @() VariablesToExport = @() AliasesToExport = @() - RequiredModules = @(@{ ModuleName = 'DataAgent'; RequiredVersion = '0.12.0' }) + RequiredModules = @(@{ ModuleName = 'DataAgent'; RequiredVersion = '0.12.1' }) FileList = @('DataAgent.Test.psd1', 'DataAgent.Test.psm1', 'LICENSE', 'synthetic.csv') PrivateData = @{ PSData = @{ Tags = @('ETL', 'DataAgent'); ProjectUri = 'https://github.com/royashbrook/DataAgent'; LicenseUri = 'https://github.com/royashbrook/DataAgent/blob/main/LICENSE' } } } diff --git a/tests/acceptance.ps1 b/tests/acceptance.ps1 index 0acb5bf..c4aed11 100644 --- a/tests/acceptance.ps1 +++ b/tests/acceptance.ps1 @@ -389,7 +389,7 @@ foreach ($package in @('DataAgent','testing/DataAgent.Test')) { } Remove-Module DataAgent -Force $env:PSModulePath = (@("$root/staged", $priorModules) -join [IO.Path]::PathSeparator) -Import-Module DataAgent.Test -RequiredVersion 0.12.0 +Import-Module DataAgent.Test -RequiredVersion 0.12.1 $result = @(Test-DataAgent) Assert (@($result | Where-Object { $_ -is [IO.FileInfo] -and $_.Name -eq 'output.csv' }).Count -eq 1) 'staged optional test package exercises bundled formatter' $env:PSModulePath = $priorModules diff --git a/tests/invoice-formats.ps1 b/tests/invoice-formats.ps1 index 9d224fa..e143954 100644 --- a/tests/invoice-formats.ps1 +++ b/tests/invoice-formats.ps1 @@ -77,6 +77,50 @@ $lines = Format x12-210 $rows @{Values=$variant;Head=@('ISA*{sender_id,-10}*000{ Check ($lines[0] -ceq 'ISA*SYN *000260123' -and $lines[1] -ceq 'GS*000260123' -and $lines[-1] -ceq 'IEA*1*260123') 'legacy width and inconsistent prefix stay explicit template overrides' Check (@($lines | Where-Object { $_ -ceq 'SE*24*000000001' }).Count -eq 1) 'a legacy count offset is explicit, not the default' +# One key with two headers used to write the invoice twice, each copy with every line. refs #38. +function Rejects($Name, $Rows, $Options, $Field, $Case, $Key = 'A') { + $Options.Path = "$root/reject-$Name-$script:checks.txt" + $message = try { & "$ModulePath/fmt/$Name.ps1" -Data $Rows -Options $Options; '' } catch { "$_" } + Check ($message -like "*key '$Key'*" -and $message -like "*$Field*") "$Name $Case fails, naming the key and the field" + Check (!(Test-Path -LiteralPath $Options.Path)) "$Name $Case leaves no file" +} +function Copy-Rows($Rows) { foreach ($row in $Rows) { $row | Select-Object * } } +$clash = @(Copy-Rows $rows); $clash[1].b303 = 'B9' +Rejects x12-210 $clash @{Values=$values} b303 'a header field that differs under one key' +$clash = @(Copy-Rows $rows | Select-Object *, unused); $clash[1].unused = 'x' +Rejects x12-210 $clash @{Values=$values} unused 'a column no template uses, differing under one key' +$clash = @(Copy-Rows $rows); $clash[2].b302 = 'a' +Rejects x12-210 $clash @{Values=$values} b302 'keys that differ only by case' +$clash = @(Copy-Rows $rows); $clash[1].l507 = 'OTHER' +$plain = (Format x12-210 $rows @{Values=$values}) -join '|' +Check (((Format x12-210 $clash @{Values=$values}) -join '|') -ceq $plain.Replace('*FUEL', '*OTHER')) 'x12-210 detail fields may still differ under one key' +$flat = @( + [pscustomobject]@{id='A';bol='B1';kind='d';amount=1;note='one'} + [pscustomobject]@{id='A';bol='B1';kind='d';amount=2;note='two'} + [pscustomobject]@{id='B';bol='B1';kind='d';amount=3;note='one'} +) +$layout = @{Key='id';Head=@('id','bol');Type='kind';H='H,{id},{bol}';D='D,{amount}';I=''} +Check (((Format pdi $flat (@{} + $layout)) -join '|') -ceq 'H,A,B1|D,1|D,2|H,B,B1|D,3') 'pdi fields outside Head may still differ under one key' +$twoHeads = @(Copy-Rows $flat); $twoHeads[1].bol = 'B9' +Rejects pdi $twoHeads (@{} + $layout) bol 'a Head field that differs under one key' +# The guard asks the detail lookup's own question, so the two agree in every culture. Under tr-TR +# I and i still pull each other's lines, and I and dotless i are still two invoices. +$dotless = [string][char]0x131 +try { + foreach ($name in 'en-US', 'tr-TR') { + [Globalization.CultureInfo]::CurrentCulture = [Globalization.CultureInfo]::GetCultureInfo($name) + $clash = @(Copy-Rows $flat[0, 2]); $clash[0].id = 'I'; $clash[1].id = 'i'; $clash[1].bol = 'B9' + Rejects pdi $clash (@{} + $layout) bol "$name keys I and i" I + $clash = @(Copy-Rows $rows[0, 2]); $clash[0].b302 = 'I'; $clash[1].b302 = 'i' + Rejects x12-210 $clash @{Values=$values} b302 "$name keys I and i" I + $apart = @(Copy-Rows $flat[0, 2]); $apart[0].id = 'I'; $apart[1].id = $dotless; $apart[1].bol = 'B9' + Check (((Format pdi $apart (@{} + $layout)) -join '|') -ceq "H,I,B1|D,1|H,$dotless,B9|D,3") "$name pdi keys I and dotless i stay two invoices" + $apart = @(Copy-Rows $rows[0, 2]); $apart[0].b302 = 'I'; $apart[1].b302 = $dotless + $lines = Format x12-210 $apart @{Values=$values} + Check (@($lines -like 'ST[*]*').Count -eq 2 -and @($lines -like 'LX[*]*').Count -eq 2) "$name x12-210 keys I and dotless i stay two invoices" + } +} finally { [Globalization.CultureInfo]::CurrentCulture = $culture } + $invoices = @( @{ivh_invoicenumber='A';ivh_billdate='20261014';ivh_totalcharge=3;ShipDate='20261013';ShipTime='0830';Detail=@(@{RecordType='ITMF';bol='B1';ivd_description='DIESEL';cmd_code='DSL';ivd_quantity=2;ivd_wgt=3;ivd_unit='GA';ivd_rate=1;ivd_charge=2},@{RecordType='OTHF';ivd_rate=1;ivd_charge=1})} @{ivh_invoicenumber='B';Detail=@(@{RecordType='ITMF';ivd_rate=1.23456;ivd_charge=2.345})} @@ -106,6 +150,11 @@ try { Check ((Get-Content "$job/out.txt")[0] -ceq 'synthetic-user,synthetic-password') 'whole runner resolves nested env values into the DTN file' $log = Get-Content (Join-Path $job ('{0:yyyyMMdd}.log' -f (Get-Date))) -Raw Check ($log -match 'End' -and $log -notmatch 'synthetic-user|synthetic-password') 'whole runner completes without logging the DTN credentials' + $twoHeads | ConvertTo-Json | Set-Content "$job/rows.json" + 'param($Data,$Options); Set-Content delivered.txt $Data' | Set-Content "$job/deliver.ps1" + $config = @{directory=$job.FullName;src=@{adapter='./source.ps1';args=@{InputFile='rows.json'}};fmt=@{adapter='pdi';args=(@{Path='clash.txt'} + $layout)};dst=@{adapter='./deliver.ps1';args=@{}}} + $message = try { $null = Invoke-DataAgent $config; '' } catch { "$_" } + Check ($message -like "*key 'A'*" -and !(Test-Path "$job/clash.txt") -and !(Test-Path "$job/delivered.txt")) 'whole runner stops on a key with two headers: no file and no destination call' } finally { Remove-Item Env:DATAAGENT_INVOICE_TEST_USER -ErrorAction SilentlyContinue Set-Location $before