From 937a555e4fd5d4928d64a5fb706d083acdc42aad Mon Sep 17 00:00:00 2001 From: rajsite Date: Wed, 28 Feb 2024 21:22:49 +0000 Subject: [PATCH 1/5] applying package updates [skip ci] --- .../projects/ni/nimble-angular/CHANGELOG.json | 15 +++++++++++++++ .../projects/ni/nimble-angular/CHANGELOG.md | 10 +++++++++- .../projects/ni/nimble-angular/package.json | 4 ++-- ...ents-46e491a1-b49a-4d5c-b64e-6eb6fae50e0b.json | 7 ------- package-lock.json | 8 ++++---- packages/nimble-blazor/package.json | 2 +- packages/nimble-components/CHANGELOG.json | 15 +++++++++++++++ packages/nimble-components/CHANGELOG.md | 10 +++++++++- packages/nimble-components/package.json | 2 +- 9 files changed, 56 insertions(+), 17 deletions(-) delete mode 100644 change/@ni-nimble-components-46e491a1-b49a-4d5c-b64e-6eb6fae50e0b.json diff --git a/angular-workspace/projects/ni/nimble-angular/CHANGELOG.json b/angular-workspace/projects/ni/nimble-angular/CHANGELOG.json index e8b7bf13b5..81fa7d8153 100644 --- a/angular-workspace/projects/ni/nimble-angular/CHANGELOG.json +++ b/angular-workspace/projects/ni/nimble-angular/CHANGELOG.json @@ -1,6 +1,21 @@ { "name": "@ni/nimble-angular", "entries": [ + { + "date": "Wed, 28 Feb 2024 21:22:49 GMT", + "version": "20.2.13", + "tag": "@ni/nimble-angular_v20.2.13", + "comments": { + "patch": [ + { + "author": "beachball", + "package": "@ni/nimble-angular", + "comment": "Bump @ni/nimble-components to v21.7.1", + "commit": "not available" + } + ] + } + }, { "date": "Wed, 28 Feb 2024 19:35:04 GMT", "version": "20.2.12", diff --git a/angular-workspace/projects/ni/nimble-angular/CHANGELOG.md b/angular-workspace/projects/ni/nimble-angular/CHANGELOG.md index e580df39ab..db79e75155 100644 --- a/angular-workspace/projects/ni/nimble-angular/CHANGELOG.md +++ b/angular-workspace/projects/ni/nimble-angular/CHANGELOG.md @@ -1,9 +1,17 @@ # Change Log - @ni/nimble-angular -This log was last generated on Wed, 28 Feb 2024 19:35:04 GMT and should not be manually modified. +This log was last generated on Wed, 28 Feb 2024 21:22:49 GMT and should not be manually modified. +## 20.2.13 + +Wed, 28 Feb 2024 21:22:49 GMT + +### Patches + +- Bump @ni/nimble-components to v21.7.1 + ## 20.2.12 Wed, 28 Feb 2024 19:35:04 GMT diff --git a/angular-workspace/projects/ni/nimble-angular/package.json b/angular-workspace/projects/ni/nimble-angular/package.json index 211816a498..58bdfdcf42 100644 --- a/angular-workspace/projects/ni/nimble-angular/package.json +++ b/angular-workspace/projects/ni/nimble-angular/package.json @@ -1,6 +1,6 @@ { "name": "@ni/nimble-angular", - "version": "20.2.12", + "version": "20.2.13", "description": "Angular components for the NI Nimble Design System", "scripts": { "invoke-publish": "cd ../../../ && npm run build:library && cd dist/ni/nimble-angular && npm publish" @@ -31,7 +31,7 @@ "@angular/forms": "^15.2.10", "@angular/localize": "^15.2.10", "@angular/router": "^15.2.10", - "@ni/nimble-components": "^21.7.0" + "@ni/nimble-components": "^21.7.1" }, "dependencies": { "tslib": "^2.2.0" diff --git a/change/@ni-nimble-components-46e491a1-b49a-4d5c-b64e-6eb6fae50e0b.json b/change/@ni-nimble-components-46e491a1-b49a-4d5c-b64e-6eb6fae50e0b.json deleted file mode 100644 index fe47e607fe..0000000000 --- a/change/@ni-nimble-components-46e491a1-b49a-4d5c-b64e-6eb6fae50e0b.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "patch", - "comment": "Create web worker class for future faster rendering wafer map", - "packageName": "@ni/nimble-components", - "email": "110180309+Razvan1928@users.noreply.github.com", - "dependentChangeType": "patch" -} diff --git a/package-lock.json b/package-lock.json index b24cc99513..a048d1cda9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,7 +83,7 @@ }, "angular-workspace/projects/ni/nimble-angular": { "name": "@ni/nimble-angular", - "version": "20.2.12", + "version": "20.2.13", "license": "MIT", "dependencies": { "tslib": "^2.2.0" @@ -94,7 +94,7 @@ "@angular/forms": "^15.2.10", "@angular/localize": "^15.2.10", "@angular/router": "^15.2.10", - "@ni/nimble-components": "^21.7.0" + "@ni/nimble-components": "^21.7.1" } }, "node_modules/@11ty/dependency-tree": { @@ -33897,7 +33897,7 @@ }, "packages/nimble-blazor": { "name": "@ni/nimble-blazor", - "version": "14.3.9", + "version": "14.3.10", "hasInstallScript": true, "license": "MIT", "devDependencies": { @@ -34019,7 +34019,7 @@ }, "packages/nimble-components": { "name": "@ni/nimble-components", - "version": "21.7.0", + "version": "21.7.1", "license": "MIT", "dependencies": { "@microsoft/fast-colors": "^5.3.1", diff --git a/packages/nimble-blazor/package.json b/packages/nimble-blazor/package.json index 6bd8a13519..6b80a7936b 100644 --- a/packages/nimble-blazor/package.json +++ b/packages/nimble-blazor/package.json @@ -1,6 +1,6 @@ { "name": "@ni/nimble-blazor", - "version": "14.3.9", + "version": "14.3.10", "description": "Blazor components for the NI Nimble Design System", "scripts": { "postinstall": "node build/generate-playwright-version-properties/source/index.js", diff --git a/packages/nimble-components/CHANGELOG.json b/packages/nimble-components/CHANGELOG.json index 33e4a52a42..b9702703b3 100644 --- a/packages/nimble-components/CHANGELOG.json +++ b/packages/nimble-components/CHANGELOG.json @@ -1,6 +1,21 @@ { "name": "@ni/nimble-components", "entries": [ + { + "date": "Wed, 28 Feb 2024 21:22:49 GMT", + "version": "21.7.1", + "tag": "@ni/nimble-components_v21.7.1", + "comments": { + "patch": [ + { + "author": "110180309+Razvan1928@users.noreply.github.com", + "package": "@ni/nimble-components", + "commit": "52693524c553f3b115e6c1004c902d10e7dd4050", + "comment": "Create web worker class for future faster rendering wafer map" + } + ] + } + }, { "date": "Wed, 28 Feb 2024 19:35:04 GMT", "version": "21.7.0", diff --git a/packages/nimble-components/CHANGELOG.md b/packages/nimble-components/CHANGELOG.md index 4f08eec2e9..e8f7ddece4 100644 --- a/packages/nimble-components/CHANGELOG.md +++ b/packages/nimble-components/CHANGELOG.md @@ -1,9 +1,17 @@ # Change Log - @ni/nimble-components -This log was last generated on Wed, 28 Feb 2024 19:35:04 GMT and should not be manually modified. +This log was last generated on Wed, 28 Feb 2024 21:22:49 GMT and should not be manually modified. +## 21.7.1 + +Wed, 28 Feb 2024 21:22:49 GMT + +### Patches + +- Create web worker class for future faster rendering wafer map ([ni/nimble@5269352](https://github.com/ni/nimble/commit/52693524c553f3b115e6c1004c902d10e7dd4050)) + ## 21.7.0 Wed, 28 Feb 2024 19:35:04 GMT diff --git a/packages/nimble-components/package.json b/packages/nimble-components/package.json index c9f6a63b5b..23208164f4 100644 --- a/packages/nimble-components/package.json +++ b/packages/nimble-components/package.json @@ -1,6 +1,6 @@ { "name": "@ni/nimble-components", - "version": "21.7.0", + "version": "21.7.1", "description": "Styled web components for the NI Nimble Design System", "scripts": { "build": "npm run generate-icons && npm run generate-workers && npm run build-components && npm run bundle-components && npm run generate-scss && npm run build-storybook", From 62af2a597dbc070a55bbc49194bffbcb72fde5f2 Mon Sep 17 00:00:00 2001 From: Jesse Attas Date: Wed, 28 Feb 2024 17:23:57 -0600 Subject: [PATCH 2/5] Add docs and test for table column header text inside a (#1877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## ๐Ÿคจ Rationale [Some localization use cases](https://dev.azure.com/ni/DevCentral/_git/Skyline/pullrequest/661277?_a=files&path=/Web/Workspaces/SystemLinkShared/projects/asset-manager-lib/src/assets/asset-move-dialog/asset-move-dialog.component.html) require placing table header text inside a span rather than being text content of the header. This works fine today (text is rendered all caps, ellipsized when necessary, title set when necessary) but we don't document or test this case. ## ๐Ÿ‘ฉโ€๐Ÿ’ป Implementation 1. Add a note in storybook that this is allowed 2. Add a unit test to ensure title is set ## ๐Ÿงช Testing 1. New unit test for title. 2. Looked for unit / chromatic tests for ellipsis but didn't see any to copy and didn't think it was worth writing one for this case 3. Locally verified in Angular example app that header text in a span behaves identically to header text as text content ## โœ… Checklist - [ ] I have updated the project documentation to reflect my changes or determined no changes are needed. --------- Co-authored-by: Milan Raj --- ...nents-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json | 7 +++++++ .../base/tests/table-column.stories.ts | 2 +- .../src/table/tests/table.spec.ts | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json diff --git a/change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json b/change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json new file mode 100644 index 0000000000..49df43b637 --- /dev/null +++ b/change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json @@ -0,0 +1,7 @@ +{ + "type": "none", + "comment": "Docs and test for table column header text in span", + "packageName": "@ni/nimble-components", + "email": "jattasNI@users.noreply.github.com", + "dependentChangeType": "patch" +} diff --git a/packages/nimble-components/src/table-column/base/tests/table-column.stories.ts b/packages/nimble-components/src/table-column/base/tests/table-column.stories.ts index 4008f32dac..74e12a052d 100644 --- a/packages/nimble-components/src/table-column/base/tests/table-column.stories.ts +++ b/packages/nimble-components/src/table-column/base/tests/table-column.stories.ts @@ -210,7 +210,7 @@ interface HeaderContentTableArgs extends SharedTableArgs { } const headerContentIntro = 'The content of each column header comes from whatever is slotted in the column element.'; -const headerTextContent = `If you provide only text content, Nimble will style it +const headerTextContent = `If you provide only text content (or text content inside a \`\`), Nimble will style it and add a \`title\` to show a tooltip when truncated.`; const headerIconContent = 'If you provide icon content, you should set your own `title` on the icon element.'; const headerTitleContent = 'Titles should use "Headline Casing" and Nimble will automatically capitalize them for display in the header.'; diff --git a/packages/nimble-components/src/table/tests/table.spec.ts b/packages/nimble-components/src/table/tests/table.spec.ts index bf084bd5ba..8d0fad78c7 100644 --- a/packages/nimble-components/src/table/tests/table.spec.ts +++ b/packages/nimble-components/src/table/tests/table.spec.ts @@ -215,6 +215,20 @@ describe('Table', () => { expect(pageObject.getHeaderTitle(0)).toBe(headerContents); }); + it('sets title when header text is ellipsized in a span', async () => { + const headerContents = 'a very long value that should get ellipsized due to not fitting within the default header width'; + await element.setData(simpleTableData); + await connect(); + await waitForUpdatesAsync(); + const span = document.createElement('span'); + span.textContent = headerContents; + element.columns[0]!.textContent = ''; + element.columns[0]!.appendChild(span); + pageObject.dispatchEventToHeader(0, new MouseEvent('mouseover')); + await waitForUpdatesAsync(); + expect(pageObject.getHeaderTitle(0)).toBe(headerContents); + }); + it('does not set title when header text is fully visible', async () => { const headerContents = 'short value'; await element.setData(simpleTableData); From 7b87e719341368d4552fe9ebee462bd087b97d62 Mon Sep 17 00:00:00 2001 From: rajsite Date: Wed, 28 Feb 2024 23:45:15 +0000 Subject: [PATCH 3/5] applying package updates [skip ci] --- .../projects/ni/nimble-angular/package.json | 2 +- ...ents-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json | 7 ------- package-lock.json | 4 ++-- packages/nimble-blazor/package.json | 2 +- packages/nimble-components/CHANGELOG.json | 15 +++++++++++++++ 5 files changed, 19 insertions(+), 11 deletions(-) delete mode 100644 change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json diff --git a/angular-workspace/projects/ni/nimble-angular/package.json b/angular-workspace/projects/ni/nimble-angular/package.json index 58bdfdcf42..438051c040 100644 --- a/angular-workspace/projects/ni/nimble-angular/package.json +++ b/angular-workspace/projects/ni/nimble-angular/package.json @@ -1,6 +1,6 @@ { "name": "@ni/nimble-angular", - "version": "20.2.13", + "version": "20.2.14", "description": "Angular components for the NI Nimble Design System", "scripts": { "invoke-publish": "cd ../../../ && npm run build:library && cd dist/ni/nimble-angular && npm publish" diff --git a/change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json b/change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json deleted file mode 100644 index 49df43b637..0000000000 --- a/change/@ni-nimble-components-4ec55901-eaa7-4bbb-a68e-0fdfd13c03a7.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "none", - "comment": "Docs and test for table column header text in span", - "packageName": "@ni/nimble-components", - "email": "jattasNI@users.noreply.github.com", - "dependentChangeType": "patch" -} diff --git a/package-lock.json b/package-lock.json index a048d1cda9..7be2da96f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,7 +83,7 @@ }, "angular-workspace/projects/ni/nimble-angular": { "name": "@ni/nimble-angular", - "version": "20.2.13", + "version": "20.2.14", "license": "MIT", "dependencies": { "tslib": "^2.2.0" @@ -33897,7 +33897,7 @@ }, "packages/nimble-blazor": { "name": "@ni/nimble-blazor", - "version": "14.3.10", + "version": "14.3.11", "hasInstallScript": true, "license": "MIT", "devDependencies": { diff --git a/packages/nimble-blazor/package.json b/packages/nimble-blazor/package.json index 6b80a7936b..5f3cbd59ce 100644 --- a/packages/nimble-blazor/package.json +++ b/packages/nimble-blazor/package.json @@ -1,6 +1,6 @@ { "name": "@ni/nimble-blazor", - "version": "14.3.10", + "version": "14.3.11", "description": "Blazor components for the NI Nimble Design System", "scripts": { "postinstall": "node build/generate-playwright-version-properties/source/index.js", diff --git a/packages/nimble-components/CHANGELOG.json b/packages/nimble-components/CHANGELOG.json index b9702703b3..2ba209de3c 100644 --- a/packages/nimble-components/CHANGELOG.json +++ b/packages/nimble-components/CHANGELOG.json @@ -1,6 +1,21 @@ { "name": "@ni/nimble-components", "entries": [ + { + "date": "Wed, 28 Feb 2024 23:45:15 GMT", + "version": "21.7.1", + "tag": "@ni/nimble-components_v21.7.1", + "comments": { + "none": [ + { + "author": "jattasNI@users.noreply.github.com", + "package": "@ni/nimble-components", + "commit": "62af2a597dbc070a55bbc49194bffbcb72fde5f2", + "comment": "Docs and test for table column header text in span" + } + ] + } + }, { "date": "Wed, 28 Feb 2024 21:22:49 GMT", "version": "21.7.1", From f3ec1ba17e88c1669c133cb5bebf5cb5ec09546c Mon Sep 17 00:00:00 2001 From: Natan Muntean <33986780+munteannatan@users.noreply.github.com> Date: Thu, 29 Feb 2024 09:32:20 +0200 Subject: [PATCH 4/5] Update spec with wafer map API decision (#1834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## ๐Ÿคจ Rationale Prototyped Apache Arrow API and recorded performance. ## ๐Ÿ‘ฉโ€๐Ÿ’ป Implementation Updated the spec file. Prototype can be found here https://github.com/ni/nimble/pull/1788 ## ๐Ÿงช Testing N/A ## โœ… Checklist - [x] I have updated the project documentation to reflect my changes or determined no changes are needed. --- ...-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json | 7 + .../src/wafer-map/specs/features/rendering.md | 206 +++++++++++++++--- 2 files changed, 183 insertions(+), 30 deletions(-) create mode 100644 change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json diff --git a/change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json b/change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json new file mode 100644 index 0000000000..3069713da1 --- /dev/null +++ b/change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json @@ -0,0 +1,7 @@ +{ + "type": "none", + "comment": "Update rendering spec for Wafer Map component with API changes", + "packageName": "@ni/nimble-components", + "email": "33986780+munteannatan@users.noreply.github.com", + "dependentChangeType": "none" +} diff --git a/packages/nimble-components/src/wafer-map/specs/features/rendering.md b/packages/nimble-components/src/wafer-map/specs/features/rendering.md index effc10146d..517ba80f8a 100644 --- a/packages/nimble-components/src/wafer-map/specs/features/rendering.md +++ b/packages/nimble-components/src/wafer-map/specs/features/rendering.md @@ -10,7 +10,6 @@ The proposed design should consider the following factors: - Minimize rendering time and improve overall performance - Measure and improve performance metrics - Maintain compatibility with existing design patterns and web standards -- Avoid introducing new requirements on clients or breaking any APIs - Address any potential impact on testing, documentation, security, and other relevant areas By addressing these challenges, we aim to enhance the rendering capabilities of our application and provide a smoother and more responsive user interface. @@ -82,46 +81,42 @@ The POC is found in this branch [Worker Rendering POC](https://github.com/ni/nim ### Data Structure and Interface -We have two possible solutions for representing the data in the memory. They will be decided with a spec update. The fist one is an in-house solution: +The best solution to solve the API of the wafermap is to use Apache Arrow as the wafer component API, and Typed Arrays as the worker API for their iterating performance and transferability to worker threads. + +The Public API will be the following: ```TS -class WaferData { - // the x coordinates of each column of dies - dieColIndexArray: Int32Array; - // the lengths of each row of dies - rowLengthsArray: Int32Array; - // the y coordinates of each die as a matrix row by row - dieRowIndexLayer: Int32Array; - // the value of each die as a matrix row by row - dieValuesLayer: Float64Array; - // the highlight approach is still undecided, we have two options: - // the highlight state of each die as a matrix; user will have to pre-calculate tags into highlighted conditions. - dieHighlightsLayer: Int8Array; - // a 32 bitset array of tags for each die; aligns more closely to the existing public api but limits users to 32 tags. - dieHighlightsLayer: Int32Array; - // metadata array for each die; it will not be sent to the worker - metadata : unknown[] +import { Table } from 'apache-arrow'; + +export class WaferMap extends FoundationElement { + ... + public diesTable: Table | undefined; + ... } ``` -Using TypedArrays has the benefit of direct transfer to web workers without structured cloning of the object by transferring the arrayBuffers and reconstructing the object. Other benefits of typedArrays include the low access time when iterating over the values, more memory efficiency and faster direct access to metadata layers values. The previous inputs can be adapted to this new structure to maintain backwards compatibility. +It will be using the [Apache Arrow Table](https://arrow.apache.org/docs/js/classes/Arrow_dom.Table.html). +It will require at least three columns for the `diesTable`: -This API will have [optimized byte-array interop from Blazor](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/6.0/byte-array-interop) and should be supported by Angular as a [vanilla javascript feature](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer). +- The `rowIndex` and `colIndex` will be `Int32` columns +- The `value` will be a `Float64` column. -The alternative to the above mentioned data structure is an [apache arrow](https://arrow.apache.org/docs/js/index.html) table with columns and metadata. +They will be checked at runtime and a `WaferMapValidity` flag will be raised signaling an `invalidTableInput`. -Pros of using Apache Arrow: +The schema will be extensible. This will induce a breaking change in the API, as the metadata which was previously `unknown` will not recorded in table, but the hover event will reference an index which can be used by the client to select the metadata outside the component. -- A row based format that aligns well with the existing public api -- Well supported and tested format -- Nice public API to use, we don't have to invent a new format, just document our schema for the arrow tables -- Designed for large dataset visualizations +This approach has the benefits of a row based format that aligns well with the existing public API, as well as a nice public API that easily allows future improvements. It allows for more advanced filtering techniques such as using inner and outer joins for tables, slicing the tables to distribute values to separate workers and applying operations over whole columns. + +We are going to split the columns relevant to rendering from the table (rows, columns, values) and transfer them to the worker separately. This can be done with a very small overhead using the method below on the resulting vector. After being transferred, the buffers can be cached to speed up value access and filtering. -In order to choose from these alternatives we will prototype and check: +```TS + const colIndex: Int32Array = diesTable.getChild('colIndex').toArray(); + const rowIndex: Int32Array = diesTable.getChild('rowIndex').toArray(); + ... +``` -- Does it have comparable memory performance -- Does it perform well or have significant overhead -- Is it easy to divide and use in parallel +The [JavaScript implementation of Apache Arrow](https://arrow.apache.org/docs/js/index.html) provides TypeScript Types which will work in Angular applications. +The [C# implementation of Apache Arrow](https://github.com/apache/arrow/blob/main/csharp/README.md) is also providing support for reading Arrow IPC streams which can be used to convert inputs from Blazor. ### Rendering @@ -210,8 +205,69 @@ The current expectation is for a singular wafer component to be displayed on the ### Alternative Data Structures and Interfaces +The alternative to using Apache Arrow tables is an in-house solution: + +```TS +class WaferData { + // the x coordinates of each column of dies + dieColIndexArray: Int32Array; + // the lengths of each row of dies + rowLengthsArray: Int32Array; + // the y coordinates of each die as a matrix row by row + dieRowIndexLayer: Int32Array; + // the value of each die as a matrix row by row + dieValuesLayer: Float64Array; + // we have two options to highlight: + // the highlight state of each die as a matrix; user will have to pre-calculate tags into highlighted conditions. + dieHighlightsLayer: Int8Array; + // a 32 bitset array of tags for each die; aligns more closely to the existing public api but limits users to 32 tags. + dieHighlightsLayer: Int32Array; + // metadata array for each die; it will not be sent to the worker + metadata : unknown[] +} +``` + +Using TypedArrays has the benefit of direct transfer to web workers without structured cloning of the object by transferring the arrayBuffers and reconstructing the object. Other benefits of typedArrays include the low access time when iterating over the values, more memory efficiency and faster direct access to metadata layers values. The previous inputs can be adapted to this new structure to maintain backwards compatibility. + +This API will have [optimized byte-array interop from Blazor](https://learn.microsoft.com/en-us/dotnet/core/compatibility/aspnet-core/6.0/byte-array-interop) and should be supported by Angular as a [vanilla javascript feature](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer). + +Pros of using Apache Arrow: + +- A row based format that aligns well with the existing public api +- Well supported and tested format +- Nice public API to use, we don't have to invent a new format, just document our schema for the arrow tables +- Designed for large dataset visualizations + Another option is to break each object property as a separate attribute for the wafer map component. This can also lead to increased complexity and confusion for the user which will need to pass several structured objects instead of a singular object. +#### Alternative Iteration and Filtering with Apache Arrow Table + +The limits for the apache arrow table approach are the following: + +1. Apache arrow documentation shows [list types are supported](https://arrow.apache.org/docs/status.html) but more research is needed to understand their usage, if they are useful in Tables, and their performance if used. +2. There is no support currently for [searching or filtering the table](https://github.com/apache/arrow/issues/13233). Searching for dies based on their position is crucial for highlighting and sending the highlighted die metadata with the `die-hover` event. More research is needed to see if alternative libraries can be used for filtering / data analysis. ([POC with arquero](https://stackblitz.com/edit/geoarrow-worker-arquero-demo?file=src%2Fmain.ts)). +3. Apache arrow does not yet have first class support for efficiently transferring arrow data structures in workers. When asked they said they are [supportive of adding the APIs](https://github.com/apache/arrow/issues/39017#issuecomment-1955653556). +4. The iteration over stored rows is very slow compared to typed arrays as seen in the table below. This impacts the goals we set for this rendering improvement. + +Alternatives for solving these problems are the following: + +1. A dynamic number of columns for storing tags, but the performance may suffer. +2. Possible solutions for this are searching by iterating over the whole table, which is not feasible (see 4.) or using typed arrays and caching to speed up the search for the relevant columns. +3. The use of a higher level library [geoarrow](https://github.com/geoarrow/geoarrow-js/blob/main/src/worker/transferable.ts). ([POC](https://stackblitz.com/edit/geoarrow-worker-arquero-demo?file=src%2Fmain.ts)). +4. In the following table are presented different iteration strategies over 1M long arrays, and how they compare with the chosen method and the basic typed array iteration: + +| name | duration (ms) [1] | duration (ms) [2] | detail | +| ----------------------- | ----------------- | ----------------- | --------------------------------------------------------------- | +| typed array | 7 | 6 | basic typed arrays iteration | +| typed array from table | 6 | 5 | typed arrays converted from Table columns | +| vector from typed array | 76 | 66 | arrow Vectors directly created from typed arrays | +| vector from table | 965 | 1012 | arrow Vector converted from the arrow Table with `makeVector()` | +| list array from table | 943 | 980 | list array converted from the arrow Table with `toArray()` | +| table get() | 1350 | 1030 | arrow Table using `table.get(rowIndex)` | +| table [iterator] | 1091 | 1011 | arrow Table using the [iterator] | + +The memory impact is not very significant, amounting to 74.01MB for 1M dies compared with 44.65MB for the previously prototyped API. + ### Alternative Rendering Alternatives to the described rendering are splitting the data and canvas and using multiple threads to enhance performance even more. This approach introduces the overhead of managing multiple canvases, splitting the dataset and handling any race conditions, which may not be needed if the single worker approach satisfies the performance requirements. @@ -228,6 +284,95 @@ We may also implement an external queue canceling functionality. ## Open Issues +### Highlighting API + +```TS +import { Table } from 'apache-arrow'; + +export class WaferMap extends FoundationElement { + ... + public highlightedTable: Table | undefined; + ... +} +``` + +The `highlightedTable` will have the same columns as the `diesTable`, but they will contain rows only partially filled with values, which will be used to filter the `diesTable` and enable highlighting. The values which are not empty on each individual row, including `colIndex`, `rowIndex`, `value` and others will be used to filter the table as an `AND` operation. Multiple rows will be used as filters with the `OR` operation. More details regarding highlights will be discussed in an open issue. + +The current proposal is for the highlight table to be used as a filter for the main dies table. This can be realized by using the [`semijoin`](https://uwdata.github.io/arquero/api/verbs#semijoin) operation from the Arquero library. this will function as follows. + +The main table: + +| (index) | colIndex | rowIndex | value | firstTag | secondTag | +| ------- | -------- | -------- | ------------------ | -------- | --------- | +| 0 | 0 | 2 | 14.239999771118164 | a | b | +| 1 | 1 | 2 | 76.43000030517578 | b | c | +| 2 | 1 | 1 | 44.630001068115234 | g | null | +| 3 | 1 | 3 | 67.93000030517578 | a | null | +| 4 | 2 | 2 | 72.70999908447266 | h | e | +| 5 | 2 | 1 | 79.04000091552734 | b | null | +| 6 | 2 | 0 | 26.489999771118164 | c | null | +| 7 | 2 | 3 | 37.790000915527344 | null | null | +| 8 | 2 | 4 | 59.81999969482422 | null | null | +| 9 | 3 | 2 | 52.900001525878906 | null | null | +| 10 | 3 | 1 | 98.5 | g | null | +| 11 | 3 | 3 | 20.829999923706055 | c | null | +| 12 | 4 | 2 | 62.79999923706055 | g | null | + +The highlight table: + +| (index) | firstTag | +| ------- | -------- | +| 0 | a | +| 1 | b | +| 2 | c | + +The filtered table: + +| (index) | colIndex | rowIndex | value | firstTag | secondTag | +| ------- | -------- | -------- | ------------------ | -------- | --------- | +| 0 | 0 | 2 | 14.239999771118164 | a | b | +| 1 | 1 | 2 | 76.43000030517578 | b | c | +| 2 | 1 | 3 | 67.93000030517578 | a | null | +| 3 | 2 | 1 | 79.04000091552734 | b | null | +| 4 | 2 | 0 | 26.489999771118164 | c | null | +| 5 | 3 | 3 | 20.829999923706055 | c | null | + +The filter matched the rows with the same values from the highlight table. This can be used for tags when filtering value ranges, values themselves, column and row indexes or other types of supported data types. + +The details of the implementation and more refined filtering will be discussed. + +Another option is using the existing `highlightedTags` API with a `List` column in the table [(listed as supported)](https://arrow.apache.org/docs/status.html). + +Specific open questions: + +- For a highlightedTags table api: + - Should the API be constrained in the [supported arrow types](https://arrow.apache.org/docs/status.html)? + - Should just primitive types be supported? + - Should just types supported in the JavaScript and C# languages be supported? + - Should just types well-supported in all existing implementation be supported? + - Would certain types, ex. like strings, lead to poor performance and be discouraged? + - Are there implementation challenges transferring buffers of arbitrary types across the Web Worker boundary? + - Should the API be constrained in the set of columns that participate in highlighting instead of all columns? Maybe columns with a specific name prefix like `highlighted_`? + - Is there real known benefits for specifying per floating point `value`? Or specific `rowIndex` / `columnIndex` independently? + - Does the highlightTable need to contain all columns used in diesTable? Can it just be a subset of columns? +- For a tags columns api: + - Do columns of List work in tables? + - Do dictionary columns work in tables to improve efficiency compared to List? + - What is the performance of a List / Dictionary column api compared to the alternatives? + +### Rendering Iterating + +From preliminary tests it seems that typed array iteration is the most performant approach for rendering. +Further inquiries will be made of apache-arrow dev team to make sure the best approach. + +### Highlights and Metadata + +We decided to use [arquero](https://uwdata.github.io/arquero/) to filter highlighted dies and metadata. +This approach shows promise, but it may pose a risk. +If it will be apparent that it's not useful, we will resort to reusing and adapting the existing logic. + +### Progress Indicator + User Indication for [interactions in progress (>200ms)](https://web.dev/articles/inp) possibilities: - the wafer-map itself will show a spinner @@ -235,6 +380,7 @@ User Indication for [interactions in progress (>200ms)](https://web.dev/articles - the wafer-map will use bitmap scaling in addition to a spinner - the wafer-map will immediately show the spinner / fire event or only after, for example 200ms - the renderer will report progress for larger wait times. +- the rendering will be done sequentially in animation frames so the user will see the progress at 60Hz A follow-on HLD update will specify the approved decision. From 7a142576616a115e3c3092fe3613ae05fba71f6d Mon Sep 17 00:00:00 2001 From: rajsite Date: Thu, 29 Feb 2024 07:48:58 +0000 Subject: [PATCH 5/5] applying package updates [skip ci] --- ...ents-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json | 7 ------- packages/nimble-components/CHANGELOG.json | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 7 deletions(-) delete mode 100644 change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json diff --git a/change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json b/change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json deleted file mode 100644 index 3069713da1..0000000000 --- a/change/@ni-nimble-components-c0f72f9f-b879-43ad-bc5b-ee8b74e2700f.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "type": "none", - "comment": "Update rendering spec for Wafer Map component with API changes", - "packageName": "@ni/nimble-components", - "email": "33986780+munteannatan@users.noreply.github.com", - "dependentChangeType": "none" -} diff --git a/packages/nimble-components/CHANGELOG.json b/packages/nimble-components/CHANGELOG.json index 2ba209de3c..1ff68783f9 100644 --- a/packages/nimble-components/CHANGELOG.json +++ b/packages/nimble-components/CHANGELOG.json @@ -1,6 +1,21 @@ { "name": "@ni/nimble-components", "entries": [ + { + "date": "Thu, 29 Feb 2024 07:48:58 GMT", + "version": "21.7.1", + "tag": "@ni/nimble-components_v21.7.1", + "comments": { + "none": [ + { + "author": "33986780+munteannatan@users.noreply.github.com", + "package": "@ni/nimble-components", + "commit": "f3ec1ba17e88c1669c133cb5bebf5cb5ec09546c", + "comment": "Update rendering spec for Wafer Map component with API changes" + } + ] + } + }, { "date": "Wed, 28 Feb 2024 23:45:15 GMT", "version": "21.7.1",