From 2741422c0aa0c4fb10595f3f58bd04c647584179 Mon Sep 17 00:00:00 2001 From: Neil <55785687+carkom@users.noreply.github.com> Date: Wed, 6 Dec 2023 17:33:19 +0000 Subject: [PATCH] Custom Reports optimization (#1988) * Range fix and payee fix * bug fixes and UI tweaks * range options, hover UI * Select - UnSelect All Buttons * fix hidden group bug * YAxis PrivacyFilter * notes * more privacyFilter graphs * overflowY fix * Loading Indicator * Fix Filter button hover * data revamp * review fixes * LoadingIndicator fixes * remove a loop * filterbuttontype * lint fixes * review fixes * filtersbutton * updates * Split out functions to separate files * uncategorized optimization * rename ambiguous variables * notes * remove indexStack * renaming variables * Improve scrolling of tableGraph * revert renaming variables * code fixes * lint fixes * review fixes * fix * review fixes * variable name changes * const eslint fixes * remove indexStack --- .../{ChooseGraph.js => ChooseGraph.tsx} | 82 +-- .../src/components/reports/ReportOptions.tsx | 132 +++++ .../src/components/reports/ReportTable.tsx | 31 +- .../components/reports/ReportTableHeader.tsx | 154 ++--- .../components/reports/ReportTableList.tsx | 8 +- .../components/reports/ReportTableTotals.tsx | 186 +++--- .../components/reports/graphs/AreaGraph.tsx | 8 +- .../components/reports/graphs/BarGraph.tsx | 10 +- .../reports/graphs/BarLineGraph.tsx | 12 +- .../components/reports/graphs/DonutGraph.tsx | 12 +- .../components/reports/graphs/LineGraph.tsx | 7 +- .../reports/graphs/StackedBarGraph.tsx | 9 +- .../reports/reports/CustomReport.js | 56 +- .../reports/reports/CustomReportCard.js | 8 +- .../spreadsheets/default-spreadsheet.tsx | 537 ++++-------------- .../reports/spreadsheets/filterHiddenItems.ts | 24 + .../spreadsheets/grouped-spreadsheet.ts | 141 +++++ .../reports/spreadsheets/makeQuery.ts | 86 +++ .../reports/spreadsheets/recalculate.ts | 69 +++ .../src/components/reports/util.ts | 11 - packages/loot-core/src/server/aql/compiler.ts | 6 + packages/loot-core/src/types/models/rule.d.ts | 3 +- upcoming-release-notes/1988.md | 6 + 23 files changed, 882 insertions(+), 716 deletions(-) rename packages/desktop-client/src/components/reports/{ChooseGraph.js => ChooseGraph.tsx} (62%) create mode 100644 packages/desktop-client/src/components/reports/spreadsheets/filterHiddenItems.ts create mode 100644 packages/desktop-client/src/components/reports/spreadsheets/grouped-spreadsheet.ts create mode 100644 packages/desktop-client/src/components/reports/spreadsheets/makeQuery.ts create mode 100644 packages/desktop-client/src/components/reports/spreadsheets/recalculate.ts create mode 100644 upcoming-release-notes/1988.md diff --git a/packages/desktop-client/src/components/reports/ChooseGraph.js b/packages/desktop-client/src/components/reports/ChooseGraph.tsx similarity index 62% rename from packages/desktop-client/src/components/reports/ChooseGraph.js rename to packages/desktop-client/src/components/reports/ChooseGraph.tsx index bf2edbb2e44..4dca723153a 100644 --- a/packages/desktop-client/src/components/reports/ChooseGraph.js +++ b/packages/desktop-client/src/components/reports/ChooseGraph.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useRef } from 'react'; import View from '../common/View'; @@ -15,8 +15,6 @@ import ReportTableList from './ReportTableList'; import ReportTableTotals from './ReportTableTotals'; export function ChooseGraph({ - start, - end, data, mode, graphType, @@ -27,18 +25,23 @@ export function ChooseGraph({ setScrollWidth, months, }) { - function saveScrollWidth(parent, child) { - const width = parent > 0 && child > 0 && parent - child; + const saveScrollWidth = value => { + setScrollWidth(!value ? 0 : value); + }; - setScrollWidth(!width ? 0 : width); - } + const headerScrollRef = useRef(null); + const listScrollRef = useRef(null); + const totalScrollRef = useRef(null); + + const handleScrollTotals = scroll => { + headerScrollRef.current.scrollLeft = scroll.target.scrollLeft; + listScrollRef.current.scrollLeft = scroll.target.scrollLeft; + }; if (graphType === 'AreaGraph') { return ( @@ -48,8 +51,6 @@ export function ChooseGraph({ return ( - ); + return ; } if (graphType === 'DonutGraph') { return ( - ); + return ; } if (graphType === 'StackedBarGraph') { - return ( - - ); + return ; } if (graphType === 'TableGraph') { return ( - + - + - + ); } diff --git a/packages/desktop-client/src/components/reports/ReportOptions.tsx b/packages/desktop-client/src/components/reports/ReportOptions.tsx index 73e2fff5b51..69f0ef1626c 100644 --- a/packages/desktop-client/src/components/reports/ReportOptions.tsx +++ b/packages/desktop-client/src/components/reports/ReportOptions.tsx @@ -1,3 +1,10 @@ +import { + type AccountEntity, + type CategoryEntity, + type CategoryGroupEntity, + type PayeeEntity, +} from 'loot-core/src/types/models'; + const balanceTypeOptions = [ { description: 'Expense', format: 'totalDebts' }, { description: 'Income', format: 'totalAssets' }, @@ -44,3 +51,128 @@ const intervalOptions = [ { value: 5, description: 'Yearly', name: 5, ]; */ +export type QueryDataEntity = { + date: string; + category: string; + categoryGroup: string; + account: string; + accountOffBudget: boolean; + payee: string; + transferAccount: string; + amount: number; +}; + +export type UncategorizedEntity = CategoryEntity & { + /* + When looking at uncategorized and hidden transactions we + need a way to group them. To do this we give them a unique + uncategorized_id. We also need a way to filter the + transctions from our query. For this we use the 3 variables + below. + */ + uncategorized_id: string; + is_off_budget: boolean; + is_transfer: boolean; + has_category: boolean; +}; + +const uncategorizedCategory: UncategorizedEntity = { + name: 'Uncategorized', + id: null, + uncategorized_id: '1', + hidden: false, + is_off_budget: false, + is_transfer: false, + has_category: false, +}; +const transferCategory: UncategorizedEntity = { + name: 'Transfers', + id: null, + uncategorized_id: '2', + hidden: false, + is_off_budget: false, + is_transfer: true, + has_category: false, +}; +const offBudgetCategory: UncategorizedEntity = { + name: 'Off Budget', + id: null, + uncategorized_id: '3', + hidden: false, + is_off_budget: true, + is_transfer: false, + has_category: true, +}; + +type UncategorizedGroupEntity = CategoryGroupEntity & { + categories?: UncategorizedEntity[]; +}; + +const uncategouncatGrouprizedGroup: UncategorizedGroupEntity = { + name: 'Uncategorized & Off Budget', + id: null, + hidden: false, + categories: [uncategorizedCategory, transferCategory, offBudgetCategory], +}; + +export const categoryLists = ( + showOffBudgetHidden: boolean, + showUncategorized: boolean, + categories: { list: CategoryEntity[]; grouped: CategoryGroupEntity[] }, +) => { + const categoryList = showUncategorized + ? [ + ...categories.list, + uncategorizedCategory, + transferCategory, + offBudgetCategory, + ] + : categories.list; + const categoryGroup = showUncategorized + ? [ + ...categories.grouped.filter(f => showOffBudgetHidden || !f.hidden), + uncategouncatGrouprizedGroup, + ] + : categories.grouped; + return [categoryList, categoryGroup] as const; +}; + +export const groupBySelections = ( + groupBy: string, + categoryList: CategoryEntity[], + categoryGroup: CategoryGroupEntity[], + payees: PayeeEntity[], + accounts: AccountEntity[], +) => { + let groupByList; + let groupByLabel; + switch (groupBy) { + case 'Category': + groupByList = categoryList; + groupByLabel = 'category'; + break; + case 'Group': + groupByList = categoryGroup; + groupByLabel = 'categoryGroup'; + break; + case 'Payee': + groupByList = payees; + groupByLabel = 'payee'; + break; + case 'Account': + groupByList = accounts; + groupByLabel = 'account'; + break; + case 'Month': + groupByList = categoryList; + groupByLabel = 'category'; + break; + case 'Year': + groupByList = categoryList; + groupByLabel = 'category'; + break; + default: + throw new Error('Error loading data into the spreadsheet.'); + } + return [groupByList, groupByLabel]; +}; diff --git a/packages/desktop-client/src/components/reports/ReportTable.tsx b/packages/desktop-client/src/components/reports/ReportTable.tsx index 11673cc45d1..a16f47c4cd9 100644 --- a/packages/desktop-client/src/components/reports/ReportTable.tsx +++ b/packages/desktop-client/src/components/reports/ReportTable.tsx @@ -1,24 +1,37 @@ -import React, { useLayoutEffect, useRef } from 'react'; +import React, { useLayoutEffect, useRef, type ReactNode } from 'react'; +import { type RefProp } from 'react-spring'; +import { type CSSProperties } from '../../style'; import View from '../common/View'; -export default function ReportTable({ saveScrollWidth, style, children }) { - const contentRef = useRef(); +type ReportTableProps = { + saveScrollWidth?: (value: number) => void; + listScrollRef?: RefProp; + style?: CSSProperties; + children?: ReactNode; +}; + +export default function ReportTable({ + saveScrollWidth, + listScrollRef, + style, + children, +}: ReportTableProps) { + const contentRef = useRef(null); useLayoutEffect(() => { if (contentRef.current && saveScrollWidth) { - saveScrollWidth( - contentRef.current.offsetParent - ? contentRef.current.parentElement.offsetWidth - : 0, - contentRef.current ? contentRef.current.offsetWidth : 0, - ); + saveScrollWidth(contentRef.current ? contentRef.current.offsetWidth : 0); } }); return ( ; + balanceType: string; + headerScrollRef?: Ref; +}; + export default function ReportTableHeader({ scrollWidth, groupBy, interval, balanceType, -}) { + headerScrollRef, +}: ReportTableHeaderProps) { return ( - - - {interval - ? interval.map(header => { - return ( - - ); - }) - : balanceType === 'Net' && ( - <> - - - - )} - - - {scrollWidth > 0 && } - + > + + {interval + ? interval.map(header => { + return ( + + ); + }) + : balanceType === 'Net' && ( + <> + + + + )} + + + {scrollWidth > 0 && } + + ); } diff --git a/packages/desktop-client/src/components/reports/ReportTableList.tsx b/packages/desktop-client/src/components/reports/ReportTableList.tsx index d789834dbae..ddfdeca2c40 100644 --- a/packages/desktop-client/src/components/reports/ReportTableList.tsx +++ b/packages/desktop-client/src/components/reports/ReportTableList.tsx @@ -6,7 +6,7 @@ import { integerToCurrency, } from 'loot-core/src/shared/util'; -import { styles, theme } from '../../style'; +import { type CSSProperties, styles, theme } from '../../style'; import View from '../common/View'; import { Row, Cell } from '../table'; @@ -18,11 +18,11 @@ type TableRowProps = { totalAssets: number; totalDebts: number; }; - balanceTypeOp?: string | null; + balanceTypeOp?: string; groupByItem: string; mode: string; monthsCount: number; - style?: object | null; + style?: CSSProperties; }; const TableRow = memo( @@ -197,7 +197,7 @@ export default function ReportTableList({ const groupByItem = ['Month', 'Year'].includes(groupBy) ? 'date' : 'name'; const groupByData = groupBy === 'Category' - ? 'groupData' + ? 'groupedData' : ['Month', 'Year'].includes(groupBy) ? 'monthData' : 'data'; diff --git a/packages/desktop-client/src/components/reports/ReportTableTotals.tsx b/packages/desktop-client/src/components/reports/ReportTableTotals.tsx index 1286a7f87dd..f60a91b5ab5 100644 --- a/packages/desktop-client/src/components/reports/ReportTableTotals.tsx +++ b/packages/desktop-client/src/components/reports/ReportTableTotals.tsx @@ -7,6 +7,7 @@ import { } from 'loot-core/src/shared/util'; import { styles, theme } from '../../style'; +import View from '../common/View'; import { Row, Cell } from '../table'; export default function ReportTableTotals({ @@ -15,100 +16,113 @@ export default function ReportTableTotals({ balanceTypeOp, mode, monthsCount, + totalScrollRef, + handleScrollTotals, }) { const average = amountToInteger(data[balanceTypeOp]) / monthsCount; return ( - - - {mode === 'time' - ? data.monthData.map(item => { - return ( - 100000 && - amountToCurrency(item[balanceTypeOp]) - } - width="flex" - privacyFilter - /> - ); - }) - : balanceTypeOp === 'totalTotals' && ( - <> - 100000 && - amountToCurrency(data.totalAssets) - } - width="flex" - /> - 100000 && - amountToCurrency(data.totalDebts) - } - width="flex" - /> - - )} - 100000 && - amountToCurrency(data[balanceTypeOp]) - } - width="flex" - privacyFilter - /> - 100000 && - integerToCurrency(Math.round(average)) - } - width="flex" - privacyFilter - /> + > + + {mode === 'time' + ? data.monthData.map(item => { + return ( + 100000 && + amountToCurrency(item[balanceTypeOp]) + } + width="flex" + privacyFilter + /> + ); + }) + : balanceTypeOp === 'totalTotals' && ( + <> + 100000 && + amountToCurrency(data.totalAssets) + } + width="flex" + /> + 100000 && + amountToCurrency(data.totalDebts) + } + width="flex" + /> + + )} + 100000 && + amountToCurrency(data[balanceTypeOp]) + } + width="flex" + privacyFilter + /> + 100000 && + integerToCurrency(Math.round(average)) + } + width="flex" + privacyFilter + /> - {scrollWidth > 0 && } - + {scrollWidth > 0 && } + + ); } diff --git a/packages/desktop-client/src/components/reports/graphs/AreaGraph.tsx b/packages/desktop-client/src/components/reports/graphs/AreaGraph.tsx index b50331b52f3..50022af9df1 100644 --- a/packages/desktop-client/src/components/reports/graphs/AreaGraph.tsx +++ b/packages/desktop-client/src/components/reports/graphs/AreaGraph.tsx @@ -93,8 +93,8 @@ const CustomTooltip = ({ type AreaGraphProps = { style?: CSSProperties; data; - balanceTypeOp; - compact: boolean; + balanceTypeOp: string; + compact?: boolean; }; function AreaGraph({ style, data, balanceTypeOp, compact }: AreaGraphProps) { @@ -151,7 +151,7 @@ function AreaGraph({ style, data, balanceTypeOp, compact }: AreaGraphProps) { )} {compact ? null : ( { type BarGraphProps = { style?: CSSProperties; data; - groupBy; + groupBy: string; balanceTypeOp; - empty; - compact: boolean; - domain?: { - y?: [number, number]; - }; + empty: boolean; + compact?: boolean; }; function BarGraph({ @@ -146,7 +143,6 @@ function BarGraph({ empty, balanceTypeOp, compact, - domain, }: BarGraphProps) { const privacyMode = usePrivacyMode(); diff --git a/packages/desktop-client/src/components/reports/graphs/BarLineGraph.tsx b/packages/desktop-client/src/components/reports/graphs/BarLineGraph.tsx index 390fa5e5a10..816ffc8d532 100644 --- a/packages/desktop-client/src/components/reports/graphs/BarLineGraph.tsx +++ b/packages/desktop-client/src/components/reports/graphs/BarLineGraph.tsx @@ -72,18 +72,10 @@ const CustomTooltip = ({ active, payload, label }: CustomTooltipProps) => { type BarLineGraphProps = { style?: CSSProperties; graphData; - compact: boolean; - domain?: { - y?: [number, number]; - }; + compact?: boolean; }; -function BarLineGraph({ - style, - graphData, - compact, - domain, -}: BarLineGraphProps) { +function BarLineGraph({ style, graphData, compact }: BarLineGraphProps) { const tickFormatter = tick => { return `${Math.round(tick).toLocaleString()}`; // Formats the tick values as strings with commas }; diff --git a/packages/desktop-client/src/components/reports/graphs/DonutGraph.tsx b/packages/desktop-client/src/components/reports/graphs/DonutGraph.tsx index 41798b395a4..a8ad506a00e 100644 --- a/packages/desktop-client/src/components/reports/graphs/DonutGraph.tsx +++ b/packages/desktop-client/src/components/reports/graphs/DonutGraph.tsx @@ -95,13 +95,10 @@ const CustomLegend = ({ active, payload, label }: CustomLegendProps) => { type DonutGraphProps = { style?: CSSProperties; data; - groupBy; - balanceTypeOp; - empty; - compact: boolean; - domain?: { - y?: [number, number]; - }; + groupBy: string; + balanceTypeOp: string; + empty: boolean; + compact?: boolean; }; function DonutGraph({ @@ -111,7 +108,6 @@ function DonutGraph({ empty, balanceTypeOp, compact, - domain, }: DonutGraphProps) { const colorScale = getColorScale('qualitative'); const yAxis = ['Month', 'Year'].includes(groupBy) ? 'date' : 'name'; diff --git a/packages/desktop-client/src/components/reports/graphs/LineGraph.tsx b/packages/desktop-client/src/components/reports/graphs/LineGraph.tsx index 1c373de3936..069082c2630 100644 --- a/packages/desktop-client/src/components/reports/graphs/LineGraph.tsx +++ b/packages/desktop-client/src/components/reports/graphs/LineGraph.tsx @@ -71,13 +71,10 @@ const CustomTooltip = ({ active, payload, label }: CustomTooltipProps) => { type LineGraphProps = { style?: CSSProperties; graphData; - compact: boolean; - domain?: { - y?: [number, number]; - }; + compact?: boolean; }; -function LineGraph({ style, graphData, compact, domain }: LineGraphProps) { +function LineGraph({ style, graphData, compact }: LineGraphProps) { const tickFormatter = tick => { return `${Math.round(tick).toLocaleString()}`; // Formats the tick values as strings with commas }; diff --git a/packages/desktop-client/src/components/reports/graphs/StackedBarGraph.tsx b/packages/desktop-client/src/components/reports/graphs/StackedBarGraph.tsx index 8b0ba322245..37e2b3ea41c 100644 --- a/packages/desktop-client/src/components/reports/graphs/StackedBarGraph.tsx +++ b/packages/desktop-client/src/components/reports/graphs/StackedBarGraph.tsx @@ -116,8 +116,7 @@ const CustomLegend = ({ active, payload, label }: CustomLegendProps) => { type StackedBarGraphProps = { style?: CSSProperties; data; - balanceTypeOp; - compact: boolean; + compact?: boolean; }; function StackedBarGraph({ style, data, compact }: StackedBarGraphProps) { @@ -132,14 +131,14 @@ function StackedBarGraph({ style, data, compact }: StackedBarGraphProps) { }} > {(width, height, portalHost) => - data.stackedData && ( + data.monthData && (
{!compact &&
} { @@ -163,7 +162,7 @@ function StackedBarGraph({ style, data, compact }: StackedBarGraphProps) { tickLine={{ stroke: theme.pageText }} /> )} - {data.groupBy.reverse().map((c, index) => ( + {data.data.reverse().map((c, index) => ( { + return groupedSpreadsheet({ + start, + end, + categories, + selectedCategories, + filters, + conditionsOp, + hidden, + uncat, + }); + }, [ + start, + end, + categories, + selectedCategories, + filters, + conditionsOp, + hidden, + uncat, + ]); + const getGraphData = useMemo(() => { setDataCheck(false); - return defaultSpreadsheet( + return defaultSpreadsheet({ start, end, - groupBy, - ReportOptions.balanceTypeMap.get(balanceType), categories, selectedCategories, - payees, - accounts, filters, conditionsOp, hidden, uncat, + groupBy, + balanceTypeOp, + payees, + accounts, setDataCheck, - ); + }); }, [ start, end, @@ -135,7 +158,10 @@ export default function CustomReport() { hidden, uncat, ]); - const data = useReport('default', getGraphData); + const graphData = useReport('default', getGraphData); + const groupedData = useReport('grouped', getGroupData); + + const data = { ...graphData, groupedData }; const [scrollWidth, setScrollWidth] = useState(0); @@ -267,15 +293,7 @@ export default function CustomReport() { right={ - {amountToCurrency( - Math.abs( - data[ - ReportOptions.balanceTypeMap.get( - balanceType, - ) - ], - ), - )} + {amountToCurrency(Math.abs(data[balanceTypeOp]))} } @@ -317,9 +335,7 @@ export default function CustomReport() { diff --git a/packages/desktop-client/src/components/reports/reports/CustomReportCard.js b/packages/desktop-client/src/components/reports/reports/CustomReportCard.js index 6ad6a7c4a28..b4b3d53114f 100644 --- a/packages/desktop-client/src/components/reports/reports/CustomReportCard.js +++ b/packages/desktop-client/src/components/reports/reports/CustomReportCard.js @@ -21,7 +21,13 @@ function CustomReportCard() { const groupBy = 'Category'; const getGraphData = useMemo(() => { - return defaultSpreadsheet(start, end, groupBy, 'totalDebts', categories); + return defaultSpreadsheet({ + start, + end, + groupBy, + balanceTypeOp: 'totalDebts', + categories, + }); }, [start, end, categories]); const data = useReport('default', getGraphData); diff --git a/packages/desktop-client/src/components/reports/spreadsheets/default-spreadsheet.tsx b/packages/desktop-client/src/components/reports/spreadsheets/default-spreadsheet.tsx index e3a9d037fd2..47c1cf8cdb1 100644 --- a/packages/desktop-client/src/components/reports/spreadsheets/default-spreadsheet.tsx +++ b/packages/desktop-client/src/components/reports/spreadsheets/default-spreadsheet.tsx @@ -1,61 +1,55 @@ import * as d from 'date-fns'; -import q, { runQuery } from 'loot-core/src/client/query-helpers'; +import { runQuery } from 'loot-core/src/client/query-helpers'; import { send } from 'loot-core/src/platform/client/fetch'; import * as monthUtils from 'loot-core/src/shared/months'; -import { integerToAmount, amountToInteger } from 'loot-core/src/shared/util'; - -import { index, indexStack } from '../util'; - -export default function createSpreadsheet( +import { integerToAmount } from 'loot-core/src/shared/util'; +import { + type AccountEntity, + type PayeeEntity, + type CategoryEntity, + type RuleConditionEntity, + type CategoryGroupEntity, +} from 'loot-core/src/types/models'; + +import { categoryLists, groupBySelections } from '../ReportOptions'; + +import filterHiddenItems from './filterHiddenItems'; +import makeQuery from './makeQuery'; +import recalculate from './recalculate'; + +export type createSpreadsheetProps = { + start: string; + end: string; + categories: { list: CategoryEntity[]; grouped: CategoryGroupEntity[] }; + selectedCategories: CategoryEntity[]; + conditions: RuleConditionEntity[]; + conditionsOp: string; + hidden: boolean; + uncat: boolean; + groupBy?: string; + balanceTypeOp?: string; + payees?: PayeeEntity[]; + accounts?: AccountEntity[]; + setDataCheck?: (value: boolean) => void; +}; + +export default function createSpreadsheet({ start, end, - groupBy, - balanceTypeOp, categories, selectedCategories, - payees, - accounts, conditions = [], conditionsOp, hidden, uncat, + groupBy, + balanceTypeOp, + payees, + accounts, setDataCheck, -) { - const uncatCat = { - name: 'Uncategorized', - id: null, - uncat_id: '1', - hidden: 0, - offBudget: false, - }; - const uncatTransfer = { - name: 'Transfers', - id: null, - uncat_id: '2', - hidden: 0, - transfer: false, - }; - const uncatOff = { - name: 'OffBudget', - id: null, - uncat_id: '3', - hidden: 0, - offBudget: true, - }; - - const uncatGroup = { - name: 'Uncategorized', - id: null, - hidden: 0, - categories: [uncatCat, uncatTransfer, uncatOff], - }; - const catList = uncat - ? [...categories.list, uncatCat, uncatTransfer, uncatOff] - : categories.list; - const catGroup = uncat - ? [...categories.grouped, uncatGroup] - : categories.grouped; +}) { + const [catList, catGroup] = categoryLists(hidden, uncat, categories); const categoryFilter = (catList || []).filter( category => @@ -66,35 +60,13 @@ export default function createSpreadsheet( ), ); - let groupByList; - let groupByLabel; - switch (groupBy) { - case 'Category': - groupByList = catList; - groupByLabel = 'category'; - break; - case 'Group': - groupByList = catList; - groupByLabel = 'category'; - break; - case 'Payee': - groupByList = payees; - groupByLabel = 'payee'; - break; - case 'Account': - groupByList = accounts; - groupByLabel = 'account'; - break; - case 'Month': - groupByList = catList; - groupByLabel = 'category'; - break; - case 'Year': - groupByList = catList; - groupByLabel = 'category'; - break; - default: - } + const [groupByList, groupByLabel] = groupBySelections( + groupBy, + catList, + catGroup, + payees, + accounts, + ); return async (spreadsheet, setData) => { if (groupByList.length === 0) { @@ -106,388 +78,99 @@ export default function createSpreadsheet( }); const conditionsOpKey = conditionsOp === 'or' ? '$or' : '$and'; - function makeQuery(splt, name) { - const query = q('transactions') - .filter( - //Show Offbudget and hidden categories - !hidden && { - $and: [ - { - 'account.offbudget': false, - $or: [ - { - 'category.hidden': false, - category: null, - }, - ], - }, - ], - $or: [ - { - 'payee.transfer_acct.offbudget': true, - 'payee.transfer_acct': null, - }, - ], - }, - ) - //Apply Category_Selector - .filter( - selectedCategories && { - $or: [ - { - category: null, - $or: categoryFilter.map(category => ({ - category: category.id, - })), - }, - ], - }, - ) - //Calculate uncategorized transactions when box checked - .filter( - splt.uncat_id === '2' - ? { - 'payee.transfer_acct.closed': false, - } - : { - 'payee.transfer_acct': null, - 'account.offbudget': splt.offBudget ? splt.offBudget : false, - }, - ) - //Apply filters and split by "Group By" - .filter({ - [conditionsOpKey]: [...filters], - [groupByLabel]: splt.id, - }) - //Apply month range filters - .filter({ - $and: [ - { date: { $transform: '$month', $gte: start } }, - { date: { $transform: '$month', $lte: end } }, - ], - }) - //Show assets or debts - .filter( - name === 'assets' ? { amount: { $gt: 0 } } : { amount: { $lt: 0 } }, - ); - - return query - .groupBy({ $month: '$date' }) - .select([ - { date: { $month: '$date' } }, - { [name]: { $sum: '$amount' } }, - ]); - } - - const graphData = await Promise.all( - groupByList.map(async splt => { - const [starting, assets, debts] = await Promise.all([ - runQuery( - q('transactions') - .filter( - !hidden && { - $and: [ - { - 'account.offbudget': false, - $or: [ - { - 'category.hidden': false, - category: null, - }, - ], - }, - ], - $or: [ - { - 'payee.transfer_acct.offbudget': true, - 'payee.transfer_acct': null, - }, - ], - }, - ) - .filter( - splt.uncat_id === '2' - ? { - 'payee.transfer_acct.closed': false, - } - : { - 'payee.transfer_acct': null, - 'account.offbudget': splt.offBudget - ? splt.offBudget - : false, - }, - ) - .filter( - selectedCategories && { - $or: categoryFilter.map(category => ({ - category: category.id, - })), - }, - ) - .filter({ - [conditionsOpKey]: [...filters], - [groupByLabel]: splt.id, - }) - .filter({ - $and: [{ date: { $lt: start + '-01' } }], - }) - .calculate({ $sum: '$amount' }), - ).then(({ data }) => data), - - runQuery(makeQuery(splt, 'assets')).then(({ data }) => data), - runQuery(makeQuery(splt, 'debts')).then(({ data }) => data), - ]); - - return { - id: splt.id, - uncat_id: splt.uncat_id, - name: splt.name, - starting, - hidden: splt.hidden, - assets: index(assets, 'date'), - debts: index(debts, 'date'), - }; - }), - ); + const [assets, debts] = await Promise.all([ + runQuery( + makeQuery( + 'assets', + start, + end, + hidden, + selectedCategories, + categoryFilter, + conditionsOpKey, + filters, + ), + ).then(({ data }) => data), + runQuery( + makeQuery( + 'debts', + start, + end, + hidden, + selectedCategories, + categoryFilter, + conditionsOpKey, + filters, + ), + ).then(({ data }) => data), + ]); const months = monthUtils.rangeInclusive(start, end); - const calcData = graphData.map(graph => { - let graphStarting = 0; - const mon = months.map(month => { - let graphAssets = 0; - let graphDebts = 0; - if (graph.assets[month] || graph.debts[month]) { - if (graph.assets[month]) { - graphAssets += graph.assets[month].assets; - } - if (graph.debts[month]) { - graphDebts += graph.debts[month].debts; - } - } - - graphStarting += graph.starting; - return { - date: month, - assets: graphAssets, - debts: graphDebts, - }; - }); - - return { - id: graph.id, - uncat_id: graph.uncat_id, - name: graph.name, - starting: graphStarting, - hidden: graph.hidden, - balances: index(mon, 'date'), - }; - }); - - const categoryGroupCalcData = catGroup - .filter(f => hidden || f.hidden === 0) - .map(group => { - let groupedStarting = 0; - const mon = months.map(month => { - let groupedAssets = 0; - let groupedDebts = 0; - graphData.map(graph => { - if (graph.assets[month] || graph.debts[month]) { - if (group.categories.map(v => v.id).includes(graph.id)) { - if (graph.assets[month]) { - groupedAssets += graph.assets[month].assets; - } - if (graph.debts[month]) { - groupedDebts += graph.debts[month].debts; - } - } - } - - groupedStarting += graph.starting; - return null; - }); - return { - date: month, - assets: groupedAssets, - debts: groupedDebts, - }; - }); - - return { - id: group.id, - name: group.name, - starting: groupedStarting, - hidden: group.hidden, - balances: index(mon, 'date'), - }; - }); - - const groupByData = groupBy === 'Group' ? categoryGroupCalcData : calcData; - - const data = groupByData.map(graph => { - const calc = recalculate(graph, start, end); - return { ...calc }; - }); - - const categoryGroupData = catGroup - .filter(f => hidden || f.hidden === 0) - .map(group => { - const catData = group.categories.map(graph => { - let catMatch = null; - calcData.map(cat => { - if ( - cat.id === null - ? cat.uncat_id === graph.uncat_id - : cat.id === graph.id - ) { - catMatch = cat; - } - return null; - }); - const calcCat = catMatch && recalculate(catMatch, start, end); - return { ...calcCat }; - }); - let groupMatch = null; - categoryGroupCalcData.map(split => { - if (split.id === group.id) { - groupMatch = split; - } - return null; - }); - const calcGroup = groupMatch && recalculate(groupMatch, start, end); - return { - ...calcGroup, - categories: catData, - }; - }); let totalAssets = 0; let totalDebts = 0; - let totalTotals = 0; - const monthData = months.map(month => { + const monthData = months.reduce((arr, month) => { let perMonthAssets = 0; let perMonthDebts = 0; - let perMonthTotals = 0; - graphData.map(graph => { - if (graph.assets[month] || graph.debts[month]) { - if (graph.assets[month]) { - perMonthAssets += graph.assets[month].assets; - } - if (graph.debts[month]) { - perMonthDebts += graph.debts[month].debts; - } - perMonthTotals = perMonthAssets + perMonthDebts; + const stacked = {}; + + groupByList.map(item => { + let stackAmounts = 0; + + const monthAssets = filterHiddenItems(item, assets) + .filter( + asset => asset.date === month && asset[groupByLabel] === item.id, + ) + .reduce((a, v) => (a = a + v.amount), 0); + perMonthAssets += monthAssets; + + const monthDebts = filterHiddenItems(item, debts) + .filter(debt => debt.date === month && debt[groupByLabel] === item.id) + .reduce((a, v) => (a = a + v.amount), 0); + perMonthDebts += monthDebts; + + if (balanceTypeOp === 'totalAssets') { + stackAmounts += monthAssets; } + if (balanceTypeOp === 'totalDebts') { + stackAmounts += monthDebts; + } + if (stackAmounts !== 0) { + stacked[item.name] = integerToAmount(Math.abs(stackAmounts)); + } + return null; }); totalAssets += perMonthAssets; totalDebts += perMonthDebts; - totalTotals += perMonthTotals; - return { + arr.push({ // eslint-disable-next-line rulesdir/typography date: d.format(d.parseISO(`${month}-01`), "MMM ''yy"), + ...stacked, totalDebts: integerToAmount(perMonthDebts), totalAssets: integerToAmount(perMonthAssets), - totalTotals: integerToAmount(perMonthTotals), - }; - }); - - const stackedData = months.map(month => { - const stacked = data.map(graph => { - let stackAmounts = 0; - if (graph.indexedMonthData[month]) { - stackAmounts += graph.indexedMonthData[month][balanceTypeOp]; - } - return { - name: graph.name, - id: graph.id, - amount: Math.abs(stackAmounts), - }; + totalTotals: integerToAmount(perMonthDebts + perMonthAssets), }); - const indexedStack = indexStack( - stacked.filter(i => i[balanceTypeOp] !== 0), - 'name', - 'amount', - ); - return { - // eslint-disable-next-line rulesdir/typography - date: d.format(d.parseISO(`${month}-01`), "MMM ''yy"), - ...indexedStack, - }; + return arr; + }, []); + + const calcData = groupByList.map(item => { + const calc = recalculate({ item, months, assets, debts, groupByLabel }); + return { ...calc }; }); setData({ - stackedData, - groupBy: groupBy === 'Group' ? catGroup : groupByList, - data, - groupData: categoryGroupData, + data: calcData, monthData, start, end, totalDebts: integerToAmount(totalDebts), totalAssets: integerToAmount(totalAssets), - totalTotals: integerToAmount(totalTotals), + totalTotals: integerToAmount(totalAssets + totalDebts), }); setDataCheck?.(true); }; } - -function recalculate(item, start, end) { - const months = monthUtils.rangeInclusive(start, end); - - let totalDebts = 0; - let totalAssets = 0; - let totalTotals = 0; - let exists = false; - - const monthData = months.reduce((arr, month) => { - let debts = 0; - let assets = 0; - let total = 0; - const last = arr.length === 0 ? null : arr[arr.length - 1]; - - if (item.balances[month]) { - exists = true; - if (item.balances[month].debts) { - debts += item.balances[month].debts; - totalDebts += item.balances[month].debts; - } - if (item.balances[month].assets) { - assets += item.balances[month].assets; - totalAssets += item.balances[month].assets; - } - total = assets + debts; - totalTotals = totalAssets + totalDebts; - } - - const dateParse = d.parseISO(`${month}-01`); - const change = last ? total - amountToInteger(last.totalTotals) : 0; - - arr.push({ - dateParse, - totalTotals: integerToAmount(total), - totalAssets: integerToAmount(assets), - totalDebts: integerToAmount(debts), - totalChange: integerToAmount(change), - // eslint-disable-next-line rulesdir/typography - date: d.format(dateParse, "MMM ''yy"), - dateLookup: month, - }); - - return arr; - }, []); - - const indexedMonthData = exists ? index(monthData, 'dateLookup') : monthData; - - return { - indexedMonthData, - monthData, - totalAssets: integerToAmount(totalAssets), - totalDebts: integerToAmount(totalDebts), - totalTotals: integerToAmount(totalTotals), - id: item.id, - name: item.name, - }; -} diff --git a/packages/desktop-client/src/components/reports/spreadsheets/filterHiddenItems.ts b/packages/desktop-client/src/components/reports/spreadsheets/filterHiddenItems.ts new file mode 100644 index 00000000000..0498ad09357 --- /dev/null +++ b/packages/desktop-client/src/components/reports/spreadsheets/filterHiddenItems.ts @@ -0,0 +1,24 @@ +import { + type QueryDataEntity, + type UncategorizedEntity, +} from '../ReportOptions'; + +function filterHiddenItems(item: UncategorizedEntity, data: QueryDataEntity[]) { + return data.filter(asset => { + if (!item.uncategorized_id) { + return true; + } + + const isTransfer = item.is_transfer + ? asset.transferAccount + : !asset.transferAccount; + const isHidden = item.has_category ? true : !asset.category; + const isOffBudget = item.is_off_budget + ? asset.accountOffBudget + : !asset.accountOffBudget; + + return isTransfer && isHidden && isOffBudget; + }); +} + +export default filterHiddenItems; diff --git a/packages/desktop-client/src/components/reports/spreadsheets/grouped-spreadsheet.ts b/packages/desktop-client/src/components/reports/spreadsheets/grouped-spreadsheet.ts new file mode 100644 index 00000000000..90d679c046e --- /dev/null +++ b/packages/desktop-client/src/components/reports/spreadsheets/grouped-spreadsheet.ts @@ -0,0 +1,141 @@ +import { runQuery } from 'loot-core/src/client/query-helpers'; +import { send } from 'loot-core/src/platform/client/fetch'; +import * as monthUtils from 'loot-core/src/shared/months'; +import { integerToAmount } from 'loot-core/src/shared/util'; + +import { categoryLists } from '../ReportOptions'; + +import { type createSpreadsheetProps } from './default-spreadsheet'; +import filterHiddenItems from './filterHiddenItems'; +import makeQuery from './makeQuery'; +import recalculate from './recalculate'; + +function createGroupedSpreadsheet({ + start, + end, + categories, + selectedCategories, + conditions = [], + conditionsOp, + hidden, + uncat, +}: createSpreadsheetProps) { + const [catList, catGroup] = categoryLists(hidden, uncat, categories); + + const categoryFilter = (catList || []).filter( + category => + !category.hidden && + selectedCategories && + selectedCategories.some( + selectedCategory => selectedCategory.id === category.id, + ), + ); + + return async (spreadsheet, setData) => { + if (catList.length === 0) { + return null; + } + + const { filters } = await send('make-filters-from-conditions', { + conditions: conditions.filter(cond => !cond.customName), + }); + const conditionsOpKey = conditionsOp === 'or' ? '$or' : '$and'; + + const [assets, debts] = await Promise.all([ + runQuery( + makeQuery( + 'assets', + start, + end, + hidden, + selectedCategories, + categoryFilter, + conditionsOpKey, + filters, + ), + ).then(({ data }) => data), + runQuery( + makeQuery( + 'debts', + start, + end, + hidden, + selectedCategories, + categoryFilter, + conditionsOpKey, + filters, + ), + ).then(({ data }) => data), + ]); + + const months = monthUtils.rangeInclusive(start, end); + + const groupedData = catGroup.map( + group => { + let totalAssets = 0; + let totalDebts = 0; + + const monthData = months.reduce((arr, month) => { + let groupedAssets = 0; + let groupedDebts = 0; + + group.categories.map(item => { + const monthAssets = filterHiddenItems(item, assets) + .filter( + asset => asset.date === month && asset.category === item.id, + ) + .reduce((a, v) => (a = a + v.amount), 0); + groupedAssets += monthAssets; + + const monthDebts = filterHiddenItems(item, debts) + .filter( + debts => debts.date === month && debts.category === item.id, + ) + .reduce((a, v) => (a = a + v.amount), 0); + groupedDebts += monthDebts; + + return null; + }); + + totalAssets += groupedAssets; + totalDebts += groupedDebts; + + arr.push({ + date: month, + totalAssets: integerToAmount(groupedAssets), + totalDebts: integerToAmount(groupedDebts), + totalTotals: integerToAmount(groupedDebts + groupedAssets), + }); + + return arr; + }, []); + + const stackedCategories = group.categories.map(item => { + const calc = recalculate({ + item, + months, + assets, + debts, + groupByLabel: 'category', + }); + return { ...calc }; + }); + + return { + id: group.id, + name: group.name, + totalAssets: integerToAmount(totalAssets), + totalDebts: integerToAmount(totalDebts), + totalTotals: integerToAmount(totalAssets + totalDebts), + monthData, + categories: stackedCategories, + }; + }, + [start, end], + ); + + setData(groupedData); + }; +} + +export default createGroupedSpreadsheet; diff --git a/packages/desktop-client/src/components/reports/spreadsheets/makeQuery.ts b/packages/desktop-client/src/components/reports/spreadsheets/makeQuery.ts new file mode 100644 index 00000000000..19d56b811d3 --- /dev/null +++ b/packages/desktop-client/src/components/reports/spreadsheets/makeQuery.ts @@ -0,0 +1,86 @@ +import q from 'loot-core/src/client/query-helpers'; +import { type CategoryEntity } from 'loot-core/src/types/models'; + +function makeQuery( + name: string, + start: string, + end: string, + hidden: boolean, + selectedCategories: CategoryEntity[], + categoryFilter: CategoryEntity[], + conditionsOpKey: string, + filters: unknown[], +) { + const query = q('transactions') + .filter( + //Show Offbudget and hidden categories + !hidden && { + $and: [ + { + 'account.offbudget': false, + $or: [ + { + 'category.hidden': false, + category: null, + }, + ], + }, + ], + $or: [ + { + 'payee.transfer_acct.offbudget': true, + 'payee.transfer_acct': null, + }, + ], + }, + ) + //Apply Category_Selector + .filter( + selectedCategories && { + $or: [ + { + category: null, + $or: categoryFilter.map(category => ({ + category: category.id, + })), + }, + ], + }, + ) + //Apply filters and split by "Group By" + .filter({ + [conditionsOpKey]: [...filters], + }) + //Apply month range filters + .filter({ + $and: [ + { date: { $transform: '$month', $gte: start } }, + { date: { $transform: '$month', $lte: end } }, + ], + }) + //Show assets or debts + .filter( + name === 'assets' ? { amount: { $gt: 0 } } : { amount: { $lt: 0 } }, + ); + + return query + .groupBy([ + { $month: '$date' }, + { $id: '$account' }, + { $id: '$payee' }, + { $id: '$category' }, + { $id: '$payee.transfer_acct.id' }, + ]) + .select([ + { date: { $month: '$date' } }, + { category: { $id: '$category.id' } }, + { categoryGroup: { $id: '$category.group.id' } }, + { account: { $id: '$account.id' } }, + { accountOffBudget: { $id: '$account.offbudget' } }, + { payee: { $id: '$payee.id' } }, + { transferAccount: { $id: '$payee.transfer_acct.id' } }, + { amount: { $sum: '$amount' } }, + ]); +} + +export default makeQuery; diff --git a/packages/desktop-client/src/components/reports/spreadsheets/recalculate.ts b/packages/desktop-client/src/components/reports/spreadsheets/recalculate.ts new file mode 100644 index 00000000000..e6b516dfaef --- /dev/null +++ b/packages/desktop-client/src/components/reports/spreadsheets/recalculate.ts @@ -0,0 +1,69 @@ +import * as d from 'date-fns'; + +import { amountToInteger, integerToAmount } from 'loot-core/src/shared/util'; + +import { type QueryDataEntity } from '../ReportOptions'; + +import filterHiddenItems from './filterHiddenItems'; + +type recalculateProps = { + item; + months: Array; + assets: QueryDataEntity[]; + debts: QueryDataEntity[]; + groupByLabel: string; +}; + +function recalculate({ + item, + months, + assets, + debts, + groupByLabel, +}: recalculateProps) { + let totalAssets = 0; + let totalDebts = 0; + const monthData = months.reduce((arr, month) => { + const last = arr.length === 0 ? null : arr[arr.length - 1]; + + const monthAssets = filterHiddenItems(item, assets) + .filter(asset => asset.date === month && asset[groupByLabel] === item.id) + .reduce((a, v) => (a = a + v.amount), 0); + totalAssets += monthAssets; + + const monthDebts = filterHiddenItems(item, debts) + .filter(debt => debt.date === month && debt[groupByLabel] === item.id) + .reduce((a, v) => (a = a + v.amount), 0); + totalDebts += monthDebts; + + const dateParse = d.parseISO(`${month}-01`); + + const change = last + ? monthAssets + monthDebts - amountToInteger(last.totalTotals) + : 0; + + arr.push({ + dateParse, + totalAssets: integerToAmount(monthAssets), + totalDebts: integerToAmount(monthDebts), + totalTotals: integerToAmount(monthAssets + monthDebts), + change, + // eslint-disable-next-line rulesdir/typography + date: d.format(dateParse, "MMM ''yy"), + dateLookup: month, + }); + + return arr; + }, []); + + return { + id: item.id, + name: item.name, + totalAssets: integerToAmount(totalAssets), + totalDebts: integerToAmount(totalDebts), + totalTotals: integerToAmount(totalAssets + totalDebts), + monthData, + }; +} + +export default recalculate; diff --git a/packages/desktop-client/src/components/reports/util.ts b/packages/desktop-client/src/components/reports/util.ts index 4d28f2ee4d4..35ff2166e5b 100644 --- a/packages/desktop-client/src/components/reports/util.ts +++ b/packages/desktop-client/src/components/reports/util.ts @@ -29,17 +29,6 @@ export function index< return result; } -export function indexStack< - T extends Record, - K extends keyof T, ->(data: T[], fieldName: K, field: K) { - const result: Record = {}; - data.forEach(item => { - result[item[fieldName]] = item[field]; - }); - return result; -} - export function indexCashFlow< T extends { date: string; isTransfer: boolean; amount: number }, >(data: T[], date: string, isTransfer: string) { diff --git a/packages/loot-core/src/server/aql/compiler.ts b/packages/loot-core/src/server/aql/compiler.ts index 2610939a515..70bdbfe8993 100644 --- a/packages/loot-core/src/server/aql/compiler.ts +++ b/packages/loot-core/src/server/aql/compiler.ts @@ -587,6 +587,12 @@ const compileFunction = saveStack('function', (state, func) => { ); } + // id functions + case '$id': { + validateArgLength(args, 1); + return typed(val(state, args[0]), args[0].type); + } + // date functions case '$month': { validateArgLength(args, 1); diff --git a/packages/loot-core/src/types/models/rule.d.ts b/packages/loot-core/src/types/models/rule.d.ts index 12549e05495..6b9eb147f55 100644 --- a/packages/loot-core/src/types/models/rule.d.ts +++ b/packages/loot-core/src/types/models/rule.d.ts @@ -9,7 +9,7 @@ export interface RuleEntity { tombstone?: boolean; } -interface RuleConditionEntity { +export interface RuleConditionEntity { field: unknown; op: | 'is' @@ -28,6 +28,7 @@ interface RuleConditionEntity { options?: unknown; conditionsOp?: unknown; type?: string; + customName?: string; } export type RuleActionEntity = diff --git a/upcoming-release-notes/1988.md b/upcoming-release-notes/1988.md new file mode 100644 index 00000000000..8943ac21619 --- /dev/null +++ b/upcoming-release-notes/1988.md @@ -0,0 +1,6 @@ +--- +category: Enhancements +authors: [carkom] +--- + +Data loading performance improvements for custom reports \ No newline at end of file