Optimize table editor count (#27612)

* Optimize table editor count

* Add additional logic on when to enforce exact count

* Update apps/studio/data/table-rows/table-rows-count-query.ts

Co-authored-by: Alaister Young <alaister@users.noreply.github.com>

* Reset enforceExactCount whenever a filter is applied

* Update showing exact row count warning logic

---------

Co-authored-by: Alaister Young <alaister@users.noreply.github.com>
This commit is contained in:
Joshen Lim
2024-07-04 15:27:01 +08:00
committed by GitHub
parent 70da0f1d1d
commit e0ae4942f4
11 changed files with 260 additions and 150 deletions

View File

@@ -271,7 +271,7 @@ const SupabaseGridLayout = (props: SupabaseGridProps) => {
onImportData={onImportData}
onEditForeignKeyColumnValue={onEditForeignKeyColumnValue}
/>
<Footer isLoading={isLoading} isRefetching={isRefetching} />
<Footer isRefetching={isRefetching} />
<Shortcuts gridRef={gridRef} />
</>
)}

View File

@@ -115,6 +115,7 @@ export function parseSupaTable(
})
return {
id: table.id,
name: table.name,
comment: table.comment,
schema: table.schema,

View File

@@ -7,11 +7,10 @@ import RefreshButton from '../header/RefreshButton'
import { Pagination } from './pagination'
export interface FooterProps {
isLoading?: boolean
isRefetching?: boolean
}
const Footer = ({ isLoading, isRefetching }: FooterProps) => {
const Footer = ({ isRefetching }: FooterProps) => {
const { id: _id } = useParams()
const id = _id ? Number(_id) : undefined
const { data: selectedTable } = useTable(id)
@@ -30,7 +29,7 @@ const Footer = ({ isLoading, isRefetching }: FooterProps) => {
<GridFooter>
{selectedView === 'data' && <Pagination />}
<div className="ml-auto flex items-center gap-2">
<div className="ml-auto flex items-center gap-x-2">
{selectedTable && selectedView === 'data' && (
<RefreshButton table={selectedTable} isRefetching={isRefetching} />
)}

View File

@@ -1,14 +1,23 @@
import { ArrowLeft, ArrowRight } from 'lucide-react'
import { ArrowLeft, ArrowRight, HelpCircle } from 'lucide-react'
import { useEffect, useState } from 'react'
import { PostgresTable } from '@supabase/postgres-meta'
import { formatFilterURLParams } from 'components/grid/SupabaseGrid.utils'
import { useProjectContext } from 'components/layouts/ProjectLayout/ProjectContext'
import { useTableRowsCountQuery } from 'data/table-rows/table-rows-count-query'
import { THRESHOLD_COUNT, useTableRowsCountQuery } from 'data/table-rows/table-rows-count-query'
import useTable from 'hooks/misc/useTable'
import { useUrlState } from 'hooks/ui/useUrlState'
import { useRoleImpersonationStateSnapshot } from 'state/role-impersonation-state'
import { useTableEditorStateSnapshot } from 'state/table-editor'
import { Button, InputNumber } from 'ui'
import {
Button,
InputNumber,
TooltipContent_Shadcn_,
TooltipTrigger_Shadcn_,
Tooltip_Shadcn_,
} from 'ui'
import ConfirmationModal from 'ui-patterns/Dialogs/ConfirmationModal'
import { useParams } from 'common'
import { useDispatch, useTrackedState } from '../../../store/Store'
import { DropdownControl } from '../../common/DropdownControl'
@@ -19,28 +28,36 @@ const rowsPerPageOptions = [
]
const Pagination = () => {
const { id: _id } = useParams()
const id = _id ? Number(_id) : undefined
const state = useTrackedState()
const dispatch = useDispatch()
const { project } = useProjectContext()
const snap = useTableEditorStateSnapshot()
const page = snap.page
const [{ filter }] = useUrlState({
arrayKeys: ['filter'],
})
const { data: selectedTable } = useTable(id)
// [Joshen] Only applicable to table entities
const rowsCountEstimate = (selectedTable as PostgresTable)?.live_rows_estimate ?? null
const [{ filter }] = useUrlState({ arrayKeys: ['filter'] })
const filters = formatFilterURLParams(filter as string[])
const page = snap.page
const table = state.table ?? undefined
const roleImpersonationState = useRoleImpersonationStateSnapshot()
const [isConfirmNextModalOpen, setIsConfirmNextModalOpen] = useState(false)
const [isConfirmPreviousModalOpen, setIsConfirmPreviousModalOpen] = useState(false)
const [isConfirmFetchExactCountModalOpen, setIsConfirmFetchExactCountModalOpen] = useState(false)
const { project } = useProjectContext()
const { data, isLoading, isSuccess, isError } = useTableRowsCountQuery(
const { data, isLoading, isSuccess, isError, isFetching } = useTableRowsCountQuery(
{
queryKey: [table?.schema, table?.name, 'count'],
queryKey: [table?.schema, table?.name, 'count-estimate'],
projectRef: project?.ref,
connectionString: project?.connectionString,
table,
filters,
enforceExactCount: snap.enforceExactCount,
impersonatedRole: roleImpersonationState.role,
},
{
@@ -57,21 +74,6 @@ const Pagination = () => {
const maxPages = Math.ceil((data?.count ?? 0) / snap.rowsPerPage)
const totalPages = (data?.count ?? 0) > 0 ? maxPages : 1
useEffect(() => {
if (page && page > totalPages) {
snap.setPage(totalPages)
}
}, [page, totalPages])
// [Joshen] Oddly without this, state.selectedRows will be stale
useEffect(() => {}, [state.selectedRows])
// [Joshen] Note: I've made pagination buttons disabled while rows are being fetched for now
// at least until we can send an abort signal to cancel requests if users are mashing the
// pagination buttons to find the data they want
const [isConfirmPreviousModalOpen, setIsConfirmPreviousModalOpen] = useState(false)
const onPreviousPage = () => {
if (page > 1) {
if (state.selectedRows.size >= 1) {
@@ -90,8 +92,6 @@ const Pagination = () => {
})
}
const [isConfirmNextModalOpen, setIsConfirmNextModalOpen] = useState(false)
const onNextPage = () => {
if (page < maxPages) {
if (state.selectedRows.size >= 1) {
@@ -110,8 +110,6 @@ const Pagination = () => {
})
}
// TODO: look at aborting useTableRowsQuery if the user presses the button quickly
const goToPreviousPage = () => {
const previousPage = page - 1
snap.setPage(previousPage)
@@ -122,96 +120,112 @@ const Pagination = () => {
snap.setPage(nextPage)
}
function onPageChange(event: React.ChangeEvent<HTMLInputElement>) {
const onPageChange = (event: React.ChangeEvent<HTMLInputElement>) => {
const value = event.target.value
const pageNum = Number(value) > maxPages ? maxPages : Number(value)
snap.setPage(pageNum || 1)
}
function onRowsPerPageChange(value: string | number) {
const onRowsPerPageChange = (value: string | number) => {
const rowsPerPage = Number(value)
snap.setRowsPerPage(isNaN(rowsPerPage) ? 100 : rowsPerPage)
}
useEffect(() => {
if (page && page > totalPages) {
snap.setPage(totalPages)
}
}, [page, totalPages])
useEffect(() => {
if (id !== undefined) {
snap.setEnforceExactCount(rowsCountEstimate !== null && rowsCountEstimate <= THRESHOLD_COUNT)
}
}, [id])
return (
<div className="sb-grid-pagination">
<div className="flex items-center gap-x-4">
{isLoading && <p className="text-sm text-foreground-light">Loading records count...</p>}
{isSuccess && (
<>
<Button
icon={<ArrowLeft />}
type="outline"
className="px-1.5"
disabled={page <= 1 || isLoading}
onClick={onPreviousPage}
/>
<p className="text-sm text-foreground-light">Page</p>
<div className="sb-grid-pagination-input-container">
<InputNumber
// [Fran] we'll have to upgrade the UI component types to accept the null value when users delete the input content
// @ts-ignore
value={page}
onChange={onPageChange}
size="tiny"
style={{
width: '3rem',
}}
max={maxPages}
min={1}
<div className="flex items-center gap-x-2">
<Button
icon={<ArrowLeft />}
type="outline"
className="px-1.5"
disabled={page <= 1 || isLoading}
onClick={onPreviousPage}
/>
<p className="text-xs text-foreground-light">Page</p>
<div className="w-12">
<InputNumber
size="tiny"
value={page}
onChange={onPageChange}
style={{ width: '3rem' }}
min={1}
max={maxPages}
/>
</div>
<p className="text-xs text-foreground-light">of {totalPages.toLocaleString()}</p>
<Button
icon={<ArrowRight />}
type="outline"
className="px-1.5"
disabled={page >= maxPages || isLoading}
onClick={onNextPage}
/>
<DropdownControl
options={rowsPerPageOptions}
onSelect={onRowsPerPageChange}
side="top"
align="start"
>
<Button asChild type="outline" style={{ padding: '3px 10px' }}>
<span>{`${snap.rowsPerPage} rows`}</span>
</Button>
</DropdownControl>
</div>
<p className="text-sm text-foreground-light">of {totalPages}</p>
<Button
icon={<ArrowRight />}
type="outline"
className="px-1.5"
disabled={page >= maxPages || isLoading}
onClick={onNextPage}
/>
<DropdownControl
options={rowsPerPageOptions}
onSelect={onRowsPerPageChange}
side="top"
align="start"
>
<Button asChild type="outline" style={{ padding: '3px 10px' }}>
<span>{`${snap.rowsPerPage} rows`}</span>
</Button>
</DropdownControl>
<p className="text-sm text-foreground-light">{`${data.count.toLocaleString()} ${
data.count === 0 || data.count > 1 ? `records` : 'record'
}`}</p>
<ConfirmationModal
visible={isConfirmPreviousModalOpen}
title="Confirm moving to previous page"
confirmLabel="Confirm"
onCancel={() => setIsConfirmPreviousModalOpen(false)}
onConfirm={() => {
onConfirmPreviousPage()
}}
>
<p className="py-4 text-sm text-foreground-light">
The currently selected lines will be deselected, do you want to proceed?
<div className="flex items-center gap-x-2">
<p className="text-xs text-foreground-light">
{`${data.count.toLocaleString()} ${
data.count === 0 || data.count > 1 ? `records` : 'record'
}`}{' '}
{data.is_estimate ? '(estimated)' : ''}
</p>
</ConfirmationModal>
<ConfirmationModal
visible={isConfirmNextModalOpen}
title="Confirm moving to next page"
confirmLabel="Confirm"
onCancel={() => setIsConfirmNextModalOpen(false)}
onConfirm={() => {
onConfirmNextPage()
}}
>
<p className="py-4 text-sm text-foreground-light">
The currently selected lines will be deselected, do you want to proceed?
</p>
</ConfirmationModal>
{data.is_estimate && (
<Tooltip_Shadcn_>
<TooltipTrigger_Shadcn_ asChild>
<Button
size="tiny"
type="text"
className="px-1.5"
loading={isFetching}
icon={<HelpCircle />}
onClick={() => {
// Show warning if either NOT a table entity, or table rows estimate is beyond threshold
if (rowsCountEstimate === null || data.count > THRESHOLD_COUNT) {
setIsConfirmFetchExactCountModalOpen(true)
} else snap.setEnforceExactCount(true)
}}
/>
</TooltipTrigger_Shadcn_>
<TooltipContent_Shadcn_ side="top" className="w-72">
This is an estimated value as your table has more than{' '}
{THRESHOLD_COUNT.toLocaleString()} rows. <br />
<span className="text-brand">
Click to retrieve the exact count of the table.
</span>
</TooltipContent_Shadcn_>
</Tooltip_Shadcn_>
)}
</div>
</>
)}
@@ -220,6 +234,54 @@ const Pagination = () => {
Error fetching records count. Please refresh the page.
</p>
)}
<ConfirmationModal
visible={isConfirmPreviousModalOpen}
title="Confirm moving to previous page"
confirmLabel="Confirm"
onCancel={() => setIsConfirmPreviousModalOpen(false)}
onConfirm={() => {
onConfirmPreviousPage()
}}
>
<p className="text-sm text-foreground-light">
The currently selected lines will be deselected, do you want to proceed?
</p>
</ConfirmationModal>
<ConfirmationModal
visible={isConfirmNextModalOpen}
title="Confirm moving to next page"
confirmLabel="Confirm"
onCancel={() => setIsConfirmNextModalOpen(false)}
onConfirm={() => {
onConfirmNextPage()
}}
>
<p className="text-sm text-foreground-light">
The currently selected lines will be deselected, do you want to proceed?
</p>
</ConfirmationModal>
<ConfirmationModal
variant="warning"
visible={isConfirmFetchExactCountModalOpen}
title="Confirm to fetch exact count for table"
confirmLabel="Retrieve exact count"
onCancel={() => setIsConfirmFetchExactCountModalOpen(false)}
onConfirm={() => {
snap.setEnforceExactCount(true)
setIsConfirmFetchExactCountModalOpen(false)
}}
>
<p className="text-sm text-foreground-light">
{rowsCountEstimate === null
? `If your table has a row count of greater than ${THRESHOLD_COUNT.toLocaleString()} rows,
retrieving the exact count of the table may cause performance issues on your database.`
: `Your table has a row count of greater than ${THRESHOLD_COUNT.toLocaleString()} rows, and
retrieving the exact count of the table may cause performance issues on your database.`}
</p>
</ConfirmationModal>
</div>
)
}

View File

@@ -249,7 +249,7 @@ const RowHeader = ({ table, sorts, filters }: RowHeaderProps) => {
const { data: countData } = useTableRowsCountQuery(
{
queryKey: [table?.schema, table?.name, 'count'],
queryKey: [table?.schema, table?.name, 'count-estimate'],
projectRef: project?.ref,
connectionString: project?.connectionString,
table,
@@ -350,14 +350,14 @@ const RowHeader = ({ table, sorts, filters }: RowHeaderProps) => {
<Button type="default" className="px-1" icon={<X />} onClick={deselectRows} />
<span className="text-xs text-foreground">
{allRowsSelected
? `${totalRows} rows selected`
? `All rows in table selected`
: selectedRows.size > 1
? `${selectedRows.size} rows selected`
: `${selectedRows.size} row selected`}
</span>
{!allRowsSelected && totalRows > allRows.length && (
<Button type="link" onClick={() => onSelectAllRows()}>
Select all {totalRows} rows
Select all rows in table
</Button>
)}
</div>
@@ -384,7 +384,7 @@ const RowHeader = ({ table, sorts, filters }: RowHeaderProps) => {
disabled={allRowsSelected && isImpersonatingRole}
>
{allRowsSelected
? `Delete ${totalRows} rows`
? `Delete all rows in table`
: selectedRows.size > 1
? `Delete ${selectedRows.size} rows`
: `Delete ${selectedRows.size} row`}

View File

@@ -15,6 +15,7 @@ import {
} from 'ui'
import { FilterOperatorOptions } from './Filter.constants'
import FilterRow from './FilterRow'
import { useTableEditorStateSnapshot } from 'state/table-editor'
export interface FilterPopoverProps {
table: SupaTable
@@ -24,6 +25,7 @@ export interface FilterPopoverProps {
const FilterPopover = ({ table, filters, setParams }: FilterPopoverProps) => {
const [open, setOpen] = useState(false)
const snap = useTableEditorStateSnapshot()
const btnText =
(filters || []).length > 0
@@ -31,6 +33,7 @@ const FilterPopover = ({ table, filters, setParams }: FilterPopoverProps) => {
: 'Filter'
const onApplyFilters = (appliedFilters: Filter[]) => {
snap.setEnforceExactCount(false)
setParams((prevParams) => {
return {
...prevParams,

View File

@@ -19,6 +19,7 @@ export interface SupaColumn {
}
export interface SupaTable {
readonly id: number
readonly columns: SupaColumn[]
readonly name: string
readonly schema?: string | null

View File

@@ -4,7 +4,7 @@ import { cn } from 'ui'
export const GridFooter = ({ children, className }: PropsWithChildren<{ className?: string }>) => {
return (
<div
className={cn('flex min-h-9 overflow-hidden items-center px-5 w-full border-t', className)}
className={cn('flex min-h-9 overflow-hidden items-center px-2 w-full border-t', className)}
>
{children}
</div>

View File

@@ -9,31 +9,87 @@ import { formatFilterValue } from './utils'
type GetTableRowsCountArgs = {
table?: SupaTable
filters?: Filter[]
enforceExactCount?: boolean
impersonatedRole?: ImpersonationRole
}
export const getTableRowsCountSqlQuery = ({ table, filters = [] }: GetTableRowsCountArgs) => {
const query = new Query()
export const THRESHOLD_COUNT = 50000
const COUNT_ESTIMATE_SQL = `
CREATE OR REPLACE FUNCTION pg_temp.count_estimate(
query text
) RETURNS integer LANGUAGE plpgsql AS $$
DECLARE
plan jsonb;
BEGIN
EXECUTE 'EXPLAIN (FORMAT JSON)' || query INTO plan;
RETURN plan->0->'Plan'->'Plan Rows';
END;
$$;
`.trim()
if (!table) {
return ``
export const getTableRowsCountSqlQuery = ({
table,
filters = [],
enforceExactCount = false,
}: GetTableRowsCountArgs) => {
if (!table) return ``
if (enforceExactCount) {
const query = new Query()
let queryChains = query.from(table.name, table.schema ?? undefined).count()
filters
.filter((x) => x.value && x.value !== '')
.forEach((x) => {
const value = formatFilterValue(table, x)
queryChains = queryChains.filter(x.column, x.operator, value)
})
return `select (${queryChains.toSql().slice(0, -1)}), false as is_estimate;`
} else {
const selectQuery = new Query()
let selectQueryChains = selectQuery.from(table.name, table.schema ?? undefined).select('*')
filters
.filter((x) => x.value && x.value != '')
.forEach((x) => {
const value = formatFilterValue(table, x)
selectQueryChains = selectQueryChains.filter(x.column, x.operator, value)
})
const selectBaseSql = selectQueryChains.toSql()
const countQuery = new Query()
let countQueryChains = countQuery.from(table.name, table.schema ?? undefined).count()
filters
.filter((x) => x.value && x.value != '')
.forEach((x) => {
const value = formatFilterValue(table, x)
countQueryChains = countQueryChains.filter(x.column, x.operator, value)
})
const countBaseSql = countQueryChains.toSql().slice(0, -1)
const sql = `
${COUNT_ESTIMATE_SQL}
with approximation as (
select reltuples as estimate
from pg_class
where oid = ${table.id}
)
select
case
when estimate = -1 then (select pg_temp.count_estimate('${selectBaseSql.replaceAll("'", "''")}'))
when estimate > ${THRESHOLD_COUNT} then ${filters.length > 0 ? `pg_temp.count_estimate('${selectBaseSql.replaceAll("'", "''")}')` : 'estimate'}
else (${countBaseSql})
end as count,
estimate = -1 or estimate > ${THRESHOLD_COUNT} as is_estimate
from approximation;
`.trim()
return sql
}
let queryChains = query.from(table.name, table.schema ?? undefined).count()
filters
.filter((x) => x.value && x.value != '')
.forEach((x) => {
const value = formatFilterValue(table, x)
queryChains = queryChains.filter(x.column, x.operator, value)
})
const sql = queryChains.toSql()
return sql
}
export type TableRowsCount = {
count: number
is_estimate?: boolean
}
export type TableRowsCountVariables = GetTableRowsCountArgs & {
@@ -51,6 +107,7 @@ export const useTableRowsCountQuery = <TData extends TableRowsCountData = TableR
connectionString,
queryKey,
table,
enforceExactCount,
impersonatedRole,
...args
}: TableRowsCountVariables,
@@ -62,14 +119,15 @@ export const useTableRowsCountQuery = <TData extends TableRowsCountData = TableR
{
projectRef,
connectionString,
sql: wrapWithRoleImpersonation(getTableRowsCountSqlQuery({ table, ...args }), {
projectRef: projectRef ?? 'ref',
role: impersonatedRole,
}),
sql: wrapWithRoleImpersonation(
getTableRowsCountSqlQuery({ table, enforceExactCount, ...args }),
{ projectRef: projectRef ?? 'ref', role: impersonatedRole }
),
queryKey: [
...(queryKey ?? []),
{
table: { name: table?.name, schema: table?.schema },
enforceExactCount,
impersonatedRole,
...args,
},
@@ -80,6 +138,7 @@ export const useTableRowsCountQuery = <TData extends TableRowsCountData = TableR
select(data) {
return {
count: data.result[0].count,
is_estimate: data.result[0].is_estimate ?? false,
} as TData
},
enabled: typeof projectRef !== 'undefined' && typeof table !== 'undefined',

View File

@@ -59,6 +59,11 @@ export const createTableEditorState = () => {
state.selectedSchemaName = schemaName
},
enforceExactCount: false,
setEnforceExactCount: (value: boolean) => {
state.enforceExactCount = value
},
page: 1,
setPage: (page: number) => {
state.page = page

View File

@@ -399,26 +399,6 @@
@apply h-full w-full px-2;
}
/*
Pagination
*/
.sb-grid-pagination {
@apply flex items-center space-x-2;
}
.sb-grid-pagination-input-container {
@apply w-12;
}
.sb-grid-pagination-input {
@apply block w-12;
}
.sb-grid-pagination-input .sbui-inputnumber {
padding-top: 0.25rem;
padding-bottom: 0.25rem;
}
/*
Footer
*/