diff --git a/.github/workflows/build-cloudberry-rocky8.yml b/.github/workflows/build-cloudberry-rocky8.yml index 4986eae11b2..faa5df153e1 100644 --- a/.github/workflows/build-cloudberry-rocky8.yml +++ b/.github/workflows/build-cloudberry-rocky8.yml @@ -324,6 +324,10 @@ jobs: "make_configs":["gpcontrib/gp_stats_collector:installcheck"], "extension":"gp_stats_collector" }, + {"test":"gpcontrib-gp-relsizes-stats", + "make_configs":["gpcontrib/gp_relsizes_stats:installcheck"], + "extension":"gp_relsizes_stats" + }, {"test":"ic-fixme", "make_configs":["src/test/regress:installcheck-fixme"], "enable_core_check":false @@ -1445,6 +1449,17 @@ jobs: exit 1 fi ;; + gp_relsizes_stats) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_relsizes_stats; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_relsizes_stats extension" + exit 1 + fi + ;; *) echo "Unknown extension: ${{ matrix.extension }}" exit 1 diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index c00dcde0486..484041953ff 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -275,6 +275,10 @@ jobs: "make_configs":["gpcontrib/gp_stats_collector:installcheck"], "extension":"gp_stats_collector" }, + {"test":"gpcontrib-gp-relsizes-stats", + "make_configs":["gpcontrib/gp_relsizes_stats:installcheck"], + "extension":"gp_relsizes_stats" + }, {"test":"ic-expandshrink", "make_configs":["src/test/isolation2:installcheck-expandshrink"] }, @@ -1457,6 +1461,17 @@ jobs: exit 1 fi ;; + gp_relsizes_stats) + if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ + source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ + echo 'CREATE EXTENSION IF NOT EXISTS gp_relsizes_stats; \ + TABLE pg_extension;' | \ + psql postgres" + then + echo "Error creating gp_relsizes_stats extension" + exit 1 + fi + ;; *) echo "Unknown extension: ${{ matrix.extension }}" exit 1 diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index 2969194cfac..64aec1aac73 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -20,6 +20,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_inject_fault \ gp_exttable_fdw \ gp_legacy_string_agg \ + gp_relsizes_stats \ gp_replica_check \ gp_toolkit \ pg_hint_plan @@ -28,6 +29,7 @@ else gp_distribution_policy \ gp_internal_tools \ gp_legacy_string_agg \ + gp_relsizes_stats \ gp_exttable_fdw \ gp_toolkit \ pg_hint_plan diff --git a/gpcontrib/gp_relsizes_stats/.clang-format b/gpcontrib/gp_relsizes_stats/.clang-format new file mode 100644 index 00000000000..efcf9ff4160 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/.clang-format @@ -0,0 +1,192 @@ +--- +Language: Cpp +# BasedOnStyle: LLVM +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveMacros: None +AlignConsecutiveAssignments: None +AlignConsecutiveBitFields: None +AlignConsecutiveDeclarations: None +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: true +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortEnumsOnASingleLine: true +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: All +AllowShortLambdasOnASingleLine: All +AllowShortIfStatementsOnASingleLine: Never +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: MultiLine +AttributeMacros: + - __capability +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + AfterExternBlock: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: true + SplitEmptyNamespace: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: true +BreakBeforeBraces: Attach +BreakBeforeInheritanceComma: false +BreakInheritanceList: BeforeColon +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakConstructorInitializers: BeforeColon +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +QualifierAlignment: Leave +CompactNamespaces: false +ConstructorInitializerIndentWidth: 4 +ContinuationIndentWidth: 4 +Cpp11BracedListStyle: true +DeriveLineEnding: true +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +PackConstructorInitializers: BinPack +BasedOnStyle: '' +ConstructorInitializerAllOnOneLineOrOnePerLine: false +AllowAllConstructorInitializersOnNextLine: true +FixNamespaceComments: true +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseLabels: false +IndentCaseBlocks: false +IndentGotoLabels: true +IndentPPDirectives: None +IndentExternBlock: AfterExternBlock +IndentRequires: false +IndentWidth: 4 +IndentWrappedFunctionNames: false +InsertTrailingCommas: None +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +LambdaBodyIndentation: Signature +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: true +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 60 +PenaltyIndentedWhitespace: 0 +PointerAlignment: Right +PPIndentWidth: -1 +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: true +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: true +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + BeforeNonEmptyParentheses: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeRangeBasedForLoopColon: true +SpaceInEmptyBlock: false +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInConditionalStatement: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpaceBeforeSquareBrackets: false +BitFieldColonSpacing: Both +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseCRLF: false +UseTab: Never +WhitespaceSensitiveMacros: + - STRINGIZE + - PP_STRINGIZE + - BOOST_PP_STRINGIZE + - NS_SWIFT_NAME + - CF_SWIFT_NAME +... + diff --git a/gpcontrib/gp_relsizes_stats/.gitignore b/gpcontrib/gp_relsizes_stats/.gitignore new file mode 100644 index 00000000000..ebe888c0253 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/.gitignore @@ -0,0 +1,6 @@ +*.o +*.so +src/protos/ +results +.vscode +compile_commands.json diff --git a/gpcontrib/gp_relsizes_stats/Makefile b/gpcontrib/gp_relsizes_stats/Makefile new file mode 100644 index 00000000000..431dfb59d5b --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/Makefile @@ -0,0 +1,20 @@ +MODULE_big = gp_relsizes_stats +OBJS = ./src/gp_relsizes_stats.o +EXTENSION = gp_relsizes_stats +EXTVERSION = 1.3 +DATA = $(wildcard sql/*--*.sql) +REGRESS = grants gp_relsizes_stats +REGRESS_OPTS = --inputdir=test/ +PGFILEDESC = "gp_relsizes_stats - an extension to track table on-disc sizes in Cloudberry" +PG_CXXFLAGS += $(COMMON_CPP_FLAGS) +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = gpcontrib/gp_relsizes_stats +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + diff --git a/gpcontrib/gp_relsizes_stats/README.md b/gpcontrib/gp_relsizes_stats/README.md new file mode 100644 index 00000000000..fe8cde533f5 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/README.md @@ -0,0 +1,70 @@ + + +# gp_relsizes_stats: Table sizes monitoring tool for Cloudberry + +### Features +gp_relsizes_stats is an extension for the Cloudberry database that calculates and stores statistics on the size of files and tables, occupied space on the disks of the master and segment hosts. + +#### Features include +- BackgroundWorker support for collecting statistics automatically +- the ability to fine-tune the timeout values between actions, for example, between launches for different databases, or during file processing to distribute the load over time + +### Supported versions and platforms +At the moment, the program is being tested only for Cloudberry and Linux. + +### Installation +This extension is part of the Cloudberry monorepo under `gpcontrib/gp_relsizes_stats`. + +Build and install from the Cloudberry monorepo root: +```bash +make -C gpcontrib/gp_relsizes_stats +sudo make -C gpcontrib/gp_relsizes_stats install +``` + +### Configuration +gp_relsizes_stats configuration parameters: +| **Parameter** | **Type** | **Default** | **Description** | +| ---------------- | --------------- | ------------ | ------------ | +| `gp_relsizes_stats.enabled` | bool | false | Using `gp_relsizes_stats.enabled` you can enable/disable background stats collection for database where extension installed (actually enable/disable background worker which collecting stats).| +| `gp_relsizes_stats.restart_naptime` | int | 21600000 | Using `gp_relsizes_stats.restart_naptime` you can set naptime between each startup of collecting process. Value set time in milliseconds. Default is equal to 6 hours.| +| `gp_relsizes_stats.database_naptime` | int | 0 | Using `gp_relsizes_stats.database_naptime` you can set naptime between collecting stats for each databases. Value set time in milliseconds. Default is equal to 0 milliseconds.| +| `gp_relsizes_stats.file_naptime` | int | 1 | Using `gp_relsizes_stats.file_naptime` you can set naptime between each file stats calculating. Value set time in milliseconds. Default is equal to 1 millisecond.| + +### Usage +You can use a background worker to collect statistics, but if you sometimes need to change the format of the settings or if you don't want to collect statistics on a regular basis, you can do so. In these situations, you could set +``` +gp_relsizes_stats.enabled = off +``` + +And use the function +``` +relsizes_stats_schema.relsizes_collect_stats_once() +``` +which can be called manually using 'select'. +It will launch a single statistics collection procedure. + + +### About collected data and tables +| Name of table | Row description | Description | +| ------------- | --------------- | ----------- | +| relsizes_stats_schema.segment_file_sizes | (segment, relfilenode, filepath, size, mtime) | Current size and last modify time of each file of specific relation on specific segment | +| relsizes_stats_schema.namespace_sizes | (nspname, nspsize) | Current size of namespace | +| relsizes_stats_schema.table_sizes | (nspname, relname, relsize) | Current size of relation in specific namespace | +| relsizes_stats_schema.table_sizes_history | (insert_date, nspname, relname, size, mtime) | Size and last modify time of relation in specific namespace with date when information was collected | diff --git a/gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control b/gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control new file mode 100644 index 00000000000..84e46dd7b0c --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/gp_relsizes_stats.control @@ -0,0 +1,5 @@ +# gp_relsizes_stats extension +comment = 'gp_relsizes_stats - an extension to track table on-disc sizes in cloudberry' +default_version = '1.3' +module_pathname = '$libdir/gp_relsizes_stats' +trusted = true diff --git a/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.0--1.1.sql b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.0--1.1.sql new file mode 100644 index 00000000000..15dd1e7e5a8 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.0--1.1.sql @@ -0,0 +1,12 @@ +/* gp_relsizes_stats--1.0--1.1.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_relsizes_stats" to load this file. \quit + + +DROP FUNCTION relsizes_stats_schema.get_stats_for_database(dboid INTEGER); + +CREATE FUNCTION relsizes_stats_schema.get_stats_for_database(dboid OID, fast BOOL) +RETURNS TABLE (segment INTEGER, relfilenode OID, filepath TEXT, size BIGINT, mtime BIGINT) +AS 'MODULE_PATHNAME', 'get_stats_for_database' +LANGUAGE C STRICT EXECUTE ON ALL SEGMENTS; diff --git a/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.1--1.2.sql b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.1--1.2.sql new file mode 100644 index 00000000000..300a17ffc92 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.1--1.2.sql @@ -0,0 +1,56 @@ +/* gp_relsizes_stats--1.1--1.2.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_relsizes_stats" to load this file. \quit + +CREATE OR REPLACE VIEW relsizes_stats_schema.table_files AS + WITH part_oids AS ( + SELECT n.nspname, c1.relname, c1.oid, true own_oid + FROM pg_class c1 + JOIN pg_namespace n ON c1.relnamespace = n.oid + WHERE c1.reltablespace != (SELECT oid FROM pg_tablespace WHERE spcname = 'pg_global') + UNION ALL + SELECT n.nspname, c1.relname, c2.oid, false own_oid + FROM pg_class c1 + JOIN pg_namespace n ON c1.relnamespace = n.oid + JOIN pg_partition pp ON c1.oid = pp.parrelid + JOIN pg_partition_rule pr ON pp.oid = pr.paroid + JOIN pg_class c2 ON pr.parchildrelid = c2.oid + WHERE c1.reltablespace != (SELECT oid FROM pg_tablespace WHERE spcname = 'pg_global') + ), + table_oids AS ( + SELECT po.nspname, po.relname, po.oid, po.own_oid, 'main' AS kind + FROM part_oids po + UNION ALL + SELECT po.nspname, po.relname, t.reltoastrelid, po.own_oid, 'toast' AS kind + FROM part_oids po + JOIN pg_class t ON po.oid = t.oid + WHERE t.reltoastrelid > 0 + UNION ALL + SELECT po.nspname, po.relname, ti.indexrelid, po.own_oid, 'toast_idx' AS kind + FROM part_oids po + JOIN pg_class t ON po.oid = t.oid + JOIN pg_index ti ON t.reltoastrelid = ti.indrelid + WHERE t.reltoastrelid > 0 + UNION ALL + SELECT po.nspname, po.relname, ao.segrelid, po.own_oid, 'ao' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + UNION ALL + SELECT po.nspname, po.relname, ao.visimaprelid, po.own_oid, 'ao_vm' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + UNION ALL + SELECT po.nspname, po.relname, ao.visimapidxid, po.own_oid, 'ao_vm_idx' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + ) + SELECT table_oids.nspname, table_oids.relname, m.segment, m.relfilenode, fs.filepath, kind, size, mtime, table_oids.own_oid own_file + FROM table_oids + JOIN relsizes_stats_schema.segment_file_map m ON table_oids.oid = m.reloid + JOIN relsizes_stats_schema.segment_file_sizes fs ON m.segment = fs.segment AND m.relfilenode = fs.relfilenode; + +CREATE OR REPLACE VIEW relsizes_stats_schema.namespace_sizes AS + SELECT nspname, sum(size) AS size FROM relsizes_stats_schema.table_files + WHERE own_file + GROUP BY nspname; diff --git a/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.2--1.3.sql b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.2--1.3.sql new file mode 100644 index 00000000000..1416af82262 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.2--1.3.sql @@ -0,0 +1,9 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_relsizes_stats" to load this file. \quit + +DO $$ +BEGIN + EXECUTE 'GRANT USAGE ON SCHEMA relsizes_stats_schema TO "' || session_user || '" WITH GRANT OPTION'; + EXECUTE 'GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO "' || session_user || '" WITH GRANT OPTION'; +END +$$; diff --git a/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.3.sql b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.3.sql new file mode 100644 index 00000000000..6a56aef77b9 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/sql/gp_relsizes_stats--1.3.sql @@ -0,0 +1,96 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION gp_relsizes_stats" to load this file. \quit + + +-- CREATE TABLE IF NOT EXISTS ... (....) DISTRIBUTED BY ... +CREATE SCHEMA IF NOT EXISTS relsizes_stats_schema; + +-- create table +CREATE TABLE IF NOT EXISTS relsizes_stats_schema.segment_file_map + (segment INTEGER, reloid OID, relfilenode OID) + WITH (appendonly=true) DISTRIBUTED RANDOMLY; +-- create table +CREATE TABLE IF NOT EXISTS relsizes_stats_schema.segment_file_sizes + (segment INTEGER, relfilenode OID, filepath TEXT, size BIGINT, mtime BIGINT) + WITH (appendonly=true, OIDS=FALSE) DISTRIBUTED RANDOMLY; +TRUNCATE TABLE relsizes_stats_schema.segment_file_sizes; +-- create table for backup info +CREATE TABLE IF NOT EXISTS relsizes_stats_schema.table_sizes_history + (insert_date date NOT NULL, nspname text NOT NULL, relname text NOT NULL, size bigint NOT NULL, mtime timestamp NOT NULL) + DISTRIBUTED RANDOMLY; +TRUNCATE TABLE relsizes_stats_schema.table_sizes_history; + + +CREATE OR REPLACE VIEW relsizes_stats_schema.table_files AS + WITH part_oids AS ( + SELECT n.nspname, c1.relname, c1.oid, true own_oid + FROM pg_class c1 + JOIN pg_namespace n ON c1.relnamespace = n.oid + WHERE c1.reltablespace != (SELECT oid FROM pg_tablespace WHERE spcname = 'pg_global') + UNION ALL + SELECT n.nspname, c1.relname, c2.oid, false own_oid + FROM pg_class c1 + JOIN pg_namespace n ON c1.relnamespace = n.oid + JOIN pg_partition pp ON c1.oid = pp.parrelid + JOIN pg_partition_rule pr ON pp.oid = pr.paroid + JOIN pg_class c2 ON pr.parchildrelid = c2.oid + WHERE c1.reltablespace != (SELECT oid FROM pg_tablespace WHERE spcname = 'pg_global') + ), + table_oids AS ( + SELECT po.nspname, po.relname, po.oid, po.own_oid, 'main' AS kind + FROM part_oids po + UNION ALL + SELECT po.nspname, po.relname, t.reltoastrelid, po.own_oid, 'toast' AS kind + FROM part_oids po + JOIN pg_class t ON po.oid = t.oid + WHERE t.reltoastrelid > 0 + UNION ALL + SELECT po.nspname, po.relname, ti.indexrelid, po.own_oid, 'toast_idx' AS kind + FROM part_oids po + JOIN pg_class t ON po.oid = t.oid + JOIN pg_index ti ON t.reltoastrelid = ti.indrelid + WHERE t.reltoastrelid > 0 + UNION ALL + SELECT po.nspname, po.relname, ao.segrelid, po.own_oid, 'ao' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + UNION ALL + SELECT po.nspname, po.relname, ao.visimaprelid, po.own_oid, 'ao_vm' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + UNION ALL + SELECT po.nspname, po.relname, ao.visimapidxid, po.own_oid, 'ao_vm_idx' AS kind + FROM part_oids po + JOIN pg_appendonly ao ON po.oid = ao.relid + ) + SELECT table_oids.nspname, table_oids.relname, m.segment, m.relfilenode, fs.filepath, kind, size, mtime, table_oids.own_oid own_file + FROM table_oids + JOIN relsizes_stats_schema.segment_file_map m ON table_oids.oid = m.reloid + JOIN relsizes_stats_schema.segment_file_sizes fs ON m.segment = fs.segment AND m.relfilenode = fs.relfilenode; +CREATE OR REPLACE VIEW relsizes_stats_schema.table_sizes AS + SELECT nspname, relname, sum(size) AS size, to_timestamp(MAX(mtime)) AS mtime FROM relsizes_stats_schema.table_files + GROUP BY nspname, relname; +CREATE OR REPLACE VIEW relsizes_stats_schema.namespace_sizes AS + SELECT nspname, sum(size) AS size FROM relsizes_stats_schema.table_files + WHERE own_file + GROUP BY nspname; +-- Here go any C or PL/SQL functions, table or view definitions etc +-- for example: + +CREATE FUNCTION relsizes_stats_schema.get_stats_for_database(dboid OID, fast BOOL) +RETURNS TABLE (segment INTEGER, relfilenode OID, filepath TEXT, size BIGINT, mtime BIGINT) +AS 'MODULE_PATHNAME', 'get_stats_for_database' +LANGUAGE C STRICT EXECUTE ON ALL SEGMENTS; + +CREATE FUNCTION relsizes_stats_schema.relsizes_collect_stats_once() +RETURNS void +AS 'MODULE_PATHNAME', 'relsizes_collect_stats_once' +LANGUAGE C STRICT EXECUTE ON MASTER; + + +DO $$ +BEGIN + EXECUTE 'GRANT USAGE ON SCHEMA relsizes_stats_schema TO "' || session_user || '" WITH GRANT OPTION'; + EXECUTE 'GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO "' || session_user || '" WITH GRANT OPTION'; +END +$$; diff --git a/gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c b/gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c new file mode 100644 index 00000000000..f26cf7989d2 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c @@ -0,0 +1,1021 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * gp_relsizes_stats.c + * + * IDENTIFICATION + * gpcontrib/gp_relsizes_stats/src/gp_relsizes_stats.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +/* Required headers for background workers */ +#include "miscadmin.h" +#include "postmaster/bgworker.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lwlock.h" +#include "storage/proc.h" +#include "storage/shmem.h" + +/* Additional headers for extension functionality */ +#include "access/xact.h" +#include "executor/spi.h" +#include "fmgr.h" +#include "lib/stringinfo.h" +#include "pgstat.h" +#include "tcop/utility.h" + +#include "catalog/namespace.h" +#include "cdb/cdbvars.h" +#include "commands/defrem.h" +#include "funcapi.h" + +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/guc.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" + +#include +#include +#include +#include +#include + +#define FILEINFO_ARGS_CNT 5 +#define HOUR_TIME 3600000 /* milliseconds in hour */ +#define MINUTE_TIME 60000 /* milliseconds in minute */ +#define FILE_NAPTIME 1 /* default naptime between file processing in milliseconds */ + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(get_stats_for_database); +PG_FUNCTION_INFO_V1(relsizes_collect_stats_once); +Datum get_stats_for_database(PG_FUNCTION_ARGS); +Datum relsizes_collect_stats_once(PG_FUNCTION_ARGS); + +static void worker_sigterm(SIGNAL_ARGS); +static Oid *get_databases_oids(int *databases_cnt, MemoryContext ctx, bool create_transaction); +static int update_segment_file_map_table(void); +static int update_table_sizes_history(void); +static void get_stats_for_databases(Oid *databases_oids, int databases_cnt, bool fast); +static void run_database_stats_worker(bool fast, Oid db); +static int plugin_created(void); +static BgwHandleStatus WaitForBackgroundWorkerShutdownSafely(BackgroundWorkerHandle *handle); +static int delete_data_in_history(void); +static int put_data_into_history(void); +void _PG_init(void); + +void relsizes_collect_stats(Datum main_arg); +void relsizes_database_stats_job(Datum args); + +/* Global variables */ +static int worker_restart_naptime = 0; +static int worker_database_naptime = 0; +static int worker_file_naptime = 0; +static bool enabled = false; + +static volatile sig_atomic_t got_sigterm = false; + +typedef union DbWorkerArg { + Datum d; + struct { + Oid db; + bool fast; + } s; +} DbWorkerArg; + +StaticAssertDecl(sizeof(Datum) == sizeof(DbWorkerArg), + "Invalid size of structure in DbWorkerArg"); + +/* + * Signal handler for SIGTERM in background worker processes. + * + * This handler is called when the postmaster requests the background worker + * to shut down. It sets the got_sigterm flag and wakes up the main worker + * loop by setting the process latch. + * + * The function follows PostgreSQL signal handling conventions: + * - Saves and restores errno + * - Uses only async-signal-safe operations + * - Sets a flag that the main loop can check + */ +static void worker_sigterm(SIGNAL_ARGS) { + int save_errno = errno; + got_sigterm = true; + if (MyProc) { + SetLatch(&MyProc->procLatch); + } + errno = save_errno; +} + +/* + * Wait for a background worker to stop with timeout and error handling. + * + * This is a modified version that adds timeout functionality and improved + * error handling to prevent infinite loops in case of hung workers. + * Returns BGWH_STOPPED on success, BGWH_POSTMASTER_DIED on error/timeout. + */ +static BgwHandleStatus WaitForBackgroundWorkerShutdownSafely(BackgroundWorkerHandle *handle) { + BgwHandleStatus status = BGWH_NOT_YET_STARTED; + int rc; + int attempts = 0; + const int max_attempts = 5 * HOUR_TIME / 100; /* maximum 5 hours wait time */ + + PG_TRY(); + { + while (attempts < max_attempts) { + pid_t pid; + + status = GetBackgroundWorkerPid(handle, &pid); + if (status == BGWH_STOPPED) { + return status; + } + + /* Add 100ms timeout instead of infinite wait */ + rc = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 100L, WAIT_EVENT_BGWORKER_SHUTDOWN); + + ResetLatch(&MyProc->procLatch); + + if (rc & WL_POSTMASTER_DEATH) { + status = BGWH_POSTMASTER_DIED; + break; + } + + /* Check for interrupts but don't let them break the entire process */ + if (QueryCancelPending || ProcDiePending) { + ereport(WARNING, (errmsg("WaitForBackgroundWorkerShutdownSafely: received interrupt signal, stopping wait"))); + status = BGWH_POSTMASTER_DIED; /* Return status as if postmaster died */ + break; + } + + attempts++; + } + + /* If maximum attempts reached */ + if (attempts >= max_attempts) { + ereport(WARNING, (errmsg("WaitForBackgroundWorkerShutdownSafely: timeout after %d attempts", max_attempts))); + status = BGWH_POSTMASTER_DIED; /* Return error status */ + } + } + PG_CATCH(); + { + /* Log error but do NOT re-throw exception */ + ereport(WARNING, (errmsg("WaitForBackgroundWorkerShutdownSafely: caught exception, returning error status"))); + /* Return error status instead of PG_RE_THROW() */ + return BGWH_POSTMASTER_DIED; + } + PG_END_TRY(); + return status; +} + +/* + * Retrieve list of database OIDs from the catalog. + * + * This function queries pg_database to get all user databases (excluding + * system databases like template0, template1, diskquota, and gpperfmon). + * + * Parameters: + * databases_cnt - Output parameter, set to number of databases found + * ctx - Memory context to allocate result in (for cross-call persistence) + * create_transaction - Whether to create a new transaction for the query + * + * Returns: + * Array of OIDs allocated in ctx, or NULL on error. + * The array length is databases_cnt. + * + * Note: Caller is responsible for freeing the returned memory when done. + */ +static Oid *get_databases_oids(int *databases_cnt, MemoryContext ctx, bool create_transaction) { + const char *sql = + "SELECT oid" + " FROM pg_database" + " WHERE datname NOT IN ('template0', 'template1', 'diskquota', 'gpperfmon')"; + const char *error = NULL; + + Oid *databases_oids = NULL; + *databases_cnt = 0; + + if (create_transaction) { + SetCurrentStatementStartTimestamp(); + StartTransactionCommand(); + PushActiveSnapshot(GetTransactionSnapshot()); + pgstat_report_activity(STATE_RUNNING, sql); + } + + if (SPI_connect() < 0) { + error = "get_databases_oids: SPI_connect failed"; + goto finish_transaction; + } + + if (SPI_execute(sql, true, 0) != SPI_OK_SELECT) { + error = "get_databases_oids: SPI_execute failed (select datname, oid)"; + goto finish_spi; + } + + /* Prepare tuple processing variables */ + + *databases_cnt = SPI_processed; + MemoryContext old_context = MemoryContextSwitchTo(ctx); + databases_oids = palloc((*databases_cnt) * sizeof(*databases_oids)); + MemoryContextSwitchTo(old_context); + + for (int i = 0; i < SPI_processed; ++i) { + Datum oid_datum; + bool oid_isnull; + + oid_datum = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &oid_isnull); + + databases_oids[i] = DatumGetObjectId(oid_datum); + } + +finish_spi: + SPI_finish(); +finish_transaction: + if (create_transaction) { + PopActiveSnapshot(); + CommitTransactionCommand(); + pgstat_report_stat(false); + pgstat_report_activity(STATE_IDLE, NULL); + } + + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + return NULL; /* Return NULL on error */ + } + + return databases_oids; +} + +/* + * Update the segment_file_map table with current relation file mappings. + * + * This function refreshes the mapping between relation OIDs and their + * physical file nodes across all segments. It first deletes the existing + * data and then repopulates it by querying pg_class on all segments. + * + * The mapping is essential for correlating file statistics collected + * from the filesystem with actual database relations. + * + * Returns: + * 0 on success, negative value on error + * + * Note: This function assumes it's running within an active SPI context. + */ +static int update_segment_file_map_table() { + int retcode = 0; + char *sql_delete = "DELETE FROM relsizes_stats_schema.segment_file_map"; + char *sql_insert = "INSERT INTO relsizes_stats_schema.segment_file_map SELECT gp_segment_id, oid, relfilenode FROM " + "gp_dist_random('pg_class')"; + char *error = NULL; + pgstat_report_activity(STATE_RUNNING, sql_delete); + retcode = SPI_execute(sql_delete, false, 0); + if (retcode != SPI_OK_DELETE) { + error = "update_segment_file_map_table: failed to delete from table"; + goto cleanup; + } + + pgstat_report_activity(STATE_RUNNING, sql_insert); + retcode = SPI_execute(sql_insert, false, 0); + if (retcode != SPI_OK_INSERT) { + error = "update_segment_file_map_table: failed to insert new rows into table"; + goto cleanup; + } + +cleanup: + pgstat_report_activity(STATE_IDLE, NULL); + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + } + return retcode; +} + +/* + * Check if a character is a digit (0-9). + * + * Simple utility function used by fill_relfilenode() to parse + * numeric portions of filenames. + * + * Returns: + * true if character is a digit, false otherwise + */ +static bool is_number(char symbol) { return '0' <= symbol && symbol <= '9'; } + +/* + * Extract relfilenode from filename by finding the first sequence of digits + * in the filename and converting it to numeric value + */ +static unsigned int fill_relfilenode(char *name) { + unsigned int result = 0, pos = 0; + size_t name_len = strlen(name); + + while (pos < name_len && !is_number(name[pos])) { + ++pos; + } + while (pos < name_len && is_number(name[pos])) { + /* Check for overflow to prevent integer overflow */ + if (result > (UINT_MAX - (name[pos] - '0')) / 10) { + break; /* Stop on potential overflow */ + } + result = (result * 10 + (name[pos] - '0')); + ++pos; + } + return result; +} + +/* + * Background worker entry point for database-specific statistics collection. + * + * This function is executed by dynamically spawned background workers to + * collect file size statistics for a specific database. Each worker: + * 1. Connects to the target database + * 2. Verifies the extension is installed + * 3. Updates the segment file mapping + * 4. Collects file size statistics from all segments + * 5. Updates the historical statistics table + * + * The function runs within its own transaction and handles errors gracefully + * by logging warnings rather than aborting the entire collection process. + * + * Parameters: + * args - Background worker argument which contains database OID and the flag + * which indicates make pauses or not + * + * Note: This function is called via the background worker framework and + * should not be called directly. + */ +void relsizes_database_stats_job(Datum args) { + int retcode = 0; + char *error = NULL; + DbWorkerArg wa = { .d = args }; + + optimizer = false; + pqsignal(SIGTERM, worker_sigterm); + BackgroundWorkerUnblockSignals(); + + BackgroundWorkerInitializeConnectionByOid(wa.s.db, InvalidOid, 0); + + SetCurrentStatementStartTimestamp(); + StartTransactionCommand(); + + retcode = SPI_connect(); + if (retcode < 0) { + error = "relsizes_database_stats_job: SPI_connect failed"; + goto finish_transaction; + } + PushActiveSnapshot(GetTransactionSnapshot()); + + /* Verify extension is installed */ + int created = plugin_created(); + if (created < 0) { + error = "relsizes_database_stats_job: SPI execute failed while looking for plugin"; + goto finish_spi; + } else if (created == 0) { + goto finish_spi; + } + + retcode = update_segment_file_map_table(); + if (retcode < 0) { + error = "relsizes_database_stats_job: updating segment_file_map failed"; + goto finish_spi; + } + + char *sql_delete = "DELETE FROM relsizes_stats_schema.segment_file_sizes"; + pgstat_report_activity(STATE_RUNNING, sql_delete); + retcode = SPI_execute(sql_delete, false, 0); + if (retcode != SPI_OK_DELETE) { + error = "relsizes_database_stats_job: SPI_execute failed (delete from segment_file_sizes)"; + goto finish_spi; + } + + /* Remove this condition after decision how to upgrade extensions is made. */ + if (SearchSysCacheExists3(PROCNAMEARGSNSP, + CStringGetDatum("get_stats_for_database"), + PointerGetDatum(buildoidvector((Oid[]){INT4OID}, 1)), + ObjectIdGetDatum(get_namespace_oid("relsizes_stats_schema", true)))) + { + const char* sql_get_stats = + "INSERT INTO relsizes_stats_schema.segment_file_sizes (segment, relfilenode, filepath, size, mtime) " + "SELECT * FROM relsizes_stats_schema.get_stats_for_database($1)"; + pgstat_report_activity(STATE_RUNNING, sql_get_stats); + retcode = SPI_execute_with_args(sql_get_stats, 1, + (Oid[]){INT4OID}, + (Datum[]){Int32GetDatum((int32) MyDatabaseId)}, + NULL, false, 0); + } else { + const char* sql_get_stats = + "INSERT INTO relsizes_stats_schema.segment_file_sizes (segment, relfilenode, filepath, size, mtime) " + "SELECT * FROM relsizes_stats_schema.get_stats_for_database($1, $2)"; + pgstat_report_activity(STATE_RUNNING, sql_get_stats); + retcode = SPI_execute_with_args(sql_get_stats, 2, + (Oid[]){OIDOID, BOOLOID}, + (Datum[]){Int32GetDatum((int32) MyDatabaseId), BoolGetDatum(wa.s.fast)}, + NULL, false, 0); + } + if (retcode != SPI_OK_INSERT) { + error = "relsizes_database_stats_job: SPI_execute failed (insert into segment_file_sizes)"; + goto finish_spi; + } + + retcode = update_table_sizes_history(); + if (retcode < 0) { + error = "relsizes_database_stats_job: updating tables sizes history table failed"; + goto finish_spi; + } + +finish_spi: + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + /* Don't abort execution, continue with cleanup */ + } + SPI_finish(); +finish_transaction: + if (ActiveSnapshotSet()) + PopActiveSnapshot(); + CommitTransactionCommand(); + pgstat_report_stat(false); + pgstat_report_activity(STATE_IDLE, NULL); +} + +/* + * Spawn and manage a background worker for database statistics collection. + * + * This function creates a new background worker to collect statistics for + * a specific database. + * + * The function: + * 1. Configures a new background worker with appropriate settings + * 2. Registers and starts the worker + * 3. Waits for the worker to complete + * 4. Handles any errors during worker execution + * + * If the worker fails to start or encounters errors during execution, + * warnings are logged but the function returns normally to allow + * processing of remaining databases. + * + * Parameters: + * fast - Don't make pauses + * db - OID of the database which worker will collect statistics from + * + * Note: This function may take significant time to complete as it waits + * for the background worker to finish processing the entire database. + */ +static void run_database_stats_worker(bool fast, Oid db) { + bool ret; + MemoryContext old_ctx; + BackgroundWorkerHandle *handle; + BgwHandleStatus status; + + /* Configure background worker */ + BackgroundWorker database_worker = { + .bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION, + .bgw_start_time = BgWorkerStart_RecoveryFinished, + .bgw_restart_time = BGW_NEVER_RESTART, + .bgw_library_name = "gp_relsizes_stats", + .bgw_function_name = "relsizes_database_stats_job", + .bgw_notify_pid = MyProcPid, + .bgw_main_arg = ((DbWorkerArg){ .s.db = db, .s.fast = fast }).d, + .bgw_start_rule = NULL, + }; + snprintf(database_worker.bgw_name, BGW_MAXLEN, "database_relsizes_collector_worker for %u", db); + old_ctx = MemoryContextSwitchTo(TopMemoryContext); + ret = RegisterDynamicBackgroundWorker(&database_worker, &handle); + MemoryContextSwitchTo(old_ctx); + if (!ret) { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("could not register background process"), + errhint("You may need to increase max_worker_processes."))); + } + pid_t pid; + status = WaitForBackgroundWorkerStartup(handle, &pid); + if (status == BGWH_STOPPED) + return; + if (status != BGWH_STARTED) { + ereport(WARNING, (errmsg("Failed to start background worker [%s], skipping", database_worker.bgw_name))); + return; + } + status = WaitForBackgroundWorkerShutdownSafely(handle); + if (status != BGWH_STOPPED) { + ereport(WARNING, (errmsg("Failure during background worker execution [%s], continuing", database_worker.bgw_name))); + /* Don't abort execution, just log and continue */ + } +} + +/* + * SQL-callable function to collect file statistics for a database. + * + * This function scans the filesystem directory corresponding to a database + * and returns statistics for all regular files found. It's designed to run + * on individual segments to collect local file information. + * + * The function: + * 1. Validates the function call context (must support returning a set) + * 2. Sets up a tuplestore for result collection + * 3. Scans the database directory (base//) + * 4. For each regular file, extracts relfilenode from filename + * 5. Collects file size and modification time via lstat() + * 6. Returns results as a set of tuples + * + * Parameters: + * Database OID (oid) - identifies which database directory to scan + * Fast (bool) - When true, don't sleep between each collect-phase for files + * + * Returns: + * Set of tuples containing: + * - segment: current segment ID + * - relfilenode: extracted from filename + * - filepath: full path to the file + * - size: file size in bytes + * - mtime: modification time as Unix timestamp + * + * Note: Includes configurable delays between file processing to reduce I/O load + */ +/* + * Scan a single directory and add file stats to the tuple store. + * Returns false if the directory could not be opened (non-fatal). + */ +static void +scan_db_dir(const char *dir_path, int segment_id, bool fast, + TupleDesc tupdesc, Tuplestorestate *tupstore) +{ + DIR *current_dir = AllocateDir(dir_path); + if (!current_dir) + { + ereport(WARNING, + (errmsg("get_stats_for_database: could not open directory \"%s\": %m", + dir_path))); + return; + } + + struct dirent *file; + while ((file = ReadDir(current_dir, dir_path)) != NULL) + { + char *filename = file->d_name; + if (strcmp(filename, ".") == 0 || strcmp(filename, "..") == 0) + continue; + + char *file_path = psprintf("%s/%s", dir_path, filename); + struct stat stb; + if (lstat(file_path, &stb) < 0) + { + ereport(WARNING, + (errmsg("get_stats_for_database: lstat failed for \"%s\" (unexpected behavior)", + file_path))); + pfree(file_path); + continue; + } + + if (S_ISREG(stb.st_mode)) + { + unsigned int relfilenode = fill_relfilenode(filename); + if (relfilenode == 0) + { + /* Skip non-relation files (PG_VERSION, pg_filenode.map, etc.) */ + pfree(file_path); + continue; + } + + Datum outputValues[FILEINFO_ARGS_CNT]; + bool outputNulls[FILEINFO_ARGS_CNT] = { false }; + + outputValues[0] = Int32GetDatum(segment_id); + outputValues[1] = ObjectIdGetDatum(relfilenode); + outputValues[2] = CStringGetTextDatum(file_path); + outputValues[3] = Int64GetDatum(stb.st_size); + outputValues[4] = Int64GetDatum(stb.st_mtime); + + tuplestore_putvalues(tupstore, tupdesc, outputValues, outputNulls); + + if (fast) + CHECK_FOR_INTERRUPTS(); + else + { + int retcode = WaitLatch(&MyProc->procLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + worker_file_naptime, WAIT_EVENT_BUFFER_IO); + ResetLatch(&MyProc->procLatch); + CHECK_FOR_INTERRUPTS(); + if (retcode & WL_POSTMASTER_DEATH) + proc_exit(1); + } + } + pfree(file_path); + } + + FreeDir(current_dir); +} + +Datum get_stats_for_database(PG_FUNCTION_ARGS) { + int segment_id = GpIdentity.segindex; + Oid dboid = PG_GETARG_OID(0); + bool fast = (PG_NARGS() < 2) ? false : PG_GETARG_BOOL(1); + + const char *error = NULL; + + ReturnSetInfo *rsinfo = (ReturnSetInfo *)fcinfo->resultinfo; + /* Validate function call context */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) { + error = "get_stats_for_database: set-valued function called in context that cannot accept a set"; + goto finish_data; + } + if (!(rsinfo->allowedModes & SFRM_Materialize)) { + error = "get_stats_for_database: materialize mode required, but it is not allowed in this context"; + goto finish_data; + } + + /* Setup output tuple store */ + MemoryContext oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); + TupleDesc tupdesc; + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) { + MemoryContextSwitchTo(oldcontext); + error = "get_stats_for_database: incorrect return type in fcinfo (must be a row type)"; + goto finish_data; + } + tupdesc = BlessTupleDesc(tupdesc); + + bool randomAccess = (rsinfo->allowedModes & SFRM_Materialize_Random) != 0; + Tuplestorestate *tupstore = tuplestore_begin_heap(randomAccess, false, work_mem); + + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + /* Scan default tablespace: $DataDir/base/ */ + { + char *default_dir = psprintf("%s/base/%u", DataDir, dboid); + scan_db_dir(default_dir, segment_id, fast, tupdesc, tupstore); + pfree(default_dir); + } + + /* Scan non-default tablespaces: $DataDir/pg_tblspc/// */ + { + char *tblspc_base = psprintf("%s/pg_tblspc", DataDir); + DIR *tblspc_dir = AllocateDir(tblspc_base); + if (tblspc_dir) + { + struct dirent *spc_entry; + while ((spc_entry = ReadDir(tblspc_dir, tblspc_base)) != NULL) + { + if (strcmp(spc_entry->d_name, ".") == 0 || + strcmp(spc_entry->d_name, "..") == 0) + continue; + + /* + * Each entry is a symlink to the tablespace directory. + * Inside it there is a version subdirectory (e.g. PG_14_202107181) + * and then per-database subdirectories named by dboid. + */ + char *spc_path = psprintf("%s/%s", tblspc_base, spc_entry->d_name); + DIR *ver_dir = AllocateDir(spc_path); + if (ver_dir) + { + struct dirent *ver_entry; + while ((ver_entry = ReadDir(ver_dir, spc_path)) != NULL) + { + if (strcmp(ver_entry->d_name, ".") == 0 || + strcmp(ver_entry->d_name, "..") == 0) + continue; + + char *db_dir = psprintf("%s/%s/%u", + spc_path, ver_entry->d_name, dboid); + scan_db_dir(db_dir, segment_id, fast, tupdesc, tupstore); + pfree(db_dir); + } + FreeDir(ver_dir); + } + pfree(spc_path); + } + FreeDir(tblspc_dir); + } + pfree(tblspc_base); + } + +finish_data: + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + /* Don't abort execution, return result */ + } + + return (Datum)0; +} + +/* + * Orchestrate statistics collection across multiple databases. + * + * This function iterates through a list of databases and spawns a background + * worker for each one to collect file statistics. It implements load balancing + * by distributing the database processing naptime across all databases. + * + * The function: + * 1. Iterates through the provided database list + * 2. Spawns a background worker for each database + * 3. Waits between databases based on configured naptime + * 4. Handles interrupts and postmaster death gracefully + * + * Parameters: + * databases_oids - Array of [name, oid] pairs for databases to process + * databases_cnt - Number of databases in the array + * fast - Don't make pauses + * + * Note: The inter-database naptime is divided by the number of databases + * to maintain consistent overall collection timing. + */ +static void get_stats_for_databases(Oid *databases_oids, int databases_cnt, bool fast) { + for (int i = 0; i < databases_cnt; ++i) { + run_database_stats_worker(fast, databases_oids[i]); + + if (fast) + CHECK_FOR_INTERRUPTS(); + else { + int naptime = (databases_cnt > 0) ? (worker_database_naptime / databases_cnt) : worker_database_naptime; + int retcode = WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, naptime, WAIT_EVENT_BGWORKER_STARTUP); + ResetLatch(&MyProc->procLatch); + CHECK_FOR_INTERRUPTS(); + /* emergency bailout if postmaster has died */ + if (retcode & WL_POSTMASTER_DEATH) { + proc_exit(1); + } + } + } +} + +/* + * Check if the gp_relsizes_stats extension is installed in the current database. + * + * This function queries pg_extension to verify that the extension has been + * properly installed before attempting to collect statistics. This prevents + * errors when the background worker tries to access extension-specific tables + * and functions. + * + * Returns: + * Number of matching extension records (should be 1 if installed), + * or -1 on query execution error + * + * Note: This function assumes it's running within an active SPI context. + */ +static int plugin_created() { + char *sql = "SELECT * FROM pg_extension WHERE extname = 'gp_relsizes_stats'"; + pgstat_report_activity(STATE_RUNNING, sql); + int retcode = SPI_execute(sql, true, 0); + pgstat_report_activity(STATE_IDLE, NULL); + return (retcode == SPI_OK_SELECT ? SPI_processed : -1); +} + +/* + * Clear all data from the table_sizes_history table. + * + * This function deletes the historical statistics table as part of the + * statistics refresh process. The table is cleared before inserting new + * current statistics to maintain a snapshot of table sizes at collection time. + * + * Returns: + * 0 on successful truncation, -1 on error + * + * Note: This function assumes it's running within an active SPI context. + */ +static int delete_data_in_history() { + char *sql = "DELETE FROM relsizes_stats_schema.table_sizes_history"; + + pgstat_report_activity(STATE_RUNNING, sql); + return (SPI_execute(sql, false, 0) == SPI_OK_DELETE ? 0 : -1); +} + +/* + * Insert current table size statistics into the history table. + * + * This function populates the table_sizes_history table with current + * statistics from the table_sizes view, adding the current date as + * the collection timestamp. This creates a historical record of + * table sizes for trend analysis. + * + * Returns: + * 0 on successful insertion, -1 on error or if no rows were inserted + * + * Note: This function assumes it's running within an active SPI context. + */ +static int put_data_into_history() { + char *sql = "INSERT INTO relsizes_stats_schema.table_sizes_history SELECT CURRENT_DATE, * FROM " + "relsizes_stats_schema.table_sizes"; + + pgstat_report_activity(STATE_RUNNING, sql); + return (SPI_execute(sql, false, 0) == SPI_OK_INSERT && SPI_processed >= 0 ? 0 : -1); +} + +/* + * Refresh the table_sizes_history table with current statistics. + * + * This function implements a complete refresh of the historical statistics + * table by first clearing all existing data and then inserting fresh + * statistics from the current collection. This ensures the history table + * contains a consistent snapshot of table sizes at the time of collection. + * + * The function performs these operations: + * 1. Deletes from the existing history table + * 2. Inserts current statistics with today's date + * + * Returns: + * 0 on successful update, negative value on error + * + * Note: This function assumes it's running within an active SPI context. + * Errors are logged as warnings but don't abort the operation. + */ +static int update_table_sizes_history() { + int retcode = 0; + char *error = NULL; + + retcode = delete_data_in_history(); + if (retcode < 0) { + error = "update_table_sizes_history: delete old data failed"; + goto cleanup; + } + + retcode = put_data_into_history(); + if (retcode < 0) { + error = "update_table_sizes_history: put actual data into history failed"; + } + +cleanup: + pgstat_report_activity(STATE_IDLE, NULL); + if (error != NULL) { + ereport(WARNING, (errmsg("%s: %m", error))); + } + return retcode; +} + +/* + * One cycle of the main background worker. + * + * The function performs these operations: + * 1. Retrieves list of all user databases + * 2. Spawns background workers to collect statistics for each database + * 3. Waits for all workers to complete before returning + * + * Parameters: + * from_worker - true when the worker calls the function, false when + * the function is called from user query. + */ +static void relsizes_collect_stats_once_internal(bool from_worker) { + int databases_cnt; + Oid *databases_oids; + + databases_oids = get_databases_oids(&databases_cnt, CurrentMemoryContext, from_worker); + if (databases_oids != NULL) { + get_stats_for_databases(databases_oids, databases_cnt, !from_worker); + pfree(databases_oids); + } else { + ereport(WARNING, (errmsg("Failed to get database OIDs"))); + } +} + +/* + * Main background worker entry point for continuous statistics collection. + * + * This function implements the main loop for the primary background worker + * that periodically collects table size statistics across all databases. + * It runs continuously until terminated by a SIGTERM signal. + * + * The worker performs these operations in each cycle: + * 1. Checks if the extension is enabled via GUC parameter + * 2. Collects statistics for each user database + * 3. Sleeps for the configured restart_naptime before next cycle + * + * The function handles: + * - Graceful shutdown on SIGTERM + * - Postmaster death detection + * - Configuration changes (enable/disable) + * - Database list changes between cycles + * - Error recovery (continues operation if individual database fails) + * + * Parameters: + * main_arg - Background worker main argument (currently unused) + * + * Note: This function should only be called via the background worker + * framework and runs in the "postgres" database context. + */ +void relsizes_collect_stats(Datum main_arg) { + optimizer = false; + pqsignal(SIGTERM, worker_sigterm); + BackgroundWorkerUnblockSignals(); + BackgroundWorkerInitializeConnection("postgres", NULL, 0); + + while (!got_sigterm) { + if (enabled) + relsizes_collect_stats_once_internal(true); + + int retcode = + WaitLatch(&MyProc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, worker_restart_naptime, WAIT_EVENT_BGWORKER_STARTUP); + ResetLatch(&MyProc->procLatch); + CHECK_FOR_INTERRUPTS(); + if (retcode & WL_POSTMASTER_DEATH) { + proc_exit(1); + } + } +} + +/* + * SQL-callable function to perform one-time statistics collection. + * + * This function provides a way to manually trigger statistics collection + * for all databases without relying on the background worker. It's useful + * for on-demand collection, testing, or when the background worker is disabled. + * The function performs the same operations as one cycle of the main + * background worker. + * + * Unlike the continuous background worker, this function: + * - Runs in the context of the calling session + * - Does not check the enabled GUC parameter + * - Returns after a single collection cycle + * - Can be called from any database where the extension is installed + * + * Returns: + * void (success/failure indicated by exception or completion) + * + * Usage: + * SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + */ +Datum relsizes_collect_stats_once(PG_FUNCTION_ARGS) { + relsizes_collect_stats_once_internal(false); + + PG_RETURN_VOID(); +} + +/* + * Extension initialization function. + * + * This function is called when the extension library is loaded. It performs + * all necessary setup for the extension including: + * 1. Defining GUC (configuration) parameters + * 2. Registering the main background worker + * + * GUC Parameters defined: + * - gp_relsizes_stats.enabled: Enable/disable the background worker + * - gp_relsizes_stats.restart_naptime: Delay between collection cycles (ms) + * - gp_relsizes_stats.database_naptime: Delay between database processing (ms) + * - gp_relsizes_stats.file_naptime: Delay between file processing (ms) + * + * The function only registers the background worker if called during + * shared_preload_libraries processing, ensuring proper initialization order. + * + * Background Worker Configuration: + * - Name: "gp_relsizes_stats_worker" + * - Entry point: relsizes_collect_stats() + * - Database: "postgres" (for catalog access) + * - Restart: Never (manual restart required) + * - Start time: After recovery completion + * + * Note: This function is called automatically by PostgreSQL when the + * extension is loaded via shared_preload_libraries. + */ +void _PG_init(void) { + /* Define GUC variables */ + DefineCustomBoolVariable("gp_relsizes_stats.enabled", "Enable main background worker flag", NULL, &enabled, false, + PGC_SIGHUP, GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); + DefineCustomIntVariable("gp_relsizes_stats.restart_naptime", "Duration between every collect-phases (in ms).", NULL, + &worker_restart_naptime, + 6 * HOUR_TIME, /* 6 hours delay between collect-phases */ + 0, INT_MAX, PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable("gp_relsizes_stats.database_naptime", "Duration between collect-phase for db (in ms).", + NULL, &worker_database_naptime, + 0, /* No delay between databases by default */ + 0, INT_MAX, PGC_SIGHUP, 0, NULL, NULL, NULL); + DefineCustomIntVariable("gp_relsizes_stats.file_naptime", "Duration between each collect-phase for files (in ms).", + NULL, &worker_file_naptime, + FILE_NAPTIME, /* 1ms delay between files */ + 0, INT_MAX, PGC_SIGHUP, 0, NULL, NULL, NULL); + + if (process_shared_preload_libraries_in_progress) { + /* Configure and register main background worker */ + RegisterBackgroundWorker(&(BackgroundWorker){ + .bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION, + .bgw_start_time = BgWorkerStart_RecoveryFinished, + .bgw_restart_time = BGW_NEVER_RESTART, + .bgw_library_name = "gp_relsizes_stats", + .bgw_function_name = "relsizes_collect_stats", + .bgw_notify_pid = 0, + .bgw_start_rule = NULL, + .bgw_name = "gp_relsizes_stats_worker" + }); + } +} diff --git a/gpcontrib/gp_relsizes_stats/test/expected/gp_relsizes_stats.out b/gpcontrib/gp_relsizes_stats/test/expected/gp_relsizes_stats.out new file mode 100644 index 00000000000..eeb3cfea1b7 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/expected/gp_relsizes_stats.out @@ -0,0 +1,187 @@ +CREATE EXTENSION gp_relsizes_stats; +CREATE TABLE employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + department_id INT, + date_of_birth DATE +); +INSERT INTO employees (first_name, last_name, department_id, date_of_birth) VALUES +('John', 'Doe', 1, '1988-06-15'), +('Jane', 'Smith', 2, '1990-07-20'), +('Emily', 'Jones', 1, '1985-08-30'); +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + size +------- + 65536 +(1 row) + +-- Fill table with a lot of different rows +insert into employees (first_name, last_name, department_id, date_of_birth) +select 'First' || i, 'Last' || i, (i % 10) + 1, DATE '1980-01-01' + (i % 365 * 365 / 30) +from generate_series(1, 10001)i; +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +-- Check that collected stats are correct +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + size +-------- + 950272 +(1 row) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +-- Validate that after rerun stats collection size of table has not change +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + size +-------- + 950272 +(1 row) + +-- Cleanup +DROP TABLE employees; +-- +-- relsizes_collect_stats_once should collect files sizes without pauses +-- The naptime value is 1ms, so the pauses take at least 10s to process 10k files. +-- Check that relsizes_collect_stats_once completes in significantly less time. +SELECT EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) t1 \gset +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT (EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) - :t1) < 5; + ?column? +---------- + t +(1 row) + +-- Cleanup +DROP TABLE t; +-- +-- Check that schema size is calculated correctly when the schema +-- contains partitioned tables and ordinary ones. +-- start_ignore +DROP SCHEMA IF EXISTS test CASCADE; +NOTICE: schema "test" does not exist, skipping +-- end_ignore +CREATE SCHEMA test; +CREATE TABLE test.t1 (i INT, j INT) +DISTRIBUTED BY (i) +PARTITION BY RANGE (i) + SUBPARTITION BY RANGE (j) + SUBPARTITION TEMPLATE (SUBPARTITION sp START (0) END (2) EVERY(1)) +(PARTITION p START (0) END (3) EVERY(1)); +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_1" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_1_2_prt_sp_1" for table "t1_1_prt_p_1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_1_2_prt_sp_2" for table "t1_1_prt_p_1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_2" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_2_2_prt_sp_1" for table "t1_1_prt_p_2" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_2_2_prt_sp_2" for table "t1_1_prt_p_2" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_3" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_3_2_prt_sp_1" for table "t1_1_prt_p_3" +NOTICE: CREATE TABLE will create partition "t1_1_prt_p_3_2_prt_sp_2" for table "t1_1_prt_p_3" +INSERT INTO test.t1 (i, j) +SELECT a % 3, a % 2 FROM generate_series(0, 2 * 3 - 1) a; +CREATE TABLE test.t2 AS SELECT 1 i DISTRIBUTED BY(i); +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT size = 32768 * 2 * 3 /* t1 */ + 32768 /* t2 */ + FROM relsizes_stats_schema.namespace_sizes + WHERE nspname = 'test'; + ?column? +---------- + t +(1 row) + +SELECT relname, segment, own_file, size + FROM relsizes_stats_schema.table_files + WHERE nspname = 'test' +ORDER BY relname, segment, own_file; + relname | segment | own_file | size +-------------------------+---------+----------+------- + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 32768 + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 0 + t1 | 0 | f | 32768 + t1 | 0 | f | 0 + t1 | 0 | t | 0 + t1 | 1 | f | 0 + t1 | 1 | f | 0 + t1 | 1 | f | 0 + t1 | 1 | f | 32768 + t1 | 1 | f | 0 + t1 | 1 | f | 32768 + t1 | 1 | f | 32768 + t1 | 1 | f | 32768 + t1 | 1 | f | 0 + t1 | 1 | t | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | f | 0 + t1 | 2 | t | 0 + t1_1_prt_p_1 | 0 | t | 0 + t1_1_prt_p_1 | 1 | t | 0 + t1_1_prt_p_1 | 2 | t | 0 + t1_1_prt_p_1_2_prt_sp_1 | 0 | t | 0 + t1_1_prt_p_1_2_prt_sp_1 | 1 | t | 32768 + t1_1_prt_p_1_2_prt_sp_1 | 2 | t | 0 + t1_1_prt_p_1_2_prt_sp_2 | 0 | t | 0 + t1_1_prt_p_1_2_prt_sp_2 | 1 | t | 32768 + t1_1_prt_p_1_2_prt_sp_2 | 2 | t | 0 + t1_1_prt_p_2 | 0 | t | 0 + t1_1_prt_p_2 | 1 | t | 0 + t1_1_prt_p_2 | 2 | t | 0 + t1_1_prt_p_2_2_prt_sp_1 | 0 | t | 0 + t1_1_prt_p_2_2_prt_sp_1 | 1 | t | 32768 + t1_1_prt_p_2_2_prt_sp_1 | 2 | t | 0 + t1_1_prt_p_2_2_prt_sp_2 | 0 | t | 0 + t1_1_prt_p_2_2_prt_sp_2 | 1 | t | 32768 + t1_1_prt_p_2_2_prt_sp_2 | 2 | t | 0 + t1_1_prt_p_3 | 0 | t | 0 + t1_1_prt_p_3 | 1 | t | 0 + t1_1_prt_p_3 | 2 | t | 0 + t1_1_prt_p_3_2_prt_sp_1 | 0 | t | 32768 + t1_1_prt_p_3_2_prt_sp_1 | 1 | t | 0 + t1_1_prt_p_3_2_prt_sp_1 | 2 | t | 0 + t1_1_prt_p_3_2_prt_sp_2 | 0 | t | 32768 + t1_1_prt_p_3_2_prt_sp_2 | 1 | t | 0 + t1_1_prt_p_3_2_prt_sp_2 | 2 | t | 0 + t2 | 0 | t | 0 + t2 | 1 | t | 32768 + t2 | 2 | t | 0 +(60 rows) + +DROP SCHEMA test CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table test.t1 +drop cascades to table test.t2 diff --git a/gpcontrib/gp_relsizes_stats/test/expected/grants.out b/gpcontrib/gp_relsizes_stats/test/expected/grants.out new file mode 100644 index 00000000000..16d77930c0c --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/expected/grants.out @@ -0,0 +1,104 @@ +-- Check that user who has created gp_relsizes_stats have privileges to use all +-- tables, views and functions from the extension. +-- Check that this user can grant this privileges to others. +SELECT '\! cp "' || setting || '/pg_hba.conf" "' || setting || '/pg_hba.conf.backup"' as cp_backup +FROM pg_settings +WHERE name = 'data_directory' \gset +:cp_backup +SELECT '\! echo "local all user1,user2 trust" >> ' || setting || '/pg_hba.conf' as add_users +FROM pg_settings +WHERE name = 'data_directory' \gset +:add_users +CREATE ROLE user1 LOGIN RESOURCE QUEUE pg_default; +CREATE ROLE user2 LOGIN RESOURCE QUEUE pg_default; +CREATE DATABASE db1 OWNER user1; +\set initial_user :USER +\set initial_db :DBNAME +\c db1 user1 +CREATE EXTENSION gp_relsizes_stats; +-- Check that user who has created gp_relsizes_stats can use the extension +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +-- +(0 rows) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; +-- +(0 rows) + +-- Check that user who has created gp_relsizes_stats can grant privileges to others +GRANT USAGE ON SCHEMA relsizes_stats_schema TO user2; +GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO user2; +-- Check that user2 has got required privileges +\c - user2 +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +-- +(0 rows) + +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +-- +(0 rows) + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + relsizes_collect_stats_once +----------------------------- + +(1 row) + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; +-- +(0 rows) + +-- Cleanup +\c :"initial_db" :"initial_user" +DROP DATABASE db1; +DROP ROLE user1, user2; +SELECT '\! mv "' || setting || '/pg_hba.conf.backup" "' || setting || '/pg_hba.conf"' as cp_restore +FROM pg_settings +WHERE name = 'data_directory' \gset +:cp_restore diff --git a/gpcontrib/gp_relsizes_stats/test/sql/gp_relsizes_stats.sql b/gpcontrib/gp_relsizes_stats/test/sql/gp_relsizes_stats.sql new file mode 100644 index 00000000000..9d5d0cd89af --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/sql/gp_relsizes_stats.sql @@ -0,0 +1,96 @@ +CREATE EXTENSION gp_relsizes_stats; + +-- start_ignore +DROP TABLE IF EXISTS employees; +-- end_ignore +CREATE TABLE employees ( + employee_id SERIAL PRIMARY KEY, + first_name VARCHAR(50) NOT NULL, + last_name VARCHAR(50) NOT NULL, + department_id INT, + date_of_birth DATE +); + +INSERT INTO employees (first_name, last_name, department_id, date_of_birth) VALUES +('John', 'Doe', 1, '1988-06-15'), +('Jane', 'Smith', 2, '1990-07-20'), +('Emily', 'Jones', 1, '1985-08-30'); + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + +-- Fill table with a lot of different rows +insert into employees (first_name, last_name, department_id, date_of_birth) +select 'First' || i, 'Last' || i, (i % 10) + 1, DATE '1980-01-01' + (i % 365 * 365 / 30) +from generate_series(1, 10001)i; + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +-- Check that collected stats are correct +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +-- Validate that after rerun stats collection size of table has not change +SELECT size FROM relsizes_stats_schema.table_sizes_history WHERE relname = 'employees'; + +-- Cleanup +DROP TABLE employees; + + +-- +-- relsizes_collect_stats_once should collect files sizes without pauses +-- The naptime value is 1ms, so the pauses take at least 10s to process 10k files. +-- Check that relsizes_collect_stats_once completes in significantly less time. + +-- start_ignore +DROP TABLE IF EXISTS t; +CREATE TABLE t (i int) +DISTRIBUTED RANDOMLY +PARTITION BY RANGE (i) (PARTITION a START (0) END (10000) EVERY (1)); +-- end_ignore + +SELECT EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) t1 \gset + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT (EXTRACT(EPOCH FROM LOCALTIMESTAMP(0)) - :t1) < 5; + +-- Cleanup +DROP TABLE t; + + +-- +-- Check that schema size is calculated correctly when the schema +-- contains partitioned tables and ordinary ones. + +-- start_ignore +DROP SCHEMA IF EXISTS test CASCADE; +-- end_ignore +CREATE SCHEMA test; + +CREATE TABLE test.t1 (i INT, j INT) +DISTRIBUTED BY (i) +PARTITION BY RANGE (i) + SUBPARTITION BY RANGE (j) + SUBPARTITION TEMPLATE (SUBPARTITION sp START (0) END (2) EVERY(1)) +(PARTITION p START (0) END (3) EVERY(1)); + +INSERT INTO test.t1 (i, j) +SELECT a % 3, a % 2 FROM generate_series(0, 2 * 3 - 1) a; + +CREATE TABLE test.t2 AS SELECT 1 i DISTRIBUTED BY(i); + +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT size = 32768 * 2 * 3 /* t1 */ + 32768 /* t2 */ + FROM relsizes_stats_schema.namespace_sizes + WHERE nspname = 'test'; + +SELECT relname, segment, own_file, size + FROM relsizes_stats_schema.table_files + WHERE nspname = 'test' +ORDER BY relname, segment, own_file; + +DROP SCHEMA test CASCADE; diff --git a/gpcontrib/gp_relsizes_stats/test/sql/grants.sql b/gpcontrib/gp_relsizes_stats/test/sql/grants.sql new file mode 100644 index 00000000000..3bff541c242 --- /dev/null +++ b/gpcontrib/gp_relsizes_stats/test/sql/grants.sql @@ -0,0 +1,82 @@ +-- Check that user who has created gp_relsizes_stats have privileges to use all +-- tables, views and functions from the extension. +-- Check that this user can grant this privileges to others. + +-- start_ignore +DROP DATABASE IF EXISTS db1; +DROP ROLE IF EXISTS user1, user2; +-- end_ignore + +SELECT '\! cp "' || setting || '/pg_hba.conf" "' || setting || '/pg_hba.conf.backup"' as cp_backup +FROM pg_settings +WHERE name = 'data_directory' \gset + +:cp_backup + +SELECT '\! echo "local all user1,user2 trust" >> ' || setting || '/pg_hba.conf' as add_users +FROM pg_settings +WHERE name = 'data_directory' \gset + +:add_users + +-- start_ignore +\! gpstop -u +-- end_ignore + +CREATE ROLE user1 LOGIN RESOURCE QUEUE pg_default; +CREATE ROLE user2 LOGIN RESOURCE QUEUE pg_default; +CREATE DATABASE db1 OWNER user1; + +\set initial_user :USER +\set initial_db :DBNAME + +\c db1 user1 + +CREATE EXTENSION gp_relsizes_stats; + +-- Check that user who has created gp_relsizes_stats can use the extension +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; + +-- Check that user who has created gp_relsizes_stats can grant privileges to others +GRANT USAGE ON SCHEMA relsizes_stats_schema TO user2; +GRANT SELECT ON ALL TABLES IN SCHEMA relsizes_stats_schema TO user2; + +-- Check that user2 has got required privileges +\c - user2 +SELECT FROM relsizes_stats_schema.segment_file_map LIMIT 0; +SELECT FROM relsizes_stats_schema.segment_file_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes_history LIMIT 0; +SELECT FROM relsizes_stats_schema.table_files LIMIT 0; +SELECT FROM relsizes_stats_schema.table_sizes LIMIT 0; +SELECT FROM relsizes_stats_schema.namespace_sizes LIMIT 0; +SELECT relsizes_stats_schema.relsizes_collect_stats_once(); + +SELECT FROM relsizes_stats_schema.get_stats_for_database( + (SELECT oid FROM pg_database WHERE datname = current_database()), true) +LIMIT 0; + +-- Cleanup +\c :"initial_db" :"initial_user" + +DROP DATABASE db1; +DROP ROLE user1, user2; + +SELECT '\! mv "' || setting || '/pg_hba.conf.backup" "' || setting || '/pg_hba.conf"' as cp_restore +FROM pg_settings +WHERE name = 'data_directory' \gset + +:cp_restore + +-- start_ignore +\! gpstop -u +-- end_ignore