pgtoolkit/tools/bigtables.py
author Radek Brich <radek.brich@devl.cz>
Mon, 26 May 2014 18:18:21 +0200
changeset 103 24e94a3da209
parent 101 2a2d0d5df03b
permissions -rw-r--r--
Update bigtables tool: Sort by size with indexes, not just data.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
101
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
     1
from pgtoolkit.toolbase import SimpleTool
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
     2
from pgtoolkit import pgbrowser
95
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
     3
from pycolib.prettysize import prettysize_short
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
     4
from pycolib.ansicolor import highlight
2
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
     5
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
     6
101
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
     7
class BigTablesTool(SimpleTool):
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
     8
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
     9
    """List largest tables.
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    10
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    11
    Reads size statistics of tables and indexes from pgcatalog.
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    12
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    13
    """
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    14
2
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    15
    def __init__(self):
101
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    16
        SimpleTool.__init__(self, name='bigtables')
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    17
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    18
    def specify_args(self):
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    19
        SimpleTool.specify_args(self)
95
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    20
        self.parser.add_argument('-n', '--limit', metavar='NUM', dest='limit', type=int, default=5, help='Show NUM biggest tables.')
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    21
        self.parser.add_argument('-v', '--details', dest='details', action='store_true', help='Show sizes of data and individual indexes.')
2
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    22
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    23
    def main(self):
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    24
        browser = pgbrowser.PgBrowser(self.pgm.get_conn('target'))
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    25
38
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    26
        # scan all tables from all shemas, remember names and sizes
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    27
        all_tables = []
95
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    28
        all_indexes = []
38
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    29
        schemas = browser.list_schemas()
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    30
        for schema in schemas:
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    31
            tables = browser.list_tables(schema['name'])
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    32
            for table in tables:
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    33
                table_name = '%s.%s' % (schema['name'], table['name'])
95
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    34
                indexes = browser.list_indexes(table['name'], schema['name'])
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    35
                for index in indexes:
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    36
                    all_indexes.append({'name': index['name'], 'table': table_name, 'size': index['size']})
103
24e94a3da209 Update bigtables tool: Sort by size with indexes, not just data.
Radek Brich <radek.brich@devl.cz>
parents: 101
diff changeset
    37
                size_with_indexes = table['size'] + sum(index['size'] for index in indexes)
24e94a3da209 Update bigtables tool: Sort by size with indexes, not just data.
Radek Brich <radek.brich@devl.cz>
parents: 101
diff changeset
    38
                all_tables.append({'name': table_name, 'size': table['size'], 'indexes': indexes, 'size_with_indexes': size_with_indexes})
83
515fadd3d286 Add dependency on pycolib. Move common modules to pycolib. Add example table schema for meta DB.
Radek Brich <radek.brich@devl.cz>
parents: 38
diff changeset
    39
38
d3593869d624 Update bigtables tool: scan all schemas.
Radek Brich <radek.brich@devl.cz>
parents: 9
diff changeset
    40
        # print names and sizes of 20 largest tables
103
24e94a3da209 Update bigtables tool: Sort by size with indexes, not just data.
Radek Brich <radek.brich@devl.cz>
parents: 101
diff changeset
    41
        for table in sorted(all_tables, reverse=True, key=lambda x: x['size_with_indexes'])[:self.args.limit]:
24e94a3da209 Update bigtables tool: Sort by size with indexes, not just data.
Radek Brich <radek.brich@devl.cz>
parents: 101
diff changeset
    42
            print(highlight(1) + prettysize_short(table['size_with_indexes'], trailing_zeros=True).rjust(8) + highlight(0),
95
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    43
                  '(total)'.ljust(8),
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    44
                  highlight(1) + table['name'] + highlight(0), sep='  ')
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    45
            if self.args.details:
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    46
                print(prettysize_short(table['size'], trailing_zeros=True).rjust(8),
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    47
                      '(data)'.ljust(8), sep='  ')
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    48
                for index in sorted(table['indexes'], reverse=True, key=lambda x: x['size']):
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    49
                    print(prettysize_short(index['size'], trailing_zeros=True).rjust(8),
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    50
                          '(index)'.ljust(8), index['name'], sep='  ')
6adcb7ee4517 Update bigtables tool: Add size of indexes.
Radek Brich <brich.radek@ifortuna.cz>
parents: 93
diff changeset
    51
            print()
2
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    52
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    53
101
2a2d0d5df03b Refactor ToolBase to allow tool composition. Add TableSync tool (composited). Move more tools under pgtool.
Radek Brich <brich.radek@ifortuna.cz>
parents: 95
diff changeset
    54
cls = BigTablesTool
2
efee419b7a2d Fix pgmanager exceptions.
Radek Brich <radek.brich@devl.cz>
parents:
diff changeset
    55