author | Radek Brich <radek.brich@devl.cz> |
Tue, 11 Dec 2012 10:49:42 +0100 | |
changeset 52 | 26121a8fe78b |
parent 35 | e7f79c4a27ce |
child 56 | 94e091c23ebb |
permissions | -rwxr-xr-x |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
1 |
#!/usr/bin/env python3.2 |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
2 |
# |
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
3 |
# Copy data between tables with same table schema. |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
4 |
# |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
5 |
# Can copy multiple tables in one run. |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
6 |
# Sorts the tables according to references. |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
7 |
# |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
8 |
# This may be used instead of dump/restore. |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
9 |
# |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
10 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
11 |
import io |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
12 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
13 |
from pgtoolkit import toolbase, pgmanager, pgbrowser, pgdatacopy |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
14 |
from pgtoolkit.progresswrapper import ProgressWrapper |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
15 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
16 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
17 |
class TableCopyTool(toolbase.SrcDstTablesTool): |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
18 |
def __init__(self): |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
19 |
toolbase.SrcDstTablesTool.__init__(self, name='tablecopy', desc='Table copy tool.') |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
20 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
21 |
self.parser.add_argument('-n', '--no-action', dest='noaction', action='store_true', |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
22 |
help="Do nothing, just print tables to be copied. Useful in combination with --regex.") |
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
23 |
self.parser.add_argument('--no-sort', dest='nosort', action='store_true', |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
24 |
help="Do not sort. By default, tables are sorted by foreign key references.") |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
25 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
26 |
self.init() |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
27 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
28 |
def main(self): |
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
29 |
self.srcconn = self.pgm.get_conn('src') |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
30 |
self.dstconn = self.pgm.get_conn('dst') |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
31 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
32 |
dc = pgdatacopy.PgDataCopy(self.srcconn, self.dstconn) |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
33 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
34 |
if self.args.nosort: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
35 |
for table in self.tables(): |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
36 |
self.copy_table(dc, *table) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
37 |
else: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
38 |
# sort tables with respect to references |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
39 |
details = dict() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
40 |
pending = set() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
41 |
references = dict() |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
42 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
43 |
# build list of all table to be copied (pending) and references map |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
44 |
for table in self.tables(): |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
45 |
srcschema, srctable, dstschema, dsttable = table |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
46 |
name = dstschema + '.' + dsttable |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
47 |
details[name] = table |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
48 |
pending.add(name) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
49 |
references[name] = self.get_references(dstschema, dsttable) |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
50 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
51 |
# copy files with fulfilled references, repeat until all done |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
52 |
while pending: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
53 |
for name in list(pending): |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
54 |
ok = True |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
55 |
for ref in references[name]: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
56 |
if ref in pending: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
57 |
ok = False |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
58 |
if ok: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
59 |
table = details[name] |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
60 |
self.copy_table(dc, *table) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
61 |
pending.remove(name) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
62 |
|
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
63 |
def get_references(self, schema, table): |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
64 |
browser = pgbrowser.PgBrowser(self.dstconn) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
65 |
cons = browser.list_constraints(table, schema) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
66 |
return [con['fschema'] + '.' + con['fname'] for con in cons if con['type'] == 'f'] |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
67 |
|
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
68 |
def copy_table(self, dc, srcschema, srctable, dstschema, dsttable): |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
69 |
print('Copying [%s] %s.%s --> [%s] %s.%s' % ( |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
70 |
self.args.src, srcschema, srctable, |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
71 |
self.args.dst, dstschema, dsttable)) |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
72 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
73 |
if self.args.noaction: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
74 |
return |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
75 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
76 |
dc.set_source(srctable, srcschema) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
77 |
dc.set_destination(dsttable, dstschema) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
78 |
|
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
79 |
try: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
80 |
dc.check() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
81 |
except pgdatacopy.TargetNotEmptyError as e: |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
82 |
print(' - error:', str(e)) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
83 |
return |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
84 |
|
35
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
85 |
print(' - read ') |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
86 |
buf = io.BytesIO() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
87 |
wrapped = ProgressWrapper(buf) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
88 |
dc.read(wrapped) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
89 |
data = buf.getvalue() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
90 |
buf.close() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
91 |
|
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
92 |
print(' - write ') |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
93 |
buf = io.BytesIO(data) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
94 |
wrapped = ProgressWrapper(buf, len(data)) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
95 |
dc.write(wrapped) |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
96 |
buf.close() |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
97 |
|
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
98 |
print(' - analyze ') |
e7f79c4a27ce
TableCopyTool: order tables with respect to references.
Radek Brich <radek.brich@devl.cz>
parents:
34
diff
changeset
|
99 |
dc.analyze() |
34
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
100 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
101 |
|
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
102 |
tool = TableCopyTool() |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
103 |
tool.main() |
98c7809af415
Add PgDataCopy. Add TableCopyTool.Add SrcDstTablesTool class to toolbase, use in tablecopy, tablediff.
Radek Brich <radek.brich@devl.cz>
parents:
diff
changeset
|
104 |