Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
07052cd704 | ||
|
|
a258a44f2a | ||
|
|
f34bc2cba0 | ||
|
|
f75042ac5d | ||
|
|
bf6c1ec3d3 | ||
|
|
271087a63b | ||
|
|
53ef02da13 | ||
|
|
620492183f | ||
|
|
8eb1d3031c | ||
|
|
e6faa29d36 | ||
|
|
8b6e3f1a2e | ||
|
|
1f973cc3ff | ||
|
|
64180398eb | ||
|
|
79396ffccc | ||
|
|
b00a0cc77b | ||
|
|
7236e82911 | ||
|
|
7b58ded3b3 | ||
|
|
3c7150850e | ||
|
|
cd9f7bce91 | ||
|
|
805c416251 | ||
|
|
371e18dde4 | ||
|
|
524edc9f0b | ||
|
|
306d4577a5 | ||
|
|
894821e1ef | ||
|
|
b88b868b91 | ||
|
|
b8ee1b9733 | ||
|
|
f5a22be0a0 | ||
|
|
423c2d35ee | ||
|
|
833d00711e | ||
|
|
ace4ca8556 | ||
|
|
8a7f164b67 | ||
|
|
5180ec47dc | ||
|
|
4598673138 | ||
|
|
2c5115907f | ||
|
|
3d2a99ec64 | ||
|
|
20a1dcfe80 | ||
|
|
43f6d84de7 | ||
|
|
00b39b2b6c | ||
|
|
3a349451e2 | ||
|
|
098a3e7efe | ||
|
|
ec26edc735 | ||
|
|
1f66249648 | ||
|
|
5cc28f061e | ||
|
|
6cdebb0871 | ||
|
|
8e142dfaa7 | ||
|
|
9b3db6cd7d | ||
|
|
fb3ffecbb5 | ||
|
|
1de555b72d | ||
|
|
39cc0970bd | ||
|
|
831d8c9ae6 | ||
|
|
d75ded973a | ||
|
|
583757ab2d | ||
|
|
6dc95bd201 | ||
|
|
b4872fb19d | ||
|
|
777550e4d7 | ||
|
|
51f9aabf68 | ||
|
|
cbadf63c13 | ||
|
|
d9d6d85ecd | ||
|
|
9dcc6f061f | ||
|
|
115177e679 | ||
|
|
67536b7f1c | ||
|
|
9e6ca5ccaa | ||
|
|
7291b60643 | ||
|
|
64db556342 | ||
|
|
607b1142fa | ||
|
|
22b2dbf747 | ||
|
|
e1085cfef2 | ||
|
|
1637f18132 | ||
|
|
453547cf81 | ||
|
|
a50ea56eda | ||
|
|
402f080149 | ||
|
|
a74470d619 | ||
|
|
df9067fec5 | ||
|
|
5d38bead9a | ||
|
|
c2d3db8c2b | ||
|
|
0f10e74f69 | ||
|
|
459f15b500 | ||
|
|
5439c3a702 | ||
|
|
69093192ef | ||
|
|
f29923306d | ||
|
|
8b62949069 | ||
|
|
8346ecefea | ||
|
|
4ff83b9d74 | ||
|
|
dde480b416 | ||
|
|
57c0e0d7c0 | ||
|
|
9ca70b31ef | ||
|
|
07bed0e214 | ||
|
|
3de5c9af24 | ||
|
|
c32255c13d | ||
|
|
407d0cec58 | ||
|
|
3061f4564a | ||
|
|
9e2a0cdc38 | ||
|
|
49afea4d86 | ||
|
|
4e593fd2a9 | ||
|
|
3001e743c3 | ||
|
|
f6d44b4b24 | ||
|
|
0eedf49c28 | ||
|
|
d993b8eaee | ||
|
|
4146594df3 | ||
|
|
8261b52d33 | ||
|
|
a4820755ab | ||
|
|
c27f930c6e | ||
|
|
6355956008 | ||
|
|
2f4bfa29e5 | ||
|
|
4100c1d952 | ||
|
|
7965ee37ab | ||
|
|
fb317bb597 | ||
|
|
6b1121a658 | ||
|
|
525949de9f | ||
|
|
09f532b773 | ||
|
|
dc7fa068e3 | ||
|
|
e57ac62141 | ||
|
|
516bfe1c27 | ||
|
|
0491d12e50 | ||
|
|
b7e05731ea | ||
|
|
0bea9205d9 | ||
|
|
bf817e5de6 | ||
|
|
bfa4bdf7a0 | ||
|
|
9e851de675 | ||
|
|
98a84cdaec | ||
|
|
46206b7330 | ||
|
|
cf23a7307e | ||
|
|
515a6d1664 | ||
|
|
1fc246e82d | ||
|
|
33cf17647b | ||
|
|
df5e294c5c | ||
|
|
29df549a5b | ||
|
|
49ae0330b0 | ||
|
|
20e6881fd8 | ||
|
|
1ed56cb468 | ||
|
|
73ba745097 | ||
|
|
7b6a813149 | ||
|
|
9c45ae5db0 | ||
|
|
9c9a49920d | ||
|
|
41dfe1d575 | ||
|
|
b5bd00dc8a | ||
|
|
89b51584c5 | ||
|
|
63f26f6fb9 | ||
|
|
41b578a45e | ||
|
|
9d1abc92fa | ||
|
|
ae62039386 | ||
|
|
a7c2a28bec | ||
|
|
dc9b4dab5c | ||
|
|
7f20f01a3b | ||
|
|
57b2ba7115 | ||
|
|
bbe4ec3cde | ||
|
|
84f30c64d0 | ||
|
|
241ad96665 | ||
|
|
fc060940b4 | ||
|
|
9ef241d273 | ||
|
|
002e0aced4 | ||
|
|
be590bf1b9 | ||
|
|
3dc81826fd | ||
|
|
b6df2cab18 | ||
|
|
0601111d88 | ||
|
|
0064ee83cf | ||
|
|
a354c6d446 | ||
|
|
e5b4be7771 | ||
|
|
d5960ade4a | ||
|
|
d068374972 | ||
|
|
947936279a | ||
|
|
42492c82ca | ||
|
|
97cfb4d2a6 | ||
|
|
d80e0e2f7f | ||
|
|
eb7d0dd6a1 | ||
|
|
9e406c40b5 | ||
|
|
b071731cc2 | ||
|
|
d3ec78d996 | ||
|
|
98c383cc42 | ||
|
|
12f7e22e36 | ||
|
|
6c7d3123c2 | ||
|
|
86e4e4965d | ||
|
|
d0c2e446b7 | ||
|
|
bb1e9dd3ae | ||
|
|
dc5b8a46c6 | ||
|
|
d90f7d51aa | ||
|
|
d6fa9be6e6 | ||
|
|
fb4fb978e1 | ||
|
|
604f1254ba | ||
|
|
b49854bde5 | ||
|
|
fb771c8dd8 | ||
|
|
44b07c5f7b | ||
|
|
e98f9f6a24 | ||
|
|
95623b991f | ||
|
|
1d402e46a8 | ||
|
|
d7e0e40e0a | ||
|
|
74b9fdb3cb | ||
|
|
01107cb677 | ||
|
|
e93eaeb3fe | ||
|
|
b55570703d | ||
|
|
fd16003059 | ||
|
|
71c6afaf98 | ||
|
|
bb99610517 | ||
|
|
ff1977cbd4 | ||
|
|
14a3bcdaa8 | ||
|
|
ffea024e7c | ||
|
|
d4e8e9ac07 | ||
|
|
5399892051 | ||
|
|
8d9d92b7b1 | ||
|
|
cde3cfc30d | ||
|
|
481bf0fb03 | ||
|
|
9f6c7e5fc9 | ||
|
|
7c76166697 | ||
|
|
0d13e794e4 | ||
|
|
8437fe6dc6 | ||
|
|
dd0ead011e | ||
|
|
ffbd6893f6 | ||
|
|
bad962cf21 | ||
|
|
293bfec844 | ||
|
|
bc0b686fc0 | ||
|
|
e08ae0e996 | ||
|
|
e877644ed1 | ||
|
|
5ed8a441d4 | ||
|
|
6ff5b4ed4c | ||
|
|
3d4dad5655 | ||
|
|
ec3ae16f2e | ||
|
|
9f282f4ee3 | ||
|
|
e85c5f51b9 | ||
|
|
2327ddb52d | ||
|
|
7b3cd164d7 | ||
|
|
aeb7f633f3 | ||
|
|
7817573893 | ||
|
|
923901c81f | ||
|
|
bd81e32d1a | ||
|
|
e211c5b5f5 | ||
|
|
98e5f88b9f | ||
|
|
5ac583ba7b | ||
|
|
3bd91a774e | ||
|
|
95290f2fbd | ||
|
|
14e6c50d2e | ||
|
|
c273636efe | ||
|
|
2fbec2f760 | ||
|
|
ae500dbe45 | ||
|
|
50e972912d | ||
|
|
e364488761 | ||
|
|
c7e9414bfe | ||
|
|
f2e5e31837 | ||
|
|
10e2545ceb | ||
|
|
89970b8951 | ||
|
|
66510b82c4 | ||
|
|
6c96ef02df | ||
|
|
ddc72468fc | ||
|
|
16197b67e3 | ||
|
|
6e2056f5b4 | ||
|
|
8d63b3f95b | ||
|
|
43e4503343 | ||
|
|
c1404f0f98 | ||
|
|
46ee425bac | ||
|
|
81f457f379 | ||
|
|
07c594d988 | ||
|
|
be4c129168 | ||
|
|
ef4a805b02 | ||
|
|
5059f0bee6 | ||
|
|
c31948caeb | ||
|
|
2758eb210f | ||
|
|
465060c9a7 | ||
|
|
8f6bfb0295 | ||
|
|
cac03318eb | ||
|
|
7bf5fd7859 | ||
|
|
2c9773ed89 | ||
|
|
46fb82ce3d | ||
|
|
144aaaa986 | ||
|
|
14f26d8818 | ||
|
|
f8f7beea54 | ||
|
|
c6c22b34c8 | ||
|
|
752cdc3b11 | ||
|
|
f5c4ccff1f | ||
|
|
5e004dbef4 | ||
|
|
4fba554b47 | ||
|
|
6593dc8dbf | ||
|
|
480a5a634a | ||
|
|
f213007d4d | ||
|
|
0f8263de2e | ||
|
|
9b449be538 | ||
|
|
3db0395e0a | ||
|
|
fe66cd7734 | ||
|
|
4190b85018 | ||
|
|
0a355bbee0 | ||
|
|
407d1e3982 | ||
|
|
9b551a1c10 | ||
|
|
c0f025d2f6 | ||
|
|
56620dc3dd | ||
|
|
598addc6d7 | ||
|
|
95e701da6a | ||
|
|
f6d624c729 | ||
|
|
5ad19c5071 | ||
|
|
41a595f62a | ||
|
|
f58da1d4dc | ||
|
|
b57dbceec3 | ||
|
|
06c80ae2da | ||
|
|
0beddcd2d0 | ||
|
|
94f5aa00fc | ||
|
|
5bf80d7ef2 | ||
|
|
09a1959309 | ||
|
|
15713e64cb | ||
|
|
519dae2639 | ||
|
|
ce7a2234ee | ||
|
|
a06a92d620 | ||
|
|
1ca4da9a33 | ||
|
|
52fc08fdf5 | ||
|
|
2feaaaddda | ||
|
|
9870a84b99 | ||
|
|
bbec63fda7 | ||
|
|
809c7f0ac8 | ||
|
|
fd47ba9a9a | ||
|
|
b1a199a272 | ||
|
|
fc8d57d128 | ||
|
|
f24ea4d6c9 | ||
|
|
12b93a3a07 | ||
|
|
3fa0df7553 | ||
|
|
13978a108a | ||
|
|
d549fd5265 | ||
|
|
30d2d94a22 | ||
|
|
caa5719a86 | ||
|
|
436b81b357 | ||
|
|
600f2eda93 | ||
|
|
309edb6da6 | ||
|
|
bcd4f6e7dd | ||
|
|
fd50054781 | ||
|
|
26064905d4 | ||
|
|
24176463f4 | ||
|
|
418d41c2a1 | ||
|
|
01291f7cb5 | ||
|
|
e8b8975ec3 | ||
|
|
6a53d7d10d | ||
|
|
65870ff3e8 | ||
|
|
6f1479fa41 | ||
|
|
f694b616aa | ||
|
|
1bd41c2a0b | ||
|
|
f1d88ac6c1 | ||
|
|
0d5d918563 | ||
|
|
547bc41026 | ||
|
|
d27dc63026 | ||
|
|
83f5b9f644 | ||
|
|
addb3c1a79 | ||
|
|
88e7eafcf9 | ||
|
|
5f88d4bf22 | ||
|
|
7f00868584 | ||
|
|
6ebc6c7dae | ||
|
|
f38fe38d45 | ||
|
|
323427e951 | ||
|
|
06d3ebde1c | ||
|
|
1df8057f97 | ||
|
|
d2d449ad78 | ||
|
|
16d63e3a76 | ||
|
|
cc573a74dc | ||
|
|
1d54cf5f18 | ||
|
|
4e098f74cd | ||
|
|
b0300df5e7 | ||
|
|
9c132fe622 | ||
|
|
22334834ac | ||
|
|
3373343f23 | ||
|
|
1d0bedc454 | ||
|
|
8381ea0307 | ||
|
|
2e0b820637 | ||
|
|
9f5e1bd505 | ||
|
|
7db5c9a274 | ||
|
|
4fad212a25 | ||
|
|
f6b3bf548d | ||
|
|
ca888b96e3 | ||
|
|
2fc1610248 | ||
|
|
bc28cee1dc | ||
|
|
ea1928c904 | ||
|
|
8c3f34e0c9 | ||
|
|
956441176f | ||
|
|
83d01b7e1c | ||
|
|
fc2b2e9c64 | ||
|
|
60b45ba428 | ||
|
|
107db3f407 | ||
|
|
4937cd088f | ||
|
|
5e96539f55 | ||
|
|
708d67fa02 | ||
|
|
a95b13d804 | ||
|
|
1e60261d3f | ||
|
|
0d4f1c0212 | ||
|
|
4d30c4c87f | ||
|
|
6c02d65563 | ||
|
|
16dea48c82 | ||
|
|
f517029483 | ||
|
|
6f233b3b58 | ||
|
|
f8c85735f4 | ||
|
|
1e4c50b70d | ||
|
|
93d53a5ae8 | ||
|
|
16f2b1736e | ||
|
|
4d60607b0b | ||
|
|
beb09d85fa | ||
|
|
8ed902364c | ||
|
|
02c83fb38b | ||
|
|
9049e49f9e | ||
|
|
9d36dd9841 | ||
|
|
9905791932 | ||
|
|
dd00ca926e | ||
|
|
4092e160a4 | ||
|
|
6092802b3a | ||
|
|
42f074062b | ||
|
|
112191042b | ||
|
|
1566dd06cd | ||
|
|
251bf76be4 | ||
|
|
a6eddccfe5 | ||
|
|
3bd43bedad | ||
|
|
4750e57ce1 | ||
|
|
df7ce674e9 | ||
|
|
e968e8a5f0 | ||
|
|
c8705dd876 | ||
|
|
c7a29e331f | ||
|
|
ffcbbd5fea | ||
|
|
ebf9c0bf91 | ||
|
|
df658e8fa8 | ||
|
|
b5edb8e50c | ||
|
|
97eb0d191b | ||
|
|
857d7a620d | ||
|
|
fad64ed381 | ||
|
|
4244ebea6a | ||
|
|
327822c29a | ||
|
|
feea45c205 | ||
|
|
c4b81d1210 | ||
|
|
3988074cae | ||
|
|
ad2b69e115 | ||
|
|
f9eb960848 | ||
|
|
5927d6e5f4 | ||
|
|
6bb3e494e5 | ||
|
|
5d5d15e7a3 | ||
|
|
b742ec3de4 | ||
|
|
75f0aade0d | ||
|
|
3872c8f41d | ||
|
|
ae2fd4fa7b | ||
|
|
3eb26a65d3 | ||
|
|
157433893c | ||
|
|
236dac0c96 | ||
|
|
78d696c1e5 | ||
|
|
48a4062f75 | ||
|
|
18583d5a91 | ||
|
|
642913dbf4 | ||
|
|
2ca5473e52 | ||
|
|
6070aa8281 | ||
|
|
78dc468555 | ||
|
|
bcd996f3df | ||
|
|
a1d959d0e9 | ||
|
|
71e97bccc7 | ||
|
|
3239b18671 | ||
|
|
c3794da4ed | ||
|
|
872a561495 | ||
|
|
d1adc8b15f | ||
|
|
e7d20b35ee | ||
|
|
21f566a60a | ||
|
|
7562e9ad5d | ||
|
|
0f16f7f6a2 | ||
|
|
dfa9fc05cb | ||
|
|
c4e855b58c | ||
|
|
0d9f636356 | ||
|
|
387f79be13 | ||
|
|
87f3e4a910 | ||
|
|
01ea635328 | ||
|
|
3d1bb8a4b4 | ||
|
|
046d1cb767 | ||
|
|
3d7b06e24c | ||
|
|
063b69db2f | ||
|
|
c0b0829d5a | ||
|
|
3cf42dd36d | ||
|
|
89e558ee4f | ||
|
|
f277ef7428 | ||
|
|
b7647a749e | ||
|
|
4cc6aeb274 | ||
|
|
3de903f51f | ||
|
|
4b4f49b1eb | ||
|
|
4407720b95 | ||
|
|
36dc939f91 | ||
|
|
40cd326ed7 | ||
|
|
a890d44d2f | ||
|
|
c98426505e | ||
|
|
6fa8039b55 | ||
|
|
faf4307fcc | ||
|
|
1512e41e41 | ||
|
|
55a25c5e0e | ||
|
|
f815731412 | ||
|
|
d607064fd2 | ||
|
|
8b0f876416 | ||
|
|
bbd8f410d9 | ||
|
|
fb3d809e81 | ||
|
|
95db494b30 | ||
|
|
5ebdb74ba8 | ||
|
|
07c20c5357 | ||
|
|
31daa8dad3 | ||
|
|
a0b03c3a68 | ||
|
|
fb7bb4f87a | ||
|
|
81e9fff605 | ||
|
|
e6baca2ffb | ||
|
|
2b28a0bc0a | ||
|
|
1f389f1422 | ||
|
|
8cdee40695 | ||
|
|
c6c6874d72 | ||
|
|
78c47074a3 | ||
|
|
22f249514d | ||
|
|
8f6826c9e3 | ||
|
|
1519c59188 | ||
|
|
a9176e9969 | ||
|
|
ab7dfb38ac | ||
|
|
aa21a21647 | ||
|
|
094bad431e | ||
|
|
a8ef2b8cfe | ||
|
|
636c911d33 | ||
|
|
53dd06057f | ||
|
|
1b840d9c79 | ||
|
|
012f051f2e | ||
|
|
8684cad808 | ||
|
|
15611d4559 | ||
|
|
d542e144dc | ||
|
|
f8fd159fbc | ||
|
|
2eb9570b52 | ||
|
|
f5ae4b42db | ||
|
|
24dd870559 | ||
|
|
97ef1b8393 | ||
|
|
bdb332a5f8 | ||
|
|
8cbdc28e66 | ||
|
|
4ebf080da8 | ||
|
|
53606261bf | ||
|
|
50d049d5f5 | ||
|
|
395391525b | ||
|
|
59f3f6d6c2 | ||
|
|
12cc536541 | ||
|
|
51ec421ad8 | ||
|
|
1cf095c51d | ||
|
|
7c9a261dc0 | ||
|
|
31367fbc97 | ||
|
|
7159f40304 | ||
|
|
042e307bef | ||
|
|
f584949470 | ||
|
|
303cd74e5f | ||
|
|
3b61abf20a | ||
|
|
d23f71b03c | ||
|
|
c156f5f3a1 | ||
|
|
155a8b241c | ||
|
|
69472d5f5c | ||
|
|
4ac3ee9d75 | ||
|
|
5b98c13eff | ||
|
|
28cb030ffe | ||
|
|
be1b075605 | ||
|
|
62cddb9e74 | ||
|
|
eabba5b316 | ||
|
|
a5e584ce22 | ||
|
|
d25732a53a | ||
|
|
701bd68731 | ||
|
|
51acd3f3ad | ||
|
|
804f9c2d4f | ||
|
|
81c0a17f20 | ||
|
|
2570d5866d | ||
|
|
2c0f4f588e | ||
|
|
8b5365e62c | ||
|
|
d837bcc60c | ||
|
|
bfe502654c | ||
|
|
6632d110b6 | ||
|
|
341c35b1b8 | ||
|
|
8e6084bc9c | ||
|
|
40a40613cb | ||
|
|
23a2745a3a | ||
|
|
88c6a0a752 | ||
|
|
1f62e3f142 | ||
|
|
8424e946e4 | ||
|
|
66a0448698 | ||
|
|
2cab359222 | ||
|
|
aebc0c3cdd | ||
|
|
c2925c40c7 | ||
|
|
88115dedbf | ||
|
|
1d8a881941 | ||
|
|
8439d7195f | ||
|
|
f0f8b2c85c | ||
|
|
35af42a0dd | ||
|
|
56a56527c6 | ||
|
|
1485e567f1 | ||
|
|
180c57747d | ||
|
|
2361a7aa24 | ||
|
|
4f9422ec85 | ||
|
|
9d17987b8c | ||
|
|
94056e51cc | ||
|
|
8cd9761d29 | ||
|
|
e86f4a201c | ||
|
|
9341404de0 | ||
|
|
b031b22c80 | ||
|
|
1440a070c3 | ||
|
|
14636b77fb | ||
|
|
da4160ce04 | ||
|
|
eccc3a26d2 | ||
|
|
51afcccb6b | ||
|
|
8f7d3efb1a | ||
|
|
13a818e3b5 | ||
|
|
5e049b9fb7 | ||
|
|
33d5038bd3 | ||
|
|
39bb76224d | ||
|
|
9632882b69 | ||
|
|
803598293e | ||
|
|
50068870a1 | ||
|
|
9a19439e31 | ||
|
|
d8dd1bc427 | ||
|
|
c1134be427 | ||
|
|
2aab3394cf | ||
|
|
39a5a37822 | ||
|
|
89879968bb | ||
|
|
6ecd671cfc | ||
|
|
2f50c9c52b | ||
|
|
386ce008d9 | ||
|
|
3d6c78875f | ||
|
|
a9bcd83ff0 | ||
|
|
bf3f0f133e | ||
|
|
2aa16f5f66 | ||
|
|
63f788f07e | ||
|
|
c3d61852ee | ||
|
|
2742b9c467 | ||
|
|
748ae8f06a | ||
|
|
28524d977c | ||
|
|
b5c2f0f2bb | ||
|
|
decee4abb2 | ||
|
|
8b4ce58a1d | ||
|
|
3682bcf9d2 | ||
|
|
857dc4b862 | ||
|
|
247ca90b33 | ||
|
|
00b22e527d | ||
|
|
83035ec925 | ||
|
|
035c07716c | ||
|
|
0d9689ab9f | ||
|
|
e696a861ba | ||
|
|
801e84d4d2 | ||
|
|
ff7e060269 | ||
|
|
d4d20346f2 | ||
|
|
1b48b97aa4 | ||
|
|
d1e665bac7 | ||
|
|
dc109fc118 | ||
|
|
fb8698d89a | ||
|
|
2821341551 | ||
|
|
bcdbe0f018 | ||
|
|
5f2a8b7f30 | ||
|
|
2831b4c0d8 | ||
|
|
c95cbb5931 | ||
|
|
da89656fdc | ||
|
|
4e5dc68b4e | ||
|
|
91ecd3cb6a | ||
|
|
448a36c5cb | ||
|
|
ed5a887579 | ||
|
|
e42058f15a | ||
|
|
a7e1cacd36 | ||
|
|
734803549d | ||
|
|
14f784961f | ||
|
|
5c8da71dea | ||
|
|
a6fe4b73b3 | ||
|
|
256e8f045b | ||
|
|
2e54bda0e1 | ||
|
|
2ae8e6291c | ||
|
|
651f78cba8 | ||
|
|
d5674ed411 | ||
|
|
4a2a810c20 | ||
|
|
365b0565de | ||
|
|
65d785d560 | ||
|
|
fff688c019 | ||
|
|
6f07ac731b | ||
|
|
f00c15d7b0 | ||
|
|
1676a902f2 | ||
|
|
576d323c52 | ||
|
|
b4adf71696 | ||
|
|
c0703e7a92 | ||
|
|
d789d50bd6 | ||
|
|
b83285bd59 | ||
|
|
bc93ed027f | ||
|
|
9f974b8996 | ||
|
|
c03b03c909 | ||
|
|
cecce18c51 | ||
|
|
1f37f41ee8 | ||
|
|
150ac7d708 | ||
|
|
be1ba9204a | ||
|
|
760bc3eb2d | ||
|
|
706f23115f | ||
|
|
171f9a256d | ||
|
|
e7b9b0d0ed | ||
|
|
5645143baa | ||
|
|
ec5c0ac3e0 | ||
|
|
511f9c1ffe | ||
|
|
6325327ba1 | ||
|
|
d60d590381 | ||
|
|
5aac011e49 | ||
|
|
320d573ccd | ||
|
|
c51bef38b6 | ||
|
|
28e0548cac | ||
|
|
d0acf6ecfe | ||
|
|
4b91f503eb | ||
|
|
eb02bc77fb | ||
|
|
55f54251eb | ||
|
|
e64b7cc233 | ||
|
|
4bb46a98c6 | ||
|
|
53b55288b3 | ||
|
|
f576a64f11 | ||
|
|
d574676702 | ||
|
|
f45821581c | ||
|
|
bb6ae0898d | ||
|
|
ea7fc641ba | ||
|
|
87e6482497 | ||
|
|
1e08e21470 | ||
|
|
ccb40ff3ff | ||
|
|
3291f3f891 | ||
|
|
2031939bfc | ||
|
|
6394299937 | ||
|
|
316f6c0cf3 | ||
|
|
53630f5ff1 | ||
|
|
3feee5958a | ||
|
|
209c056258 | ||
|
|
cd6e62049c | ||
|
|
526c0a918e | ||
|
|
de7ffa60cc | ||
|
|
81b9791178 | ||
|
|
c4ec6ac0f7 | ||
|
|
cfe3d2a83e | ||
|
|
b6bd4fe02d | ||
|
|
2da976155a | ||
|
|
e69e0de9c9 | ||
|
|
cfc9f10a6e | ||
|
|
8ba71b0754 | ||
|
|
46c16bb8f1 | ||
|
|
1b079a3cda | ||
|
|
c1b23fee45 | ||
|
|
199678e1bc | ||
|
|
d5daf6f6ae | ||
|
|
03d8d3233f | ||
|
|
5c6a21b014 | ||
|
|
3965df69d5 | ||
|
|
f47436e158 | ||
|
|
f474e4cae1 | ||
|
|
7979fdb832 | ||
|
|
b3f6bc47bb | ||
|
|
e6163082d7 | ||
|
|
fe1b5c06eb | ||
|
|
297a4227ef | ||
|
|
9f774d3e6a | ||
|
|
f7699f1dec | ||
|
|
3889c93b63 | ||
|
|
ee384ed8cc | ||
|
|
e92f038f40 | ||
|
|
0b3426bf5d | ||
|
|
0861ff957d | ||
|
|
32607ef715 | ||
|
|
c643ffd6d6 | ||
|
|
b8b3883950 | ||
|
|
3547fc54cb | ||
|
|
64c41a8af0 | ||
|
|
4191ac9cfe | ||
|
|
f1738e78db | ||
|
|
de494e4b98 | ||
|
|
585509f3bc | ||
|
|
f630dc5971 | ||
|
|
9a8f78ba3b | ||
|
|
63c6168c7f | ||
|
|
1993f0783b | ||
|
|
70238f448b | ||
|
|
914c38e379 | ||
|
|
0f28b01383 | ||
|
|
6909594c0b | ||
|
|
583921e8bf | ||
|
|
61ecf2c096 | ||
|
|
0eecb74230 | ||
|
|
b7b75f3c59 | ||
|
|
fe48ddfdcc | ||
|
|
70b23c3f84 | ||
|
|
223908f9d2 | ||
|
|
1838852d4d | ||
|
|
175550643c | ||
|
|
eb0eec4471 | ||
|
|
da6c505f41 | ||
|
|
966ba843e0 | ||
|
|
5773f75698 |
@@ -1,56 +0,0 @@
|
||||
# Golang CircleCI 2.0 configuration file
|
||||
#
|
||||
# Check https://circleci.com/docs/2.0/language-go/ for more details
|
||||
version: 2.1
|
||||
|
||||
jobs:
|
||||
"golang-1_15": &template
|
||||
machine:
|
||||
# https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
|
||||
image: ubuntu-2004:202010-01
|
||||
# docker_layer_caching: true
|
||||
|
||||
# https://circleci.com/docs/2.0/configuration-reference/#resource_class
|
||||
resource_class: medium
|
||||
|
||||
# Leave working directory unspecified and use defaults:
|
||||
# https://circleci.com/blog/go-v1.11-modules-and-circleci/
|
||||
# working_directory: /go/src/github.com/golang-migrate/migrate
|
||||
|
||||
environment:
|
||||
GO111MODULE: "on"
|
||||
GO_VERSION: "1.15.x"
|
||||
|
||||
steps:
|
||||
# - setup_remote_docker:
|
||||
# version: 19.03.13
|
||||
# docker_layer_caching: true
|
||||
- run: curl -sL -o ~/bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme
|
||||
- run: curl -sfL -o ~/bin/golangci-lint.sh https://install.goreleaser.com/github.com/golangci/golangci-lint.sh
|
||||
- run: chmod +x ~/bin/gimme ~/bin/golangci-lint.sh
|
||||
- run: eval "$(gimme $GO_VERSION)"
|
||||
- run: golangci-lint.sh -b ~/bin v1.37.0
|
||||
- checkout
|
||||
- restore_cache:
|
||||
keys:
|
||||
- go-mod-v1-{{ arch }}-{{ checksum "go.sum" }}
|
||||
- run: golangci-lint run
|
||||
- run: make test COVERAGE_DIR=/tmp/coverage
|
||||
- save_cache:
|
||||
key: go-mod-v1-{{ arch }}-{{ checksum "go.sum" }}
|
||||
paths:
|
||||
- "/go/pkg/mod"
|
||||
- run: go get github.com/mattn/goveralls
|
||||
- run: goveralls -service=circle-ci -coverprofile /tmp/coverage/combined.txt
|
||||
|
||||
"golang-1_16":
|
||||
<<: *template
|
||||
environment:
|
||||
GO_VERSION: "1.16.x"
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
build:
|
||||
jobs:
|
||||
- "golang-1_15"
|
||||
- "golang-1_16"
|
||||
@@ -2,6 +2,7 @@
|
||||
FAQ.md
|
||||
README.md
|
||||
LICENSE
|
||||
Makefile
|
||||
.gitignore
|
||||
.travis.yml
|
||||
CONTRIBUTING.md
|
||||
|
||||
@@ -25,7 +25,7 @@ e.g. s3, github, go-bindata, gcs, file
|
||||
Obtained by running: `migrate -help`
|
||||
|
||||
**Loaded Database Drivers**
|
||||
e.g. spanner, stub, datastore, cockroachdb, crdb-postgres, postgres, postgresql, pgx, redshift, cassandra, cockroach, mysql
|
||||
e.g. spanner, stub, clickhouse, cockroachdb, crdb-postgres, postgres, postgresql, redshift, cassandra, cockroach, mysql
|
||||
Obtained by running: `migrate -help`
|
||||
|
||||
**Go Version**
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1280" height="640" viewBox="0 0 1280 640" role="img" aria-label="migrate">
|
||||
<rect width="1280" height="640" fill="#0A0A0A"/>
|
||||
<svg x="96" y="215" width="210" height="210" viewBox="0 0 67 67"><path d="M22.21 67V44.6369H0V67H22.21Z" fill="#fff"/><path d="M66.7038 22.3184H22.2534L0.0878906 44.6367H44.4634L66.7038 22.3184Z" fill="#fff"/><path d="M22.21 0H0V22.3184H22.21V0Z" fill="#fff"/><path d="M66.7198 0H44.5098V22.3184H66.7198V0Z" fill="#fff"/><path d="M66.7198 67V44.6369H44.5098V67H66.7198Z" fill="#fff"/></svg>
|
||||
<text x="378" y="276" font-family="Inter,system-ui,-apple-system,sans-serif" font-size="78" font-weight="800" letter-spacing="-2" fill="#ffffff">migrate</text>
|
||||
<text x="378" y="322" font-family="Inter,system-ui,sans-serif" font-size="30" fill="#ffffff" opacity=".66">Database migrations. CLI and Golang library.</text>
|
||||
<rect x="378" y="338" width="806" height="3" rx="1.5" fill="#ffffff" opacity=".9"/>
|
||||
<text x="378" y="390" font-family="Inter,system-ui,sans-serif" font-size="24" font-weight="600" fill="#ffffff" opacity=".5">github.com/hanzoai</text>
|
||||
<text x="1184" y="390" text-anchor="end" font-family="Inter,system-ui,sans-serif" font-size="24" font-weight="600" fill="#ffffff" opacity=".5">hanzo.ai</text>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
+1
-3
@@ -6,6 +6,4 @@ cli/migrate
|
||||
.godoc.pid
|
||||
vendor/
|
||||
.vscode/
|
||||
.idea
|
||||
dist/
|
||||
migrate
|
||||
.idea/
|
||||
+20
-30
@@ -1,37 +1,27 @@
|
||||
version: "2"
|
||||
run:
|
||||
# timeout for analysis, e.g. 30s, 5m, default is 1m
|
||||
timeout: 2m
|
||||
linters:
|
||||
enable:
|
||||
#- golint
|
||||
- interfacer
|
||||
- unconvert
|
||||
#- dupl
|
||||
- goconst
|
||||
- gofmt
|
||||
- misspell
|
||||
- maligned
|
||||
- unparam
|
||||
- nakedret
|
||||
- prealloc
|
||||
- revive
|
||||
- unconvert
|
||||
- unparam
|
||||
settings:
|
||||
misspell:
|
||||
locale: US
|
||||
revive:
|
||||
rules:
|
||||
- name: redundant-build-tag
|
||||
exclusions:
|
||||
generated: lax
|
||||
rules:
|
||||
- path: (.+)\.go$
|
||||
text: G104
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
#- gosec
|
||||
linters-settings:
|
||||
misspell:
|
||||
locale: US
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
max-same: 0
|
||||
max-per-linter: 0
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
# gosec: Duplicated errcheck checks
|
||||
- G104
|
||||
|
||||
-102
@@ -1,102 +0,0 @@
|
||||
project_name: migrate
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy
|
||||
builds:
|
||||
- env:
|
||||
- CGO_ENABLED=0
|
||||
goos:
|
||||
- linux
|
||||
- windows
|
||||
- darwin
|
||||
goarch:
|
||||
- amd64
|
||||
- arm
|
||||
- arm64
|
||||
- 386
|
||||
goarm:
|
||||
- 7
|
||||
main: ./cmd/migrate
|
||||
ldflags:
|
||||
- '-w -s -X main.Version={{ .Version }} -extldflags "static"'
|
||||
flags:
|
||||
- "-tags={{ .Env.DATABASE }} {{ .Env.SOURCE }}"
|
||||
- "-trimpath"
|
||||
nfpms:
|
||||
- homepage: "https://github.com/golang-migrate/migrate"
|
||||
maintainer: "dhui@users.noreply.github.com"
|
||||
license: MIT
|
||||
description: "Database migrations"
|
||||
formats:
|
||||
- deb
|
||||
file_name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
|
||||
dockers:
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
dockerfile: Dockerfile.github-actions
|
||||
use: buildx
|
||||
ids:
|
||||
- migrate
|
||||
image_templates:
|
||||
- 'migrate/migrate:{{ .Tag }}-amd64'
|
||||
build_flag_templates:
|
||||
- '--label=org.opencontainers.image.created={{ .Date }}'
|
||||
- '--label=org.opencontainers.image.title={{ .ProjectName }}'
|
||||
- '--label=org.opencontainers.image.revision={{ .FullCommit }}'
|
||||
- '--label=org.opencontainers.image.version={{ .Version }}'
|
||||
- "--label=org.opencontainers.image.source={{ .GitURL }}"
|
||||
- "--platform=linux/amd64"
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
dockerfile: Dockerfile.github-actions
|
||||
use: buildx
|
||||
ids:
|
||||
- migrate
|
||||
image_templates:
|
||||
- 'migrate/migrate:{{ .Tag }}-arm64'
|
||||
build_flag_templates:
|
||||
- '--label=org.opencontainers.image.created={{ .Date }}'
|
||||
- '--label=org.opencontainers.image.title={{ .ProjectName }}'
|
||||
- '--label=org.opencontainers.image.revision={{ .FullCommit }}'
|
||||
- '--label=org.opencontainers.image.version={{ .Version }}'
|
||||
- "--label=org.opencontainers.image.source={{ .GitURL }}"
|
||||
- "--platform=linux/arm64"
|
||||
|
||||
docker_manifests:
|
||||
- name_template: 'migrate/migrate:{{ .Tag }}'
|
||||
image_templates:
|
||||
- 'migrate/migrate:{{ .Tag }}-amd64'
|
||||
- 'migrate/migrate:{{ .Tag }}-arm64'
|
||||
- name_template: 'migrate/migrate:{{ .Major }}'
|
||||
image_templates:
|
||||
- 'migrate/migrate:{{ .Tag }}-amd64'
|
||||
- 'migrate/migrate:{{ .Tag }}-arm64'
|
||||
- name_template: 'migrate/migrate:latest'
|
||||
image_templates:
|
||||
- 'migrate/migrate:{{ .Tag }}-amd64'
|
||||
- 'migrate/migrate:{{ .Tag }}-arm64'
|
||||
archives:
|
||||
- name_template: "{{ .ProjectName }}.{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
format: zip
|
||||
checksum:
|
||||
name_template: 'sha256sum.txt'
|
||||
release:
|
||||
draft: true
|
||||
prerelease: auto
|
||||
source:
|
||||
enabled: true
|
||||
format: zip
|
||||
changelog:
|
||||
skip: false
|
||||
sort: asc
|
||||
filters:
|
||||
exclude:
|
||||
- '^docs:'
|
||||
- '^test:'
|
||||
- Merge pull request
|
||||
- Merge branch
|
||||
- go mod tidy
|
||||
snapshot:
|
||||
name_template: "{{ .Tag }}-next"
|
||||
+9
-9
@@ -6,8 +6,8 @@ matrix:
|
||||
- go: master
|
||||
include:
|
||||
# Supported versions of Go: https://golang.org/dl/
|
||||
- go: "1.13.x"
|
||||
- go: "1.14.x"
|
||||
- go: "1.15.x"
|
||||
- go: master
|
||||
|
||||
go_import_path: github.com/golang-migrate/migrate
|
||||
@@ -34,7 +34,7 @@ before_install:
|
||||
- sudo apt-get update
|
||||
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
|
||||
# Install golangci-lint
|
||||
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0
|
||||
- curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0
|
||||
- echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
|
||||
|
||||
install:
|
||||
@@ -57,7 +57,7 @@ deploy:
|
||||
secure: hWH1HLPpzpfA8pXQ93T1qKQVFSpQp0as/JLQ7D91jHuJ8p+RxVeqblDrR6HQY/95R/nyiE9GJmvUolSuw5h449LSrGxPtVWhdh6EnkxlQHlen5XeMhVjRjFV0sE9qGe8v7uAkiTfRO61ktTWHrEAvw5qpyqnNISodmZS78XIasPODQbNlzwINhWhDTHIjXGb4FpizYaL3OGCanrxfR9fQyCaqKGGBjRq3Mfq8U6Yd4mApmsE+uJxgaZV8K5zBqpkSzQRWhcVGNL5DuLsU3gfSJOo7kZeA2G71SHffH577dBoqtCZ4VFv169CoUZehLWCb+7XKJZmHXVujCURATSySLGUOPc6EoLFAn3YtsCA04mS4bZVo5FZPWVwfhjmkhtDR4f6wscKp7r1HsFHSOgm59QfETQdrn4MnZ44H2Jd39axqndn5DvK9EcZVjPHynOPnueXP2u6mTuUgh2VyyWBCDO3CNo0fGlo7VJI69IkIWNSD87K9cHZWYMClyKZkUzS+PmRAhHRYbVd+9ZjKOmnU36kUHNDG/ft1D4ogsY+rhVtXB4lgWDM5adri+EIScYdYnB1/pQexLBigcJY9uE7nQTR0U6QgVNYvun7uRNs40E0c4voSfmPdFO0FlOD2y1oQhnaXfWLbu9nMcTcs4RFGrcC7NzkUN4/WjG8s285V6w=
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
file:
|
||||
@@ -78,7 +78,7 @@ deploy:
|
||||
package_glob: '*.deb'
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
- provider: packagecloud
|
||||
@@ -90,7 +90,7 @@ deploy:
|
||||
package_glob: '*.deb'
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
- provider: packagecloud
|
||||
@@ -102,7 +102,7 @@ deploy:
|
||||
package_glob: '*.deb'
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
- provider: packagecloud
|
||||
@@ -114,7 +114,7 @@ deploy:
|
||||
package_glob: '*.deb'
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
- provider: packagecloud
|
||||
@@ -126,13 +126,13 @@ deploy:
|
||||
package_glob: '*.deb'
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
- provider: script
|
||||
script: ./docker-deploy.sh
|
||||
skip_cleanup: true
|
||||
on:
|
||||
go: "1.15.x"
|
||||
go: "1.14.x"
|
||||
repo: golang-migrate/migrate
|
||||
tags: true
|
||||
|
||||
+6
-4
@@ -1,11 +1,13 @@
|
||||
FROM golang:1.26.4-alpine3.21 AS builder
|
||||
FROM golang:1.14-alpine3.11 AS builder
|
||||
ARG VERSION
|
||||
|
||||
RUN apk add --no-cache git gcc musl-dev make
|
||||
RUN apk add --no-cache git gcc musl-dev
|
||||
|
||||
WORKDIR /go/src/github.com/golang-migrate/migrate
|
||||
|
||||
ENV GO111MODULE=on
|
||||
ENV DATABASES="postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird sqlite3 neo4j"
|
||||
ENV SOURCES="file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab"
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
|
||||
@@ -13,9 +15,9 @@ RUN go mod download
|
||||
|
||||
COPY . ./
|
||||
|
||||
RUN make build-docker
|
||||
RUN go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$DATABASES $SOURCES" ./cmd/migrate
|
||||
|
||||
FROM alpine:3.21
|
||||
FROM alpine:3.11
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
ARG DOCKER_IMAGE
|
||||
FROM $DOCKER_IMAGE
|
||||
|
||||
RUN apk add --no-cache git gcc musl-dev make
|
||||
|
||||
WORKDIR /go/src/github.com/golang-migrate/migrate
|
||||
|
||||
ENV GO111MODULE=on
|
||||
ENV COVERAGE_DIR=/tmp/coverage
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
|
||||
RUN go mod download
|
||||
|
||||
COPY . ./
|
||||
|
||||
CMD ["make", "test"]
|
||||
@@ -1,11 +0,0 @@
|
||||
FROM alpine:3.19
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
COPY migrate /usr/local/bin/migrate
|
||||
|
||||
RUN ln -s /usr/local/bin/migrate /usr/bin/migrate
|
||||
RUN ln -s /usr/local/bin/migrate /migrate
|
||||
|
||||
ENTRYPOINT ["migrate"]
|
||||
CMD ["--help"]
|
||||
@@ -16,7 +16,7 @@
|
||||
NilMigration defines a migration without a body. NilVersion is defined as const -1.
|
||||
|
||||
#### What is the difference between uint(version) and int(targetVersion)?
|
||||
version refers to an existing migration version coming from a source and therefore can never be negative.
|
||||
version refers to an existing migration version coming from a source and therefor can never be negative.
|
||||
targetVersion can either be a version OR represent a NilVersion, which equals -1.
|
||||
|
||||
#### What's the difference between Next/Previous and Up/Down?
|
||||
@@ -50,10 +50,10 @@
|
||||
and whenever we want, not just once at the beginning of all tests.
|
||||
|
||||
#### Can I maintain my driver in my own repository?
|
||||
Yes, technically thats possible. We want to encourage you to contribute your driver to this repository though.
|
||||
Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
|
||||
The driver's functionality is dictated by migrate's interfaces. That means there should really
|
||||
just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
|
||||
just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the
|
||||
just implemented a bit differently, co-exist somewhere on Github. If users have to do research first to find the
|
||||
"best" available driver for a database in order to get started, we would have failed as an open source community.
|
||||
|
||||
#### Can I mix multiple sources during a batch of migrations?
|
||||
@@ -65,7 +65,7 @@
|
||||
and then "force" the expected version.
|
||||
|
||||
#### What happens if two programs try and update the database at the same time?
|
||||
Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations on
|
||||
Database-specific locking features are used by *some* database drivers to prevent multiple instances of migrate from running migrations at the same time
|
||||
the same database at the same time. For example, the MySQL driver uses the `GET_LOCK` function, while the Postgres driver uses
|
||||
the `pg_advisory_lock` function.
|
||||
|
||||
@@ -76,4 +76,4 @@ No, it is done automatically.
|
||||
Yes, you can use the migrate CLI in a non-Go project, but there are probably other libraries/frameworks available that offer better test and deploy integrations in that language/framework.
|
||||
|
||||
#### I have got an error `Dirty database version 1. Fix and force version`. What should I do?
|
||||
Keep calm and refer to [the getting started docs](GETTING_STARTED.md#forcing-your-database-version).
|
||||
Keep calm and refer to [the getting started docs](GETTING_STARTED.md#forcing-your-database-version).
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
# Getting started
|
||||
Before you start, you should understand the concept of forward/up and reverse/down database migrations.
|
||||
|
||||
Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases).
|
||||
Configure a database for your application. Make sure that your database driver is supported [here](README.md#databases)
|
||||
|
||||
## Create migrations
|
||||
Create some migrations using migrate CLI. Here is an example:
|
||||
@@ -10,7 +10,7 @@ migrate create -ext sql -dir db/migrations -seq create_users_table
|
||||
```
|
||||
Once you create your files, you should fill them.
|
||||
|
||||
**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created their migration later gets it merged to the repository first.
|
||||
**IMPORTANT:** In a project developed by more than one person there is a chance of migrations inconsistency - e.g. two developers can create conflicting migrations, and the developer that created his migration later gets it merged to the repository first.
|
||||
Developers and Teams should keep an eye on such cases (especially during code review).
|
||||
[Here](https://github.com/golang-migrate/migrate/issues/179#issuecomment-475821264) is the issue summary if you would like to read more.
|
||||
|
||||
@@ -28,9 +28,9 @@ migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up
|
||||
|
||||
Just add the code to your app and you're ready to go!
|
||||
|
||||
Before committing your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
|
||||
Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
|
||||
(e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
|
||||
It's also worth checking your migrations in a separate, containerized environment. You can find some tools at the [end of this document](#further-reading).
|
||||
It's also worth checking your migrations in a separate, containerized environment. You can find some tools in the end of this document.
|
||||
|
||||
**IMPORTANT:** If you would like to run multiple instances of your app on different machines be sure to use a database that supports locking when running migrations. Otherwise you may encounter issues.
|
||||
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
# Hanzo Golang Migrate
|
||||
|
||||
## Overview
|
||||
Go module: github.com/golang-migrate/migrate/v4
|
||||
|
||||
## Tech Stack
|
||||
- **Language**: Go
|
||||
|
||||
## Build & Run
|
||||
```bash
|
||||
go build ./...
|
||||
go test ./...
|
||||
```
|
||||
|
||||
## Structure
|
||||
```
|
||||
golang-migrate/
|
||||
CONTRIBUTING.md
|
||||
Dockerfile
|
||||
Dockerfile.circleci
|
||||
Dockerfile.github-actions
|
||||
FAQ.md
|
||||
GETTING_STARTED.md
|
||||
LICENSE
|
||||
MIGRATIONS.md
|
||||
Makefile
|
||||
README.md
|
||||
SECURITY.md
|
||||
cli/
|
||||
cmd/
|
||||
database/
|
||||
dktesting/
|
||||
```
|
||||
|
||||
## Key Files
|
||||
- `README.md` -- Project documentation
|
||||
- `go.mod` -- Go module definition
|
||||
- `Makefile` -- Build automation
|
||||
- `Dockerfile` -- Container build
|
||||
@@ -1,16 +1,11 @@
|
||||
SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
|
||||
DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb datastore mongodb sqlserver firebird neo4j pgx pgx5 rqlite
|
||||
DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
|
||||
SOURCE ?= file go_bindata github github_ee aws_s3 google_cloud_storage godoc_vfs gitlab
|
||||
DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb clickhouse mongodb sqlserver firebird neo4j
|
||||
DATABASE_TEST ?= $(DATABASE) sqlite
|
||||
VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
|
||||
TEST_FLAGS ?=
|
||||
REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)")
|
||||
COVERAGE_DIR ?= .coverage
|
||||
|
||||
build:
|
||||
CGO_ENABLED=0 go build -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' ./cmd/migrate
|
||||
|
||||
build-docker:
|
||||
CGO_ENABLED=0 go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$(DATABASE) $(SOURCE)" ./cmd/migrate
|
||||
|
||||
build-cli: clean
|
||||
-mkdir ./cli/build
|
||||
@@ -98,12 +93,6 @@ release:
|
||||
git tag v$(V)
|
||||
@read -p "Press enter to confirm and push to origin ..." && git push origin v$(V)
|
||||
|
||||
echo-source:
|
||||
@echo "$(SOURCE)"
|
||||
|
||||
echo-database:
|
||||
@echo "$(DATABASE)"
|
||||
|
||||
|
||||
define external_deps
|
||||
@echo '-- $(1)'; go list -f '{{join .Deps "\n"}}' $(1) | grep -v github.com/$(REPO_OWNER)/migrate | xargs go list -f '{{if not .Standard}}{{.ImportPath}}{{end}}'
|
||||
@@ -111,10 +100,10 @@ define external_deps
|
||||
endef
|
||||
|
||||
|
||||
.PHONY: build build-docker build-cli clean test-short test test-with-flags html-coverage \
|
||||
.PHONY: build-cli clean test-short test test-with-flags html-coverage \
|
||||
restore-import-paths rewrite-import-paths list-external-deps release \
|
||||
docs kill-docs open-docs kill-orphaned-docker-containers echo-source echo-database
|
||||
docs kill-docs open-docs kill-orphaned-docker-containers
|
||||
|
||||
SHELL = /bin/sh
|
||||
SHELL = /bin/bash
|
||||
RAND = $(shell echo $$RANDOM)
|
||||
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
<p align="center"><img src=".github/hero.svg" alt="migrate" width="880"></p>
|
||||
|
||||
[](https://github.com/golang-migrate/migrate/actions/workflows/ci.yaml?query=branch%3Amaster)
|
||||
[](https://pkg.go.dev/github.com/golang-migrate/migrate/v4)
|
||||
[](https://travis-ci.com/golang-migrate/migrate)
|
||||
[](https://godoc.org/github.com/golang-migrate/migrate)
|
||||
[](https://coveralls.io/github/golang-migrate/migrate?branch=master)
|
||||
[](https://packagecloud.io/golang-migrate/migrate?filter=debs)
|
||||
[](https://hub.docker.com/r/migrate/migrate/)
|
||||

|
||||

|
||||
[](https://github.com/golang-migrate/migrate/releases)
|
||||
[](https://goreportcard.com/report/github.com/golang-migrate/migrate/v4)
|
||||
[](https://goreportcard.com/report/github.com/golang-migrate/migrate)
|
||||
|
||||
# migrate
|
||||
|
||||
@@ -26,30 +24,24 @@ Forked from [mattes/migrate](https://github.com/mattes/migrate)
|
||||
Database drivers run migrations. [Add a new database?](database/driver.go)
|
||||
|
||||
* [PostgreSQL](database/postgres)
|
||||
* [PGX v4](database/pgx)
|
||||
* [PGX v5](database/pgx/v5)
|
||||
* [Redshift](database/redshift)
|
||||
* [Ql](database/ql)
|
||||
* [Cassandra / ScyllaDB](database/cassandra)
|
||||
* [SQLite](database/sqlite)
|
||||
* [SQLite3](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
|
||||
* [SQLCipher](database/sqlcipher)
|
||||
* [MySQL / MariaDB](database/mysql)
|
||||
* [Cassandra](database/cassandra)
|
||||
* [SQLite](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
|
||||
* [MySQL/ MariaDB](database/mysql)
|
||||
* [Neo4j](database/neo4j)
|
||||
* [MongoDB](database/mongodb)
|
||||
* [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
|
||||
* [Shell](database/shell) ([todo #171](https://github.com/mattes/migrate/issues/171))
|
||||
* [Google Cloud Spanner](database/spanner)
|
||||
* [CockroachDB](database/cockroachdb)
|
||||
* [YugabyteDB](database/yugabytedb)
|
||||
* [Datastore](database/datastore)
|
||||
* [ClickHouse](database/clickhouse)
|
||||
* [Firebird](database/firebird)
|
||||
* [MS SQL Server](database/sqlserver)
|
||||
* [rqlite](database/rqlite)
|
||||
|
||||
### Database URLs
|
||||
|
||||
Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?param1=true¶m2=false`
|
||||
Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?option1=true&option2=false`
|
||||
|
||||
Any [reserved URL characters](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_reserved_characters) need to be escaped. Note, the `%` character also [needs to be escaped](https://en.wikipedia.org/wiki/Percent-encoding#Percent-encoding_the_percent_character)
|
||||
|
||||
@@ -73,12 +65,9 @@ $
|
||||
Source drivers read migrations from local or remote sources. [Add a new source?](source/driver.go)
|
||||
|
||||
* [Filesystem](source/file) - read from filesystem
|
||||
* [io/fs](source/iofs) - read from a Go [io/fs](https://pkg.go.dev/io/fs#FS)
|
||||
* [Go-Bindata](source/go_bindata) - read from embedded binary data ([jteeuwen/go-bindata](https://github.com/jteeuwen/go-bindata))
|
||||
* [pkger](source/pkger) - read from embedded binary data ([markbates/pkger](https://github.com/markbates/pkger))
|
||||
* [GitHub](source/github) - read from remote GitHub repositories
|
||||
* [GitHub Enterprise](source/github_ee) - read from remote GitHub Enterprise repositories
|
||||
* [Bitbucket](source/bitbucket) - read from remote Bitbucket repositories
|
||||
* [Github](source/github) - read from remote Github repositories
|
||||
* [Github Enterprise](source/github_ee) - read from remote Github Enterprise repositories
|
||||
* [Gitlab](source/gitlab) - read from remote Gitlab repositories
|
||||
* [AWS S3](source/aws_s3) - read from Amazon Web Services S3
|
||||
* [Google Cloud Storage](source/google_cloud_storage) - read from Google Cloud Platform Storage
|
||||
@@ -89,7 +78,7 @@ Source drivers read migrations from local or remote sources. [Add a new source?]
|
||||
* Handles ctrl+c (SIGINT) gracefully.
|
||||
* No config search paths, no config files, no magic ENV var injections.
|
||||
|
||||
[CLI Documentation](cmd/migrate) (includes CLI install instructions)
|
||||
__[CLI Documentation](cmd/migrate)__
|
||||
|
||||
### Basic usage
|
||||
|
||||
@@ -113,7 +102,7 @@ $ docker run -v {{ migration dir }}:/migrations --network host migrate/migrate
|
||||
* Uses `io.Reader` streams internally for low memory overhead.
|
||||
* Thread-safe and no goroutine leaks.
|
||||
|
||||
__[Go Documentation](https://pkg.go.dev/github.com/golang-migrate/migrate/v4)__
|
||||
__[Go Documentation](https://godoc.org/github.com/golang-migrate/migrate)__
|
||||
|
||||
```go
|
||||
import (
|
||||
@@ -147,7 +136,7 @@ func main() {
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
"file:///migrations",
|
||||
"postgres", driver)
|
||||
m.Up() // or m.Steps(2) if you want to explicitly set the number of migrations to run
|
||||
m.Steps(2)
|
||||
}
|
||||
```
|
||||
|
||||
@@ -173,11 +162,6 @@ Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-file
|
||||
|
||||
[Best practices: How to write migrations.](MIGRATIONS.md)
|
||||
|
||||
## Coming from another db migration tool?
|
||||
|
||||
Check out [migradaptor](https://github.com/musinit/migradaptor/).
|
||||
*Note: migradaptor is not affiliated or supported by this project*
|
||||
|
||||
## Versions
|
||||
|
||||
Version | Supported? | Import | Notes
|
||||
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Supported Versions
|
||||
|
||||
| Version | Supported |
|
||||
| ------- | ------------------ |
|
||||
| master | :white_check_mark: |
|
||||
| 4.x | :white_check_mark: |
|
||||
| 3.x | :x: |
|
||||
| < 3.0 | :x: |
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
We prefer [coordinated disclosures](https://en.wikipedia.org/wiki/Coordinated_vulnerability_disclosure). To start one, create a GitHub security advisory following [these instructions](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability)
|
||||
|
||||
Please suggest potential impact and urgency in your reports.
|
||||
+11
-22
@@ -7,7 +7,7 @@
|
||||
[Release Downloads](https://github.com/golang-migrate/migrate/releases)
|
||||
|
||||
```bash
|
||||
$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$os-$arch.tar.gz | tar xvz
|
||||
$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz
|
||||
```
|
||||
|
||||
### MacOS
|
||||
@@ -27,8 +27,8 @@ $ scoop install migrate
|
||||
### Linux (*.deb package)
|
||||
|
||||
```bash
|
||||
$ curl -fsSL https://packagecloud.io/golang-migrate/migrate/gpgkey | sudo gpg --dearmor -o /etc/apt/keyrings/migrate.gpg
|
||||
$ echo "deb [signed-by=/etc/apt/keyrings/migrate.gpg] https://packagecloud.io/golang-migrate/migrate/ubuntu/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/migrate.list
|
||||
$ curl -L https://packagecloud.io/golang-migrate/migrate/gpgkey | apt-key add -
|
||||
$ echo "deb https://packagecloud.io/golang-migrate/migrate/ubuntu/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/migrate.list
|
||||
$ apt-get update
|
||||
$ apt-get install -y migrate
|
||||
```
|
||||
@@ -41,19 +41,13 @@ $ apt-get install -y migrate
|
||||
$ go get -u -d github.com/golang-migrate/migrate/cmd/migrate
|
||||
$ cd $GOPATH/src/github.com/golang-migrate/migrate/cmd/migrate
|
||||
$ git checkout $TAG # e.g. v4.1.0
|
||||
$ # Go 1.15 and below
|
||||
$ go build -tags 'postgres' -ldflags="-X main.Version=$(git describe --tags)" -o $GOPATH/bin/migrate $GOPATH/src/github.com/golang-migrate/migrate/cmd/migrate
|
||||
$ # Go 1.16+
|
||||
$ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@$TAG
|
||||
```
|
||||
|
||||
#### Unversioned
|
||||
|
||||
```bash
|
||||
$ # Go 1.15 and below
|
||||
$ go get -tags 'postgres' -u github.com/golang-migrate/migrate/cmd/migrate
|
||||
$ # Go 1.16+
|
||||
$ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
|
||||
```
|
||||
|
||||
#### Notes
|
||||
@@ -63,10 +57,9 @@ $ go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@l
|
||||
to build the cli for use with other databases, replace the `postgres` build tag
|
||||
with the appropriate database tag(s) for the databases desired. The tags
|
||||
correspond to the names of the sub-packages underneath the
|
||||
[`database`](../../database) package.
|
||||
[`database`](../database) package.
|
||||
1. Similarly to the database build tags, if you need to support other sources, use the appropriate build tag(s).
|
||||
1. Support for build constraints will be removed in the future: https://github.com/golang-migrate/migrate/issues/60
|
||||
1. For versions of Go 1.15 and lower, [make sure](https://github.com/golang-migrate/migrate/pull/257#issuecomment-705249902) you're not installing the `migrate` CLI from a module. e.g. there should not be any `go.mod` files in your current directory or any directory from your current directory to the root
|
||||
|
||||
## Usage
|
||||
|
||||
@@ -86,18 +79,14 @@ Options:
|
||||
-help Print usage
|
||||
|
||||
Commands:
|
||||
create [-ext E] [-dir D] [-seq] [-digits N] [-format] [-tz] NAME
|
||||
Create a set of timestamped up/down migrations titled NAME, in directory D with extension E.
|
||||
Use -seq option to generate sequential up/down migrations with N digits.
|
||||
Use -format option to specify a Go time format string. Note: migrations with the same time cause "duplicate migration version" error.
|
||||
Use -tz option to specify the timezone that will be used when generating non-sequential migrations (defaults: UTC).
|
||||
|
||||
create [-ext E] [-dir D] [-seq] [-digits N] [-format] NAME
|
||||
Create a set of timestamped up/down migrations titled NAME, in directory D with extension E.
|
||||
Use -seq option to generate sequential up/down migrations with N digits.
|
||||
Use -format option to specify a Go time format string.
|
||||
goto V Migrate to version V
|
||||
up [N] Apply all or N up migrations
|
||||
down [N] [-all] Apply all or N down migrations
|
||||
Use -all to apply all down migrations
|
||||
drop [-f] Drop everything inside database
|
||||
Use -f to bypass confirmation
|
||||
down [N] Apply all or N down migrations
|
||||
drop Drop everything inside database
|
||||
force V Set version V but don't run migration (ignores dirty state)
|
||||
version Print current migration version
|
||||
```
|
||||
@@ -131,7 +120,7 @@ $ migrate -database "$MY_MIGRATE_DATABASE"
|
||||
Check out https://stedolan.github.io/jq/
|
||||
|
||||
```bash
|
||||
$ migrate -database "$(cat config.json | jq -r '.database')"
|
||||
$ migrate -database "$(cat config.json | jq '.database')"
|
||||
```
|
||||
|
||||
### YAML files
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
# Cassandra / ScyllaDB
|
||||
# Cassandra
|
||||
|
||||
* `Drop()` method will not work on Cassandra 2.X because it rely on
|
||||
* Drop command will not work on Cassandra 2.X because it rely on
|
||||
system_schema table which comes with 3.X
|
||||
* Other methods should work properly but are **not tested**
|
||||
* Other commands should work properly but are **not tested**
|
||||
* The Cassandra driver (gocql) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
|
||||
* This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
|
||||
* The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
|
||||
|
||||
**ScyllaDB**
|
||||
|
||||
* No additional configuration is required since it is a drop-in replacement for Cassandra.
|
||||
* The `Drop()` method` works for ScyllaDB 5.1
|
||||
|
||||
|
||||
## Usage
|
||||
`cassandra://host:port/keyspace?param1=value¶m2=value2`
|
||||
@@ -25,14 +20,12 @@ system_schema table which comes with 3.X
|
||||
| `consistency` | ALL | Migration consistency
|
||||
| `protocol` | | Cassandra protocol version (3 or 4)
|
||||
| `timeout` | 1 minute | Migration timeout
|
||||
| `connect-timeout` | 600ms | Initial connection timeout to the cluster |
|
||||
| `username` | nil | Username to use when authenticating. |
|
||||
| `password` | nil | Password to use when authenticating. |
|
||||
| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
|
||||
| `sslkey` | | Key file location. The file must contain PEM encoded data. |
|
||||
| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
|
||||
| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
|
||||
| `disable-host-lookup`| false | Disable initial host lookup. |
|
||||
|
||||
`timeout` is parsed using [time.ParseDuration(s string)](https://golang.org/pkg/time/#ParseDuration)
|
||||
|
||||
|
||||
@@ -4,15 +4,16 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/gocql/gocql"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -44,7 +45,7 @@ type Config struct {
|
||||
|
||||
type Cassandra struct {
|
||||
session *gocql.Session
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
|
||||
// Open and WithInstance need to guarantee that config is never nil
|
||||
config *Config
|
||||
@@ -131,14 +132,6 @@ func (c *Cassandra) Open(url string) (database.Driver, error) {
|
||||
}
|
||||
cluster.Timeout = timeout
|
||||
}
|
||||
if len(u.Query().Get("connect-timeout")) > 0 {
|
||||
var connectTimeout time.Duration
|
||||
connectTimeout, err = time.ParseDuration(u.Query().Get("connect-timeout"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cluster.ConnectTimeout = connectTimeout
|
||||
}
|
||||
|
||||
if len(u.Query().Get("sslmode")) > 0 {
|
||||
if u.Query().Get("sslmode") != "disable" {
|
||||
@@ -162,14 +155,6 @@ func (c *Cassandra) Open(url string) (database.Driver, error) {
|
||||
}
|
||||
}
|
||||
|
||||
if len(u.Query().Get("disable-host-lookup")) > 0 {
|
||||
if flag, err := strconv.ParseBool(u.Query().Get("disable-host-lookup")); err != nil && flag {
|
||||
cluster.DisableInitialHostLookup = true
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
session, err := cluster.CreateSession()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -197,16 +182,15 @@ func (c *Cassandra) Close() error {
|
||||
}
|
||||
|
||||
func (c *Cassandra) Lock() error {
|
||||
if !c.isLocked.CompareAndSwap(false, true) {
|
||||
if c.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
c.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cassandra) Unlock() error {
|
||||
if !c.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
c.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -229,7 +213,7 @@ func (c *Cassandra) Run(migration io.Reader) error {
|
||||
return err
|
||||
}
|
||||
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -242,26 +226,16 @@ func (c *Cassandra) Run(migration io.Reader) error {
|
||||
}
|
||||
|
||||
func (c *Cassandra) SetVersion(version int, dirty bool) error {
|
||||
// DELETE instead of TRUNCATE because AWS Keyspaces does not support it
|
||||
// see: https://docs.aws.amazon.com/keyspaces/latest/devguide/cassandra-apis.html
|
||||
squery := `SELECT version FROM "` + c.config.MigrationsTable + `"`
|
||||
dquery := `DELETE FROM "` + c.config.MigrationsTable + `" WHERE version = ?`
|
||||
iter := c.session.Query(squery).Iter()
|
||||
var previous int
|
||||
for iter.Scan(&previous) {
|
||||
if err := c.session.Query(dquery, previous).Exec(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(dquery)}
|
||||
}
|
||||
}
|
||||
if err := iter.Close(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(squery)}
|
||||
query := `TRUNCATE "` + c.config.MigrationsTable + `"`
|
||||
if err := c.session.Query(query).Exec(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// Also re-write the schema version for nil dirty versions to prevent
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
query := `INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)`
|
||||
query = `INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)`
|
||||
if err := c.session.Query(query, version, dirty).Exec(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -314,7 +288,11 @@ func (c *Cassandra) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := c.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -336,7 +314,7 @@ func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err
|
||||
var ok bool
|
||||
err, ok = r.(error)
|
||||
if !ok {
|
||||
err = fmt.Errorf("failed to parse consistency \"%s\": %v", consistencyStr, r)
|
||||
err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -25,11 +25,9 @@ var (
|
||||
// Although Cassandra 2.x is supported by the Apache Foundation,
|
||||
// the migrate db driver only supports Cassandra 3.x since it uses
|
||||
// the system_schema keyspace.
|
||||
// last ScyllaDB version tested is 5.1.11
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "cassandra:3.0", Options: opts},
|
||||
{ImageName: "cassandra:3.11", Options: opts},
|
||||
{ImageName: "scylladb/scylla:5.1.11", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -61,20 +59,6 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
t.Run("test", test)
|
||||
t.Run("testMigrate", testMigrate)
|
||||
|
||||
t.Cleanup(func() {
|
||||
for _, spec := range specs {
|
||||
t.Log("Cleaning up ", spec.ImageName)
|
||||
if err := spec.Cleanup(); err != nil {
|
||||
t.Error("Error removing ", spec.ImageName, "error:", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func test(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(9042)
|
||||
if err != nil {
|
||||
@@ -95,7 +79,7 @@ func test(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testMigrate(t *testing.T) {
|
||||
func TestMigrate(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(9042)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# ClickHouse
|
||||
|
||||
`clickhouse://host:port?username=user&password=qwerty&database=clicks&x-multi-statement=true`
|
||||
|
||||
| URL Query | Description |
|
||||
|------------|-------------|
|
||||
| `x-migrations-table`| Name of the migrations table |
|
||||
| `database` | The name of the database to connect to |
|
||||
| `username` | The user to sign in as |
|
||||
| `password` | The user's password |
|
||||
| `host` | The host to connect to. |
|
||||
| `port` | The port to bind to. |
|
||||
| `x-multi-statement` | false | Enable multiple statements to be ran in a single migration (See note below) |
|
||||
|
||||
## Notes
|
||||
|
||||
* The Clickhouse driver does not natively support executing multipe statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
|
||||
* This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
|
||||
* The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
|
||||
@@ -0,0 +1,260 @@
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
)
|
||||
|
||||
var (
|
||||
multiStmtDelimiter = []byte(";")
|
||||
|
||||
DefaultMigrationsTable = "schema_migrations"
|
||||
DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB
|
||||
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
DatabaseName string
|
||||
MigrationsTable string
|
||||
MultiStatementEnabled bool
|
||||
MultiStatementMaxSize int
|
||||
}
|
||||
|
||||
func init() {
|
||||
database.Register("clickhouse", &ClickHouse{})
|
||||
}
|
||||
|
||||
func WithInstance(conn *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := conn.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ch := &ClickHouse{
|
||||
conn: conn,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := ch.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
type ClickHouse struct {
|
||||
conn *sql.DB
|
||||
config *Config
|
||||
}
|
||||
|
||||
func (ch *ClickHouse) Open(dsn string) (database.Driver, error) {
|
||||
purl, err := url.Parse(dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := migrate.FilterCustomQuery(purl)
|
||||
q.Scheme = "tcp"
|
||||
conn, err := sql.Open("clickhouse", q.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
multiStatementMaxSize := DefaultMultiStatementMaxSize
|
||||
if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 {
|
||||
multiStatementMaxSize, err = strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
ch = &ClickHouse{
|
||||
conn: conn,
|
||||
config: &Config{
|
||||
MigrationsTable: purl.Query().Get("x-migrations-table"),
|
||||
DatabaseName: purl.Query().Get("database"),
|
||||
MultiStatementEnabled: purl.Query().Get("x-multi-statement") == "true",
|
||||
MultiStatementMaxSize: multiStatementMaxSize,
|
||||
},
|
||||
}
|
||||
|
||||
if err := ch.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func (ch *ClickHouse) init() error {
|
||||
if len(ch.config.DatabaseName) == 0 {
|
||||
if err := ch.conn.QueryRow("SELECT currentDatabase()").Scan(&ch.config.DatabaseName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(ch.config.MigrationsTable) == 0 {
|
||||
ch.config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
if ch.config.MultiStatementMaxSize <= 0 {
|
||||
ch.config.MultiStatementMaxSize = DefaultMultiStatementMaxSize
|
||||
}
|
||||
|
||||
return ch.ensureVersionTable()
|
||||
}
|
||||
|
||||
func (ch *ClickHouse) Run(r io.Reader) error {
|
||||
if ch.config.MultiStatementEnabled {
|
||||
var err error
|
||||
if e := multistmt.Parse(r, multiStmtDelimiter, ch.config.MultiStatementMaxSize, func(m []byte) bool {
|
||||
tq := strings.TrimSpace(string(m))
|
||||
if tq == "" {
|
||||
return true
|
||||
}
|
||||
if _, e := ch.conn.Exec(string(m)); e != nil {
|
||||
err = database.Error{OrigErr: e, Err: "migration failed", Query: m}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); e != nil {
|
||||
return e
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
migration, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := ch.conn.Exec(string(migration)); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: migration}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
func (ch *ClickHouse) Version() (int, bool, error) {
|
||||
var (
|
||||
version int
|
||||
dirty uint8
|
||||
query = "SELECT version, dirty FROM `" + ch.config.MigrationsTable + "` ORDER BY sequence DESC LIMIT 1"
|
||||
)
|
||||
if err := ch.conn.QueryRow(query).Scan(&version, &dirty); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return version, dirty == 1, nil
|
||||
}
|
||||
|
||||
func (ch *ClickHouse) SetVersion(version int, dirty bool) error {
|
||||
var (
|
||||
bool = func(v bool) uint8 {
|
||||
if v {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
tx, err = ch.conn.Begin()
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := "INSERT INTO " + ch.config.MigrationsTable + " (version, dirty, sequence) VALUES (?, ?, ?)"
|
||||
if _, err := tx.Exec(query, version, bool(dirty), time.Now().UnixNano()); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the ClickHouse type.
|
||||
func (ch *ClickHouse) ensureVersionTable() (err error) {
|
||||
if err = ch.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := ch.Unlock(); e != nil {
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
table string
|
||||
query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'"
|
||||
)
|
||||
// check if migration table exists
|
||||
if err := ch.conn.QueryRow(query).Scan(&table); err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
// if not, create the empty migration table
|
||||
query = `
|
||||
CREATE TABLE ` + ch.config.MigrationsTable + ` (
|
||||
version Int64,
|
||||
dirty UInt8,
|
||||
sequence UInt64
|
||||
) Engine=TinyLog
|
||||
`
|
||||
if _, err := ch.conn.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ch *ClickHouse) Drop() (err error) {
|
||||
query := "SHOW TABLES FROM " + ch.config.DatabaseName
|
||||
tables, err := ch.conn.Query(query)
|
||||
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
for tables.Next() {
|
||||
var table string
|
||||
if err := tables.Scan(&table); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query = "DROP TABLE IF EXISTS " + ch.config.DatabaseName + "." + table
|
||||
|
||||
if _, err := ch.conn.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ch *ClickHouse) Lock() error { return nil }
|
||||
func (ch *ClickHouse) Unlock() error { return nil }
|
||||
func (ch *ClickHouse) Close() error { return ch.conn.Close() }
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
CREATE TABLE test_1 (
|
||||
Date Date
|
||||
Date Date
|
||||
) Engine=Memory;
|
||||
+1
-1
@@ -1,3 +1,3 @@
|
||||
CREATE TABLE test_2 (
|
||||
Date Date
|
||||
Date Date
|
||||
) Engine=Memory;
|
||||
@@ -15,7 +15,7 @@ CREATE USER IF NOT EXISTS cockroach;
|
||||
GRANT ALL ON DATABASE example TO cockroach;
|
||||
```
|
||||
|
||||
When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convenience:
|
||||
When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience:
|
||||
```
|
||||
export COCKROACHDB_URL='cockroachdb://cockroach:@localhost:26257/example?sslmode=disable'
|
||||
```
|
||||
@@ -50,7 +50,7 @@ And in the `.down.sql` let's delete it:
|
||||
```
|
||||
DROP TABLE IF EXISTS example.users;
|
||||
```
|
||||
By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](/GETTING_STARTED.md#create-migrations)
|
||||
By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](GETTING_STARTED.md#create-migrations)
|
||||
|
||||
## Run migrations
|
||||
```
|
||||
@@ -139,4 +139,4 @@ func main() {
|
||||
}
|
||||
}
|
||||
```
|
||||
You can find details [here](README.md#use-in-your-go-project)
|
||||
You can find details [here](README.md#use-in-your-go-project)
|
||||
@@ -3,18 +3,23 @@ package cockroachdb
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
"github.com/cockroachdb/cockroach-go/v2/crdb"
|
||||
import (
|
||||
"github.com/cockroachdb/cockroach-go/crdb"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
import (
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -41,7 +46,7 @@ type Config struct {
|
||||
|
||||
type CockroachDb struct {
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
|
||||
// Open and WithInstance need to guarantee that config is never nil
|
||||
config *Config
|
||||
@@ -147,71 +152,75 @@ func (c *CockroachDb) Close() error {
|
||||
// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed
|
||||
// See: https://github.com/cockroachdb/cockroach/issues/13546
|
||||
func (c *CockroachDb) Lock() error {
|
||||
return database.CasRestoreOnErr(&c.isLocked, false, true, database.ErrLocked, func() (err error) {
|
||||
return crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) (err error) {
|
||||
aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1"
|
||||
rows, err := tx.Query(query, aid)
|
||||
if err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := rows.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
// If row exists at all, lock is present
|
||||
locked := rows.Next()
|
||||
if locked && !c.config.ForceLock {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)"
|
||||
if _, err := tx.Exec(query, aid); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed
|
||||
// See: https://github.com/cockroachdb/cockroach/issues/13546
|
||||
func (c *CockroachDb) Unlock() error {
|
||||
return database.CasRestoreOnErr(&c.isLocked, true, false, database.ErrNotLocked, func() (err error) {
|
||||
err := crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) (err error) {
|
||||
aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// In the event of an implementation (non-migration) error, it is possible for the lock to not be released. Until
|
||||
// a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances
|
||||
query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1"
|
||||
if _, err := c.db.Exec(query, aid); err != nil {
|
||||
if e, ok := err.(*pq.Error); ok {
|
||||
// 42P01 is "UndefinedTableError" in CockroachDB
|
||||
// https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go
|
||||
if e.Code == "42P01" {
|
||||
// On drops, the lock table is fully removed; This is fine, and is a valid "unlocked" state for the schema
|
||||
return nil
|
||||
}
|
||||
query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1"
|
||||
rows, err := tx.Query(query, aid)
|
||||
if err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := rows.Close(); errClose != nil {
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)}
|
||||
// If row exists at all, lock is present
|
||||
locked := rows.Next()
|
||||
if locked && !c.config.ForceLock {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)"
|
||||
if _, err := tx.Exec(query, aid); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
} else {
|
||||
c.isLocked = true
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed
|
||||
// See: https://github.com/cockroachdb/cockroach/issues/13546
|
||||
func (c *CockroachDb) Unlock() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// In the event of an implementation (non-migration) error, it is possible for the lock to not be released. Until
|
||||
// a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances
|
||||
query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1"
|
||||
if _, err := c.db.Exec(query, aid); err != nil {
|
||||
if e, ok := err.(*pq.Error); ok {
|
||||
// 42P01 is "UndefinedTableError" in CockroachDB
|
||||
// https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go
|
||||
if e.Code == "42P01" {
|
||||
// On drops, the lock table is fully removed; This is fine, and is a valid "unlocked" state for the schema
|
||||
c.isLocked = false
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)}
|
||||
}
|
||||
|
||||
c.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *CockroachDb) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -276,7 +285,7 @@ func (c *CockroachDb) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -291,9 +300,6 @@ func (c *CockroachDb) Drop() (err error) {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// delete one by one ...
|
||||
@@ -318,7 +324,11 @@ func (c *CockroachDb) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := c.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -26,13 +26,13 @@ import (
|
||||
const defaultPort = 26257
|
||||
|
||||
var (
|
||||
opts = dktest.Options{Cmd: []string{"start-single-node", "--insecure"}, PortRequired: true, ReadyFunc: isReady}
|
||||
// Supported versions: https://www.cockroachlabs.com/docs/releases/release-support-policy#supported-versions
|
||||
opts = dktest.Options{Cmd: []string{"start", "--insecure"}, PortRequired: true, ReadyFunc: isReady}
|
||||
// Released versions: https://www.cockroachlabs.com/docs/releases/
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "cockroachdb/cockroach:latest-v24.3", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:latest-v24.1", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:latest-v23.2", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:latest-v23.1", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:v1.0.7", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:v1.1.9", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:v2.0.7", Options: opts},
|
||||
{ImageName: "cockroachdb/cockroach:v2.1.3", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
# Datastore
|
||||
|
||||
`datastore://host:port?username=user&password=password&database=clicks&x-multi-statement=true`
|
||||
|
||||
Hanzo Datastore is a ClickHouse-wire-compatible database engine. This driver
|
||||
connects through the native `github.com/hanzo-ds/go` driver, which registers the
|
||||
`database/sql` driver name `datastore`.
|
||||
|
||||
| URL Query | Description |
|
||||
|------------|-------------|
|
||||
| `x-migrations-table`| Name of the migrations table |
|
||||
| `x-migrations-table-engine`| Engine to use for the migrations table, defaults to TinyLog |
|
||||
| `x-cluster-name` | Name of cluster for creating `schema_migrations` table cluster wide |
|
||||
| `x-multi-statement` | Enable multiple statements to be ran in a single migration (See note below) |
|
||||
| `x-multi-statement-max-size` | Maximum size of a single migration in bytes when `x-multi-statement` is set (defaults to 10 MB) |
|
||||
| `database` | The name of the database to connect to |
|
||||
| `username` | The user to sign in as |
|
||||
| `password` | The user's password |
|
||||
| `host` | The host to connect to. |
|
||||
| `port` | The port to bind to. |
|
||||
|
||||
## Notes
|
||||
|
||||
* The Datastore driver does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
|
||||
* This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
|
||||
* The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
|
||||
* Using the default TinyLog table engine for the `schema_migrations` table prevents backing up the table. If you need to back up the database, run the migrations with `x-migrations-table-engine=MergeTree`.
|
||||
* Datastore cluster mode is not officially supported, but you can try enabling `schema_migrations` table replication by specifying a `x-cluster-name`:
|
||||
* When `x-cluster-name` is specified, `x-migrations-table-engine` also should be specified.
|
||||
* When `x-cluster-name` is specified, only the `schema_migrations` table is replicated across the cluster. You still need to write your migrations so that the application tables are replicated within the cluster.
|
||||
* If you want to create a database inside a migration, note that the `schema_migrations` table will live in the `default` database, so you cannot use `USE <database_name>` inside a migration. In this case you may omit the database in the connection string (example [here](examples/migrations/003_create_database.up.sql)).
|
||||
@@ -1,337 +0,0 @@
|
||||
// Package datastore implements the golang-migrate database.Driver interface for
|
||||
// Hanzo Datastore (a ClickHouse-wire-compatible database engine). It registers
|
||||
// the "datastore://" URL scheme and connects through the native
|
||||
// github.com/hanzo-ds/go driver, which registers the database/sql driver name
|
||||
// "datastore".
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// import _ "github.com/golang-migrate/migrate/v4/database/datastore"
|
||||
//
|
||||
// Then use a URL like:
|
||||
//
|
||||
// datastore://host:9000?username=default&database=default&x-multi-statement=true
|
||||
package datastore
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
|
||||
// Registers the "datastore" database/sql driver (ClickHouse-wire native).
|
||||
_ "github.com/hanzo-ds/go"
|
||||
)
|
||||
|
||||
var (
|
||||
multiStmtDelimiter = []byte(";")
|
||||
|
||||
// DriverName is the database/sql driver registered by github.com/hanzo-ds/go.
|
||||
DriverName = "datastore"
|
||||
|
||||
DefaultMigrationsTable = "schema_migrations"
|
||||
DefaultMigrationsTableEngine = "TinyLog"
|
||||
DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB
|
||||
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
DatabaseName string
|
||||
ClusterName string
|
||||
MigrationsTable string
|
||||
MigrationsTableEngine string
|
||||
MultiStatementEnabled bool
|
||||
MultiStatementMaxSize int
|
||||
}
|
||||
|
||||
func init() {
|
||||
database.Register("datastore", &Datastore{})
|
||||
}
|
||||
|
||||
// WithInstance creates a Datastore driver from an existing *sql.DB opened
|
||||
// against the "datastore" driver.
|
||||
func WithInstance(conn *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := conn.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ds := &Datastore{
|
||||
conn: conn,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := ds.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
// Datastore is a golang-migrate database.Driver backed by the native
|
||||
// github.com/hanzo-ds/go ClickHouse-wire driver.
|
||||
type Datastore struct {
|
||||
conn *sql.DB
|
||||
config *Config
|
||||
isLocked atomic.Bool
|
||||
}
|
||||
|
||||
func (ds *Datastore) Open(dsn string) (database.Driver, error) {
|
||||
purl, err := url.Parse(dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := migrate.FilterCustomQuery(purl)
|
||||
q.Scheme = DriverName
|
||||
conn, err := sql.Open(DriverName, q.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
multiStatementMaxSize := DefaultMultiStatementMaxSize
|
||||
if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 {
|
||||
multiStatementMaxSize, err = strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
migrationsTableEngine := DefaultMigrationsTableEngine
|
||||
if s := purl.Query().Get("x-migrations-table-engine"); len(s) > 0 {
|
||||
migrationsTableEngine = s
|
||||
}
|
||||
|
||||
ds = &Datastore{
|
||||
conn: conn,
|
||||
config: &Config{
|
||||
MigrationsTable: purl.Query().Get("x-migrations-table"),
|
||||
MigrationsTableEngine: migrationsTableEngine,
|
||||
DatabaseName: purl.Query().Get("database"),
|
||||
ClusterName: purl.Query().Get("x-cluster-name"),
|
||||
MultiStatementEnabled: purl.Query().Get("x-multi-statement") == "true",
|
||||
MultiStatementMaxSize: multiStatementMaxSize,
|
||||
},
|
||||
}
|
||||
|
||||
if err := ds.init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ds, nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) init() error {
|
||||
if len(ds.config.DatabaseName) == 0 {
|
||||
if err := ds.conn.QueryRow("SELECT currentDatabase()").Scan(&ds.config.DatabaseName); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(ds.config.MigrationsTable) == 0 {
|
||||
ds.config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
if ds.config.MultiStatementMaxSize <= 0 {
|
||||
ds.config.MultiStatementMaxSize = DefaultMultiStatementMaxSize
|
||||
}
|
||||
|
||||
if len(ds.config.MigrationsTableEngine) == 0 {
|
||||
ds.config.MigrationsTableEngine = DefaultMigrationsTableEngine
|
||||
}
|
||||
|
||||
return ds.ensureVersionTable()
|
||||
}
|
||||
|
||||
func (ds *Datastore) Run(r io.Reader) error {
|
||||
if ds.config.MultiStatementEnabled {
|
||||
var err error
|
||||
if e := multistmt.Parse(r, multiStmtDelimiter, ds.config.MultiStatementMaxSize, func(m []byte) bool {
|
||||
tq := strings.TrimSpace(string(m))
|
||||
if tq == "" {
|
||||
return true
|
||||
}
|
||||
if _, e := ds.conn.Exec(string(m)); e != nil {
|
||||
err = database.Error{OrigErr: e, Err: "migration failed", Query: m}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); e != nil {
|
||||
return e
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
migration, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := ds.conn.Exec(string(migration)); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: migration}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) Version() (int, bool, error) {
|
||||
var (
|
||||
version int
|
||||
dirty uint8
|
||||
query = "SELECT version, dirty FROM `" + ds.config.MigrationsTable + "` ORDER BY sequence DESC LIMIT 1"
|
||||
)
|
||||
if err := ds.conn.QueryRow(query).Scan(&version, &dirty); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return version, dirty == 1, nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) SetVersion(version int, dirty bool) error {
|
||||
var (
|
||||
bool = func(v bool) uint8 {
|
||||
if v {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
tx, err = ds.conn.Begin()
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := "INSERT INTO " + ds.config.MigrationsTable + " (version, dirty, sequence) VALUES (?, ?, ?)"
|
||||
if _, err := tx.Exec(query, version, bool(dirty), time.Now().UnixNano()); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the Datastore type.
|
||||
func (ds *Datastore) ensureVersionTable() (err error) {
|
||||
if err = ds.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := ds.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
table string
|
||||
query = "SHOW TABLES FROM " + quoteIdentifier(ds.config.DatabaseName) + " LIKE '" + ds.config.MigrationsTable + "'"
|
||||
)
|
||||
// check if migration table exists
|
||||
if err := ds.conn.QueryRow(query).Scan(&table); err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
|
||||
// if not, create the empty migration table
|
||||
if len(ds.config.ClusterName) > 0 {
|
||||
query = fmt.Sprintf(`
|
||||
CREATE TABLE %s ON CLUSTER %s (
|
||||
version Int64,
|
||||
dirty UInt8,
|
||||
sequence UInt64
|
||||
) Engine=%s`, ds.config.MigrationsTable, ds.config.ClusterName, ds.config.MigrationsTableEngine)
|
||||
} else {
|
||||
query = fmt.Sprintf(`
|
||||
CREATE TABLE %s (
|
||||
version Int64,
|
||||
dirty UInt8,
|
||||
sequence UInt64
|
||||
) Engine=%s`, ds.config.MigrationsTable, ds.config.MigrationsTableEngine)
|
||||
}
|
||||
|
||||
if strings.HasSuffix(ds.config.MigrationsTableEngine, "Tree") {
|
||||
query = fmt.Sprintf(`%s ORDER BY sequence`, query)
|
||||
}
|
||||
|
||||
if _, err := ds.conn.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) Drop() (err error) {
|
||||
query := "SHOW TABLES FROM " + quoteIdentifier(ds.config.DatabaseName)
|
||||
tables, err := ds.conn.Query(query)
|
||||
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
for tables.Next() {
|
||||
var table string
|
||||
if err := tables.Scan(&table); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query = "DROP TABLE IF EXISTS " + quoteIdentifier(ds.config.DatabaseName) + "." + quoteIdentifier(table)
|
||||
|
||||
if _, err := ds.conn.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) Lock() error {
|
||||
if !ds.isLocked.CompareAndSwap(false, true) {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) Unlock() error {
|
||||
if !ds.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ds *Datastore) Close() error { return ds.conn.Close() }
|
||||
|
||||
// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611
|
||||
func quoteIdentifier(name string) string {
|
||||
end := strings.IndexRune(name, 0)
|
||||
if end > -1 {
|
||||
name = name[:end]
|
||||
}
|
||||
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
|
||||
}
|
||||
@@ -1,226 +0,0 @@
|
||||
package datastore_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
sqldriver "database/sql/driver"
|
||||
"fmt"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/dhui/dktest"
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database/datastore"
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
"github.com/golang-migrate/migrate/v4/dktesting"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
|
||||
// Registers the "datastore" database/sql driver (ClickHouse-wire native).
|
||||
_ "github.com/hanzo-ds/go"
|
||||
)
|
||||
|
||||
const defaultPort = 9000
|
||||
|
||||
var (
|
||||
tableEngines = []string{"TinyLog", "MergeTree"}
|
||||
opts = dktest.Options{
|
||||
Env: map[string]string{"CLICKHOUSE_USER": "user", "CLICKHOUSE_PASSWORD": "password", "CLICKHOUSE_DB": "db"},
|
||||
PortRequired: true, ReadyFunc: isReady,
|
||||
}
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "clickhouse:24.8", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
func datastoreConnectionString(host, port, engine string) string {
|
||||
if engine != "" {
|
||||
return fmt.Sprintf(
|
||||
"datastore://%v:%v?username=user&password=password&database=db&x-multi-statement=true&x-migrations-table-engine=%v&debug=false",
|
||||
host, port, engine)
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"datastore://%v:%v?username=user&password=password&database=db&x-multi-statement=true&debug=false",
|
||||
host, port)
|
||||
}
|
||||
|
||||
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
db, err := sql.Open("datastore", datastoreConnectionString(ip, port, ""))
|
||||
|
||||
if err != nil {
|
||||
log.Println("open error", err)
|
||||
return false
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
log.Println("close error:", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err = db.PingContext(ctx); err != nil {
|
||||
switch err {
|
||||
case sqldriver.ErrBadConn:
|
||||
return false
|
||||
default:
|
||||
fmt.Println(err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func TestCases(t *testing.T) {
|
||||
for _, engine := range tableEngines {
|
||||
t.Run("Test_"+engine, func(t *testing.T) { testSimple(t, engine) })
|
||||
t.Run("Migrate_"+engine, func(t *testing.T) { testMigrate(t, engine) })
|
||||
t.Run("Version_"+engine, func(t *testing.T) { testVersion(t, engine) })
|
||||
t.Run("Drop_"+engine, func(t *testing.T) { testDrop(t, engine) })
|
||||
}
|
||||
t.Run("WithInstanceDefaultConfigValues", func(t *testing.T) { testSimpleWithInstanceDefaultConfigValues(t) })
|
||||
}
|
||||
|
||||
func testSimple(t *testing.T, engine string) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := datastoreConnectionString(ip, port, engine)
|
||||
p := &datastore.Datastore{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
})
|
||||
}
|
||||
|
||||
func testSimpleWithInstanceDefaultConfigValues(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := datastoreConnectionString(ip, port, "")
|
||||
conn, err := sql.Open("datastore", addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := datastore.WithInstance(conn, &datastore.Config{})
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
})
|
||||
}
|
||||
|
||||
func testMigrate(t *testing.T, engine string) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := datastoreConnectionString(ip, port, engine)
|
||||
p := &datastore.Datastore{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "db", d)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
})
|
||||
}
|
||||
|
||||
func testVersion(t *testing.T, engine string) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
expectedVersion := 1
|
||||
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := datastoreConnectionString(ip, port, engine)
|
||||
p := &datastore.Datastore{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
err = d.SetVersion(expectedVersion, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
version, _, err := d.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if version != expectedVersion {
|
||||
t.Fatal("Version mismatch")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testDrop(t *testing.T, engine string) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := datastoreConnectionString(ip, port, engine)
|
||||
p := &datastore.Datastore{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
err = d.Drop()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
DROP TABLE IF EXISTS driver_ratings;
|
||||
DROP TABLE IF EXISTS user_ratings;
|
||||
DROP TABLE IF EXISTS orders;
|
||||
DROP TABLE IF EXISTS driver_ratings_queue;
|
||||
DROP TABLE IF EXISTS user_ratings_queue;
|
||||
DROP TABLE IF EXISTS orders_queue;
|
||||
DROP VIEW IF EXISTS user_ratings_queue_mv;
|
||||
DROP VIEW IF EXISTS driver_ratings_queue_mv;
|
||||
DROP VIEW IF EXISTS orders_queue_mv;
|
||||
DROP DATABASE IF EXISTS analytics;
|
||||
@@ -1,81 +0,0 @@
|
||||
CREATE DATABASE IF NOT EXISTS analytics;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analytics.driver_ratings(
|
||||
rate UInt8,
|
||||
userID Int64,
|
||||
driverID String,
|
||||
orderID String,
|
||||
inserted_time DateTime DEFAULT now()
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY driverID
|
||||
ORDER BY (inserted_time);
|
||||
|
||||
CREATE TABLE analytics.driver_ratings_queue(
|
||||
rate UInt8,
|
||||
userID Int64,
|
||||
driverID String,
|
||||
orderID String
|
||||
) ENGINE = Kafka
|
||||
SETTINGS kafka_broker_list = 'broker:9092',
|
||||
kafka_topic_list = 'driver-ratings',
|
||||
kafka_group_name = 'rating_readers',
|
||||
kafka_format = 'Avro',
|
||||
kafka_max_block_size = 1048576;
|
||||
|
||||
CREATE MATERIALIZED VIEW analytics.driver_ratings_queue_mv TO analytics.driver_ratings AS
|
||||
SELECT rate, userID, driverID, orderID
|
||||
FROM analytics.driver_ratings_queue;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analytics.user_ratings(
|
||||
rate UInt8,
|
||||
userID Int64,
|
||||
driverID String,
|
||||
orderID String,
|
||||
inserted_time DateTime DEFAULT now()
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY userID
|
||||
ORDER BY (inserted_time);
|
||||
|
||||
CREATE TABLE analytics.user_ratings_queue(
|
||||
rate UInt8,
|
||||
userID Int64,
|
||||
driverID String,
|
||||
orderID String
|
||||
) ENGINE = Kafka
|
||||
SETTINGS kafka_broker_list = 'broker:9092',
|
||||
kafka_topic_list = 'user-ratings',
|
||||
kafka_group_name = 'rating_readers',
|
||||
kafka_format = 'JSON',
|
||||
kafka_max_block_size = 1048576;
|
||||
|
||||
CREATE MATERIALIZED VIEW analytics.user_ratings_queue_mv TO analytics.user_ratings AS
|
||||
SELECT rate, userID, driverID, orderID
|
||||
FROM analytics.user_ratings_queue;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS analytics.orders(
|
||||
from_place String,
|
||||
to_place String,
|
||||
userID Int64,
|
||||
driverID String,
|
||||
orderID String,
|
||||
inserted_time DateTime DEFAULT now()
|
||||
) ENGINE = MergeTree
|
||||
PARTITION BY driverID
|
||||
ORDER BY (inserted_time);
|
||||
|
||||
CREATE TABLE analytics.orders_queue(
|
||||
from_place String,
|
||||
to_place String,
|
||||
userID Int64,
|
||||
driverID String,
|
||||
orderID String
|
||||
) ENGINE = Kafka
|
||||
SETTINGS kafka_broker_list = 'broker:9092',
|
||||
kafka_topic_list = 'orders',
|
||||
kafka_group_name = 'order_readers',
|
||||
kafka_format = 'Avro',
|
||||
kafka_max_block_size = 1048576;
|
||||
|
||||
CREATE MATERIALIZED VIEW analytics.orders_queue_mv TO orders AS
|
||||
SELECT from_place, to_place, userID, driverID, orderID
|
||||
FROM analytics.orders_queue;
|
||||
+14
-14
@@ -1,4 +1,4 @@
|
||||
// Package database provides the Driver interface.
|
||||
// Package database provides the Database interface.
|
||||
// All database drivers must implement this interface, register themselves,
|
||||
// optionally provide a `WithInstance` function and pass the tests
|
||||
// in package database/testing.
|
||||
@@ -25,22 +25,22 @@ var drivers = make(map[string]Driver)
|
||||
// Driver is the interface every database driver must implement.
|
||||
//
|
||||
// How to implement a database driver?
|
||||
// 1. Implement this interface.
|
||||
// 2. Optionally, add a function named `WithInstance`.
|
||||
// This function should accept an existing DB instance and a Config{} struct
|
||||
// and return a driver instance.
|
||||
// 3. Add a test that calls database/testing.go:Test()
|
||||
// 4. Add own tests for Open(), WithInstance() (when provided) and Close().
|
||||
// All other functions are tested by tests in database/testing.
|
||||
// Saves you some time and makes sure all database drivers behave the same way.
|
||||
// 5. Call Register in init().
|
||||
// 6. Create a internal/cli/build_<driver-name>.go file
|
||||
// 7. Add driver name in 'DATABASE' variable in Makefile
|
||||
// 1. Implement this interface.
|
||||
// 2. Optionally, add a function named `WithInstance`.
|
||||
// This function should accept an existing DB instance and a Config{} struct
|
||||
// and return a driver instance.
|
||||
// 3. Add a test that calls database/testing.go:Test()
|
||||
// 4. Add own tests for Open(), WithInstance() (when provided) and Close().
|
||||
// All other functions are tested by tests in database/testing.
|
||||
// Saves you some time and makes sure all database drivers behave the same way.
|
||||
// 5. Call Register in init().
|
||||
// 6. Create a internal/cli/build_<driver-name>.go file
|
||||
// 7. Add driver name in 'DATABASE' variable in Makefile
|
||||
//
|
||||
// Guidelines:
|
||||
// - Don't try to correct user input. Don't assume things.
|
||||
// * Don't try to correct user input. Don't assume things.
|
||||
// When in doubt, return an error and explain the situation to the user.
|
||||
// - All configuration input must come from the URL string in func Open()
|
||||
// * All configuration input must come from the URL string in func Open()
|
||||
// or the Config{} struct in WithInstance. Don't os.Getenv().
|
||||
type Driver interface {
|
||||
// Open returns a new driver instance configured with parameters
|
||||
|
||||
@@ -1,19 +1,18 @@
|
||||
//go:build go1.9
|
||||
// +build go1.9
|
||||
|
||||
package firebird
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
_ "github.com/nakagami/firebirdsql"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -37,7 +36,7 @@ type Firebird struct {
|
||||
// Locking and unlocking need to use the same connection
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
|
||||
// Open and WithInstance need to guarantee that config is never nil
|
||||
config *Config
|
||||
@@ -107,21 +106,20 @@ func (f *Firebird) Close() error {
|
||||
}
|
||||
|
||||
func (f *Firebird) Lock() error {
|
||||
if !f.isLocked.CompareAndSwap(false, true) {
|
||||
if f.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
f.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Firebird) Unlock() error {
|
||||
if !f.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
f.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Firebird) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -180,7 +178,7 @@ func (f *Firebird) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -195,9 +193,6 @@ func (f *Firebird) Drop() (err error) {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// delete one by one ...
|
||||
for _, t := range tableNames {
|
||||
@@ -223,7 +218,11 @@ func (f *Firebird) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := f.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -38,9 +38,8 @@ var (
|
||||
},
|
||||
}
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "jacobalberty/firebird:v3.0", Options: opts},
|
||||
{ImageName: "jacobalberty/firebird:v4.0", Options: opts},
|
||||
{ImageName: "jacobalberty/firebird:v5.0", Options: opts},
|
||||
{ImageName: "jacobalberty/firebird:2.5-ss", Options: opts},
|
||||
{ImageName: "jacobalberty/firebird:3.0", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -13,10 +13,6 @@
|
||||
|------------|---------------------|-------------|
|
||||
| `x-migrations-collection` | `MigrationsCollection` | Name of the migrations collection |
|
||||
| `x-transaction-mode` | `TransactionMode` | If set to `true` wrap commands in [transaction](https://docs.mongodb.com/manual/core/transactions). Available only for replica set. Driver is using [strconv.ParseBool](https://golang.org/pkg/strconv/#ParseBool) for parsing|
|
||||
| `x-advisory-locking` | `true` | Feature flag for advisory locking, if set to false, disable advisory locking |
|
||||
| `x-advisory-lock-collection` | `migrate_advisory_lock` | The name of the collection to use for advisory locking.|
|
||||
| `x-advisory-lock-timeout` | `15` | The max time in seconds that migrate will wait to acquire a lock before failing. |
|
||||
| `x-advisory-lock-timeout-interval` | `10` | The max time in seconds between attempts to acquire the advisory lock, the lock is attempted to be acquired using an exponential backoff algorithm. |
|
||||
| `dbname` | `DatabaseName` | The name of the database to connect to |
|
||||
| `user` | | The user to sign in as. Can be omitted |
|
||||
| `password` | | The user's password. Can be omitted |
|
||||
|
||||
+16
-223
@@ -2,16 +2,12 @@ package mongodb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
@@ -27,54 +23,29 @@ func init() {
|
||||
|
||||
var DefaultMigrationsCollection = "schema_migrations"
|
||||
|
||||
const DefaultLockingCollection = "migrate_advisory_lock" // the collection to use for advisory locking by default.
|
||||
const lockKeyUniqueValue = 0 // the unique value to lock on. If multiple clients try to insert the same key, it will fail (locked).
|
||||
const DefaultLockTimeout = 15 // the default maximum time to wait for a lock to be released.
|
||||
const DefaultLockTimeoutInterval = 10 // the default maximum intervals time for the locking timout.
|
||||
const DefaultAdvisoryLockingFlag = true // the default value for the advisory locking feature flag. Default is true.
|
||||
const LockIndexName = "lock_unique_key" // the name of the index which adds unique constraint to the locking_key field.
|
||||
const contextWaitTimeout = 5 * time.Second // how long to wait for the request to mongo to block/wait for.
|
||||
|
||||
var (
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrLockTimeoutConfigConflict = fmt.Errorf("both x-advisory-lock-timeout-interval and x-advisory-lock-timout-interval were specified")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
)
|
||||
|
||||
type Mongo struct {
|
||||
client *mongo.Client
|
||||
db *mongo.Database
|
||||
config *Config
|
||||
isLocked atomic.Bool
|
||||
client *mongo.Client
|
||||
db *mongo.Database
|
||||
|
||||
config *Config
|
||||
}
|
||||
|
||||
type Locking struct {
|
||||
CollectionName string
|
||||
Timeout int
|
||||
Enabled bool
|
||||
Interval int
|
||||
}
|
||||
type Config struct {
|
||||
DatabaseName string
|
||||
MigrationsCollection string
|
||||
TransactionMode bool
|
||||
Locking Locking
|
||||
}
|
||||
|
||||
type versionInfo struct {
|
||||
Version int `bson:"version"`
|
||||
Dirty bool `bson:"dirty"`
|
||||
}
|
||||
|
||||
type lockObj struct {
|
||||
Key int `bson:"locking_key"`
|
||||
Pid int `bson:"pid"`
|
||||
Hostname string `bson:"hostname"`
|
||||
CreatedAt time.Time `bson:"created_at"`
|
||||
}
|
||||
type findFilter struct {
|
||||
Key int `bson:"locking_key"`
|
||||
}
|
||||
|
||||
func WithInstance(instance *mongo.Client, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
@@ -85,36 +56,17 @@ func WithInstance(instance *mongo.Client, config *Config) (database.Driver, erro
|
||||
if len(config.MigrationsCollection) == 0 {
|
||||
config.MigrationsCollection = DefaultMigrationsCollection
|
||||
}
|
||||
if len(config.Locking.CollectionName) == 0 {
|
||||
config.Locking.CollectionName = DefaultLockingCollection
|
||||
}
|
||||
if config.Locking.Timeout <= 0 {
|
||||
config.Locking.Timeout = DefaultLockTimeout
|
||||
}
|
||||
if config.Locking.Interval <= 0 {
|
||||
config.Locking.Interval = DefaultLockTimeoutInterval
|
||||
}
|
||||
|
||||
mc := &Mongo{
|
||||
client: instance,
|
||||
db: instance.Database(config.DatabaseName),
|
||||
config: config,
|
||||
}
|
||||
|
||||
if mc.config.Locking.Enabled {
|
||||
if err := mc.ensureLockTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := mc.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
func (m *Mongo) Open(dsn string) (database.Driver, error) {
|
||||
// connstring is experimental package, but it used for parse connection string in mongo.Connect function
|
||||
//connsting is experimental package, but it used for parse connection string in mongo.Connect function
|
||||
uri, err := connstring.Parse(dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -122,46 +74,16 @@ func (m *Mongo) Open(dsn string) (database.Driver, error) {
|
||||
if len(uri.Database) == 0 {
|
||||
return nil, ErrNoDatabaseName
|
||||
}
|
||||
|
||||
unknown := url.Values(uri.UnknownOptions)
|
||||
|
||||
migrationsCollection := unknown.Get("x-migrations-collection")
|
||||
lockCollection := unknown.Get("x-advisory-lock-collection")
|
||||
transactionMode, err := parseBoolean(unknown.Get("x-transaction-mode"), false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
advisoryLockingFlag, err := parseBoolean(unknown.Get("x-advisory-locking"), DefaultAdvisoryLockingFlag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lockingTimout, err := parseInt(unknown.Get("x-advisory-lock-timeout"), DefaultLockTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transactionMode, _ := strconv.ParseBool(unknown.Get("x-transaction-mode"))
|
||||
|
||||
lockTimeoutIntervalValue := unknown.Get("x-advisory-lock-timeout-interval")
|
||||
// The initial release had a typo for this argument but for backwards compatibility sake, we will keep supporting it
|
||||
// and we will error out if both values are set.
|
||||
lockTimeoutIntervalValueFromTypo := unknown.Get("x-advisory-lock-timout-interval")
|
||||
|
||||
lockTimeout := lockTimeoutIntervalValue
|
||||
|
||||
if lockTimeoutIntervalValue != "" && lockTimeoutIntervalValueFromTypo != "" {
|
||||
return nil, ErrLockTimeoutConfigConflict
|
||||
} else if lockTimeoutIntervalValueFromTypo != "" {
|
||||
lockTimeout = lockTimeoutIntervalValueFromTypo
|
||||
}
|
||||
|
||||
maxLockCheckInterval, err := parseInt(lockTimeout, DefaultLockTimeoutInterval)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(dsn))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = client.Ping(context.TODO(), nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -169,12 +91,6 @@ func (m *Mongo) Open(dsn string) (database.Driver, error) {
|
||||
DatabaseName: uri.Database,
|
||||
MigrationsCollection: migrationsCollection,
|
||||
TransactionMode: transactionMode,
|
||||
Locking: Locking{
|
||||
CollectionName: lockCollection,
|
||||
Timeout: lockingTimout,
|
||||
Enabled: advisoryLockingFlag,
|
||||
Interval: maxLockCheckInterval,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -182,39 +98,6 @@ func (m *Mongo) Open(dsn string) (database.Driver, error) {
|
||||
return mc, nil
|
||||
}
|
||||
|
||||
// Parse the url param, convert it to boolean
|
||||
// returns error if param invalid. returns defaultValue if param not present
|
||||
func parseBoolean(urlParam string, defaultValue bool) (bool, error) {
|
||||
|
||||
// if parameter passed, parse it (otherwise return default value)
|
||||
if urlParam != "" {
|
||||
result, err := strconv.ParseBool(urlParam)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// if no url Param passed, return default value
|
||||
return defaultValue, nil
|
||||
}
|
||||
|
||||
// Parse the url param, convert it to int
|
||||
// returns error if param invalid. returns defaultValue if param not present
|
||||
func parseInt(urlParam string, defaultValue int) (int, error) {
|
||||
|
||||
// if parameter passed, parse it (otherwise return default value)
|
||||
if urlParam != "" {
|
||||
result, err := strconv.Atoi(urlParam)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// if no url Param passed, return default value
|
||||
return defaultValue, nil
|
||||
}
|
||||
func (m *Mongo) SetVersion(version int, dirty bool) error {
|
||||
migrationsCollection := m.db.Collection(m.config.MigrationsCollection)
|
||||
if err := migrationsCollection.Drop(context.TODO()); err != nil {
|
||||
@@ -241,7 +124,7 @@ func (m *Mongo) Version() (version int, dirty bool, err error) {
|
||||
}
|
||||
|
||||
func (m *Mongo) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -268,8 +151,8 @@ func (m *Mongo) executeCommandsWithTransaction(ctx context.Context, cmds []bson.
|
||||
return &database.Error{OrigErr: err, Err: "failed to start transaction"}
|
||||
}
|
||||
if err := m.executeCommands(sessionContext, cmds); err != nil {
|
||||
// When command execution is failed, it's aborting transaction
|
||||
// If you tried to call abortTransaction, it`s return error that transaction already aborted
|
||||
//When command execution is failed, it's aborting transaction
|
||||
//If you tried to call abortTransaction, it`s return error that transaction already aborted
|
||||
return err
|
||||
}
|
||||
if err := sessionContext.CommitTransaction(sessionContext); err != nil {
|
||||
@@ -301,100 +184,10 @@ func (m *Mongo) Drop() error {
|
||||
return m.db.Drop(context.TODO())
|
||||
}
|
||||
|
||||
func (m *Mongo) ensureLockTable() error {
|
||||
indexes := m.db.Collection(m.config.Locking.CollectionName).Indexes()
|
||||
|
||||
indexOptions := options.Index().SetUnique(true).SetName(LockIndexName)
|
||||
_, err := indexes.CreateOne(context.TODO(), mongo.IndexModel{
|
||||
Options: indexOptions,
|
||||
Keys: findFilter{Key: -1},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the MongoDb type.
|
||||
func (m *Mongo) ensureVersionTable() (err error) {
|
||||
if err = m.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := m.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
}
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, _, err = m.Version(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Utilizes advisory locking on the config.LockingCollection collection
|
||||
// This uses a unique index on the `locking_key` field.
|
||||
func (m *Mongo) Lock() error {
|
||||
return database.CasRestoreOnErr(&m.isLocked, false, true, database.ErrLocked, func() error {
|
||||
if !m.config.Locking.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
pid := os.Getpid()
|
||||
hostname, err := os.Hostname()
|
||||
if err != nil {
|
||||
hostname = fmt.Sprintf("Could not determine hostname. Error: %s", err.Error())
|
||||
}
|
||||
|
||||
newLockObj := lockObj{
|
||||
Key: lockKeyUniqueValue,
|
||||
Pid: pid,
|
||||
Hostname: hostname,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
operation := func() error {
|
||||
timeout, cancelFunc := context.WithTimeout(context.Background(), contextWaitTimeout)
|
||||
_, err := m.db.Collection(m.config.Locking.CollectionName).InsertOne(timeout, newLockObj)
|
||||
defer cancelFunc()
|
||||
return err
|
||||
}
|
||||
exponentialBackOff := backoff.NewExponentialBackOff()
|
||||
duration := time.Duration(m.config.Locking.Timeout) * time.Second
|
||||
exponentialBackOff.MaxElapsedTime = duration
|
||||
exponentialBackOff.MaxInterval = time.Duration(m.config.Locking.Interval) * time.Second
|
||||
|
||||
err = backoff.Retry(operation, exponentialBackOff)
|
||||
if err != nil {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Mongo) Unlock() error {
|
||||
return database.CasRestoreOnErr(&m.isLocked, true, false, database.ErrNotLocked, func() error {
|
||||
if !m.config.Locking.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
filter := findFilter{
|
||||
Key: lockKeyUniqueValue,
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), contextWaitTimeout)
|
||||
_, err := m.db.Collection(m.config.Locking.CollectionName).DeleteMany(ctx, filter)
|
||||
defer cancel()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -32,10 +32,10 @@ var (
|
||||
opts = dktest.Options{PortRequired: true, ReadyFunc: isReady}
|
||||
// Supported versions: https://www.mongodb.com/support-policy
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "mongo:5.0", Options: opts},
|
||||
{ImageName: "mongo:6.0", Options: opts},
|
||||
{ImageName: "mongo:7.0", Options: opts},
|
||||
{ImageName: "mongo:8.0", Options: opts},
|
||||
{ImageName: "mongo:3.4", Options: opts},
|
||||
{ImageName: "mongo:3.6", Options: opts},
|
||||
{ImageName: "mongo:4.0", Options: opts},
|
||||
{ImageName: "mongo:4.2", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -74,22 +74,6 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
t.Run("test", test)
|
||||
t.Run("testMigrate", testMigrate)
|
||||
t.Run("testWithAuth", testWithAuth)
|
||||
t.Run("testLockWorks", testLockWorks)
|
||||
|
||||
t.Cleanup(func() {
|
||||
for _, spec := range specs {
|
||||
t.Log("Cleaning up ", spec.ImageName)
|
||||
if err := spec.Cleanup(); err != nil {
|
||||
t.Error("Error removing ", spec.ImageName, "error:", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func test(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -108,14 +92,14 @@ func test(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
dt.TestNilVersion(t, d)
|
||||
dt.TestLockAndUnlock(t, d)
|
||||
//TestLockAndUnlock(t, d) driver doesn't support lock on database level
|
||||
dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`)))
|
||||
dt.TestSetVersion(t, d)
|
||||
dt.TestDrop(t, d)
|
||||
})
|
||||
}
|
||||
|
||||
func testMigrate(t *testing.T) {
|
||||
func TestMigrate(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -141,7 +125,7 @@ func testMigrate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testWithAuth(t *testing.T) {
|
||||
func TestWithAuth(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -196,76 +180,11 @@ func testWithAuth(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testLockWorks(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := mongoConnectionString(ip, port)
|
||||
p := &Mongo{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
dt.TestRun(t, d, bytes.NewReader([]byte(`[{"insert":"hello","documents":[{"wild":"world"}]}]`)))
|
||||
|
||||
mc := d.(*Mongo)
|
||||
|
||||
err = mc.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = mc.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = mc.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = mc.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// enable locking,
|
||||
//try to hit a lock conflict
|
||||
mc.config.Locking.Enabled = true
|
||||
mc.config.Locking.Timeout = 1
|
||||
err = mc.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = mc.Lock()
|
||||
if err == nil {
|
||||
t.Fatal("should have failed, mongo should be locked already")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestTransaction(t *testing.T) {
|
||||
transactionSpecs := []dktesting.ContainerSpec{
|
||||
{ImageName: "mongo:4", Options: dktest.Options{PortRequired: true, ReadyFunc: isReady,
|
||||
Cmd: []string{"mongod", "--bind_ip_all", "--replSet", "rs0"}}},
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
for _, spec := range transactionSpecs {
|
||||
t.Log("Cleaning up ", spec.ImageName)
|
||||
if err := spec.Cleanup(); err != nil {
|
||||
t.Error("Error removing ", spec.ImageName, "error:", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
dktesting.ParallelTest(t, transactionSpecs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
| URL Query | WithInstance Config | Description |
|
||||
|------------|---------------------|-------------|
|
||||
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
|
||||
| `x-no-lock` | `NoLock` | Set to `true` to skip `GET_LOCK`/`RELEASE_LOCK` statements. Useful for [multi-master MySQL flavors](https://www.percona.com/doc/percona-xtradb-cluster/LATEST/features/pxc-strict-mode.html#explicit-table-locking). Only run migrations from one host when this is enabled. |
|
||||
| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds, functionally similar to [Server-side SELECT statement timeouts](https://dev.mysql.com/blog-archive/server-side-select-statement-timeouts/) but enforced by the client. Available for all versions of MySQL, not just >=5.7. |
|
||||
| `dbname` | `DatabaseName` | The name of the database to connect to |
|
||||
| `user` | | The user to sign in as |
|
||||
| `password` | | The user's password |
|
||||
@@ -14,7 +12,7 @@
|
||||
| `port` | | The port to bind to. |
|
||||
| `tls` | | TLS / SSL encrypted connection parameter; see [go-sql-driver](https://github.com/go-sql-driver/mysql#tls). Use any name (e.g. `migrate`) if you want to use a custom TLS config (`x-tls-` queries). |
|
||||
| `x-tls-ca` | | The location of the CA (certificate authority) file. |
|
||||
| `x-tls-cert` | | The location of the client certificate file. Must be used with `x-tls-key`. |
|
||||
| `x-tls-cert` | | The location of the client certicicate file. Must be used with `x-tls-key`. |
|
||||
| `x-tls-key` | | The location of the private key file. Must be used with `x-tls-cert`. |
|
||||
| `x-tls-insecure-skip-verify` | | Whether or not to use SSL (true\|false) |
|
||||
|
||||
@@ -29,9 +27,9 @@ import (
|
||||
"database/sql"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database/mysql"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
"github.com/golang-migrate/migrate"
|
||||
"github.com/golang-migrate/migrate/database/mysql"
|
||||
_ "github.com/golang-migrate/migrate/source/file"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
+137
-202
@@ -1,4 +1,4 @@
|
||||
//go:build go1.9
|
||||
// +build go1.9
|
||||
|
||||
package mysql
|
||||
|
||||
@@ -7,21 +7,22 @@ import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
)
|
||||
|
||||
var _ database.Driver = (*Mysql)(nil) // explicit compile time type check
|
||||
import (
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
)
|
||||
|
||||
import (
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Register("mysql", &Mysql{})
|
||||
@@ -34,14 +35,12 @@ var (
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
ErrAppendPEM = fmt.Errorf("failed to append PEM")
|
||||
ErrTLSCertKeyConfig = fmt.Errorf("to use TLS client authentication, both x-tls-cert and x-tls-key must not be empty")
|
||||
ErrTLSCertKeyConfig = fmt.Errorf("To use TLS client authentication, both x-tls-cert and x-tls-key must not be empty")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
NoLock bool
|
||||
StatementTimeout time.Duration
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
}
|
||||
|
||||
type Mysql struct {
|
||||
@@ -49,31 +48,25 @@ type Mysql struct {
|
||||
// just do everything over a single conn anyway.
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
|
||||
config *Config
|
||||
}
|
||||
|
||||
// connection instance must have `multiStatements` set to true
|
||||
func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Mysql, error) {
|
||||
// instance must have `multiStatements` set to true
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := conn.PingContext(ctx); err != nil {
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mx := &Mysql{
|
||||
conn: conn,
|
||||
db: nil,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if config.DatabaseName == "" {
|
||||
query := `SELECT DATABASE()`
|
||||
var databaseName sql.NullString
|
||||
if err := conn.QueryRowContext(ctx, query).Scan(&databaseName); err != nil {
|
||||
if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
@@ -88,6 +81,17 @@ func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Mysql
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
conn, err := instance.Conn(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mx := &Mysql{
|
||||
conn: conn,
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := mx.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -95,29 +99,6 @@ func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Mysql
|
||||
return mx, nil
|
||||
}
|
||||
|
||||
// instance must have `multiStatements` set to true
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, err := instance.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mx, err := WithConnection(ctx, conn, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mx.db = instance
|
||||
|
||||
return mx, nil
|
||||
}
|
||||
|
||||
// extractCustomQueryParams extracts the custom query params (ones that start with "x-") from
|
||||
// mysql.Config.Params (connection parameters) as to not interfere with connecting to MySQL
|
||||
func extractCustomQueryParams(c *mysql.Config) (map[string]string, error) {
|
||||
@@ -136,68 +117,6 @@ func extractCustomQueryParams(c *mysql.Config) (map[string]string, error) {
|
||||
}
|
||||
|
||||
func urlToMySQLConfig(url string) (*mysql.Config, error) {
|
||||
// Need to parse out custom TLS parameters and call
|
||||
// mysql.RegisterTLSConfig() before mysql.ParseDSN() is called
|
||||
// which consumes the registered tls.Config
|
||||
// Fixes: https://github.com/golang-migrate/migrate/issues/411
|
||||
//
|
||||
// Can't use url.Parse() since it fails to parse MySQL DSNs
|
||||
// mysql.ParseDSN() also searches for "?" to find query parameters:
|
||||
// https://github.com/go-sql-driver/mysql/blob/46351a8/dsn.go#L344
|
||||
if idx := strings.LastIndex(url, "?"); idx > 0 {
|
||||
rawParams := url[idx+1:]
|
||||
parsedParams, err := nurl.ParseQuery(rawParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctls := parsedParams.Get("tls")
|
||||
if len(ctls) > 0 {
|
||||
if _, isBool := readBool(ctls); !isBool && strings.ToLower(ctls) != "skip-verify" {
|
||||
rootCertPool := x509.NewCertPool()
|
||||
pem, err := os.ReadFile(parsedParams.Get("x-tls-ca"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
|
||||
return nil, ErrAppendPEM
|
||||
}
|
||||
|
||||
clientCert := make([]tls.Certificate, 0, 1)
|
||||
if ccert, ckey := parsedParams.Get("x-tls-cert"), parsedParams.Get("x-tls-key"); ccert != "" || ckey != "" {
|
||||
if ccert == "" || ckey == "" {
|
||||
return nil, ErrTLSCertKeyConfig
|
||||
}
|
||||
certs, err := tls.LoadX509KeyPair(ccert, ckey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clientCert = append(clientCert, certs)
|
||||
}
|
||||
|
||||
insecureSkipVerify := false
|
||||
insecureSkipVerifyStr := parsedParams.Get("x-tls-insecure-skip-verify")
|
||||
if len(insecureSkipVerifyStr) > 0 {
|
||||
x, err := strconv.ParseBool(insecureSkipVerifyStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
insecureSkipVerify = x
|
||||
}
|
||||
|
||||
err = mysql.RegisterTLSConfig(ctls, &tls.Config{
|
||||
RootCAs: rootCertPool,
|
||||
Certificates: clientCert,
|
||||
InsecureSkipVerify: insecureSkipVerify,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
config, err := mysql.ParseDSN(strings.TrimPrefix(url, "mysql://"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -220,6 +139,52 @@ func urlToMySQLConfig(url string) (*mysql.Config, error) {
|
||||
}
|
||||
config.Passwd = password
|
||||
|
||||
// use custom TLS?
|
||||
ctls := config.TLSConfig
|
||||
if len(ctls) > 0 {
|
||||
if _, isBool := readBool(ctls); !isBool && strings.ToLower(ctls) != "skip-verify" {
|
||||
rootCertPool := x509.NewCertPool()
|
||||
pem, err := ioutil.ReadFile(config.Params["x-tls-ca"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
|
||||
return nil, ErrAppendPEM
|
||||
}
|
||||
|
||||
clientCert := make([]tls.Certificate, 0, 1)
|
||||
if ccert, ckey := config.Params["x-tls-cert"], config.Params["x-tls-key"]; ccert != "" || ckey != "" {
|
||||
if ccert == "" || ckey == "" {
|
||||
return nil, ErrTLSCertKeyConfig
|
||||
}
|
||||
certs, err := tls.LoadX509KeyPair(ccert, ckey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
clientCert = append(clientCert, certs)
|
||||
}
|
||||
|
||||
insecureSkipVerify := false
|
||||
if len(config.Params["x-tls-insecure-skip-verify"]) > 0 {
|
||||
x, err := strconv.ParseBool(config.Params["x-tls-insecure-skip-verify"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
insecureSkipVerify = x
|
||||
}
|
||||
|
||||
err = mysql.RegisterTLSConfig(ctls, &tls.Config{
|
||||
RootCAs: rootCertPool,
|
||||
Certificates: clientCert,
|
||||
InsecureSkipVerify: insecureSkipVerify,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
@@ -234,33 +199,14 @@ func (m *Mysql) Open(url string) (database.Driver, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
noLockParam, noLock := customParams["x-no-lock"], false
|
||||
if noLockParam != "" {
|
||||
noLock, err = strconv.ParseBool(noLockParam)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse x-no-lock as bool: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
statementTimeoutParam := customParams["x-statement-timeout"]
|
||||
statementTimeout := 0
|
||||
if statementTimeoutParam != "" {
|
||||
statementTimeout, err = strconv.Atoi(statementTimeoutParam)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not parse x-statement-timeout as float: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
db, err := sql.Open("mysql", config.FormatDSN())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mx, err := WithInstance(db, &Config{
|
||||
DatabaseName: config.DBName,
|
||||
MigrationsTable: customParams["x-migrations-table"],
|
||||
NoLock: noLock,
|
||||
StatementTimeout: time.Duration(statementTimeout) * time.Millisecond,
|
||||
DatabaseName: config.DBName,
|
||||
MigrationsTable: customParams["x-migrations-table"],
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -271,11 +217,7 @@ func (m *Mysql) Open(url string) (database.Driver, error) {
|
||||
|
||||
func (m *Mysql) Close() error {
|
||||
connErr := m.conn.Close()
|
||||
var dbErr error
|
||||
if m.db != nil {
|
||||
dbErr = m.db.Close()
|
||||
}
|
||||
|
||||
dbErr := m.db.Close()
|
||||
if connErr != nil || dbErr != nil {
|
||||
return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
|
||||
}
|
||||
@@ -283,70 +225,62 @@ func (m *Mysql) Close() error {
|
||||
}
|
||||
|
||||
func (m *Mysql) Lock() error {
|
||||
return database.CasRestoreOnErr(&m.isLocked, false, true, database.ErrLocked, func() error {
|
||||
if m.config.NoLock {
|
||||
return nil
|
||||
}
|
||||
aid, err := database.GenerateAdvisoryLockId(
|
||||
fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if m.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
query := "SELECT GET_LOCK(?, 10)"
|
||||
var success bool
|
||||
if err := m.conn.QueryRowContext(context.Background(), query, aid).Scan(&success); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
|
||||
}
|
||||
|
||||
if !success {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Mysql) Unlock() error {
|
||||
return database.CasRestoreOnErr(&m.isLocked, true, false, database.ErrNotLocked, func() error {
|
||||
if m.config.NoLock {
|
||||
return nil
|
||||
}
|
||||
|
||||
aid, err := database.GenerateAdvisoryLockId(
|
||||
fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := `SELECT RELEASE_LOCK(?)`
|
||||
if _, err := m.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// NOTE: RELEASE_LOCK could return NULL or (or 0 if the code is changed),
|
||||
// in which case isLocked should be true until the timeout expires -- synchronizing
|
||||
// these states is likely not worth trying to do; reconsider the necessity of isLocked.
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Mysql) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
aid, err := database.GenerateAdvisoryLockId(
|
||||
fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if m.config.StatementTimeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, m.config.StatementTimeout)
|
||||
defer cancel()
|
||||
query := "SELECT GET_LOCK(?, 10)"
|
||||
var success bool
|
||||
if err := m.conn.QueryRowContext(context.Background(), query, aid).Scan(&success); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
|
||||
}
|
||||
|
||||
if success {
|
||||
m.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
func (m *Mysql) Unlock() error {
|
||||
if !m.isLocked {
|
||||
return nil
|
||||
}
|
||||
|
||||
aid, err := database.GenerateAdvisoryLockId(
|
||||
fmt.Sprintf("%s:%s", m.config.DatabaseName, m.config.MigrationsTable))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := `SELECT RELEASE_LOCK(?)`
|
||||
if _, err := m.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// NOTE: RELEASE_LOCK could return NULL or (or 0 if the code is changed),
|
||||
// in which case isLocked should be true until the timeout expires -- synchronizing
|
||||
// these states is likely not worth trying to do; reconsider the necessity of isLocked.
|
||||
|
||||
m.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Mysql) Run(migration io.Reader) error {
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := string(migr[:])
|
||||
if _, err := m.conn.ExecContext(ctx, query); err != nil {
|
||||
if _, err := m.conn.ExecContext(context.Background(), query); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
|
||||
}
|
||||
|
||||
@@ -354,15 +288,15 @@ func (m *Mysql) Run(migration io.Reader) error {
|
||||
}
|
||||
|
||||
func (m *Mysql) SetVersion(version int, dirty bool) error {
|
||||
tx, err := m.conn.BeginTx(context.Background(), &sql.TxOptions{Isolation: sql.LevelSerializable})
|
||||
tx, err := m.conn.BeginTx(context.Background(), &sql.TxOptions{})
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
|
||||
query := "DELETE FROM `" + m.config.MigrationsTable + "` LIMIT 1"
|
||||
query := "TRUNCATE `" + m.config.MigrationsTable + "`"
|
||||
if _, err := tx.ExecContext(context.Background(), query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -374,7 +308,7 @@ func (m *Mysql) SetVersion(version int, dirty bool) error {
|
||||
query := "INSERT INTO `" + m.config.MigrationsTable + "` (version, dirty) VALUES (?, ?)"
|
||||
if _, err := tx.ExecContext(context.Background(), query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -416,7 +350,7 @@ func (m *Mysql) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -431,9 +365,6 @@ func (m *Mysql) Drop() (err error) {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// disable checking foreign key constraints until finished
|
||||
@@ -469,13 +400,17 @@ func (m *Mysql) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := m.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// check if migration table exists
|
||||
var result string
|
||||
query := `SHOW TABLES LIKE '` + m.config.MigrationsTable + `'`
|
||||
query := `SHOW TABLES LIKE "` + m.config.MigrationsTable + `"`
|
||||
if err := m.conn.QueryRowContext(context.Background(), query).Scan(&result); err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
|
||||
+13
-153
@@ -2,28 +2,24 @@ package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/x509"
|
||||
"database/sql"
|
||||
sqldriver "database/sql/driver"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
import (
|
||||
"github.com/dhui/dktest"
|
||||
"github.com/go-sql-driver/mysql"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
import (
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
"github.com/golang-migrate/migrate/v4/dktesting"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const defaultPort = 3306
|
||||
@@ -33,21 +29,12 @@ var (
|
||||
Env: map[string]string{"MYSQL_ROOT_PASSWORD": "root", "MYSQL_DATABASE": "public"},
|
||||
PortRequired: true, ReadyFunc: isReady,
|
||||
}
|
||||
optsAnsiQuotes = dktest.Options{
|
||||
Env: map[string]string{"MYSQL_ROOT_PASSWORD": "root", "MYSQL_DATABASE": "public"},
|
||||
PortRequired: true, ReadyFunc: isReady,
|
||||
Cmd: []string{"--sql-mode=ANSI_QUOTES"},
|
||||
}
|
||||
// Supported versions: https://www.mysql.com/support/supportedplatforms/database.html
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "mysql:8.0", Options: opts},
|
||||
{ImageName: "mysql:8.4", Options: opts},
|
||||
{ImageName: "mysql:9.0", Options: opts},
|
||||
}
|
||||
specsAnsiQuotes = []dktesting.ContainerSpec{
|
||||
{ImageName: "mysql:8.0", Options: optsAnsiQuotes},
|
||||
{ImageName: "mysql:8.4", Options: optsAnsiQuotes},
|
||||
{ImageName: "mysql:9.0", Options: optsAnsiQuotes},
|
||||
{ImageName: "mysql:5.5", Options: opts},
|
||||
{ImageName: "mysql:5.6", Options: opts},
|
||||
{ImageName: "mysql:5.7", Options: opts},
|
||||
{ImageName: "mysql:8", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -80,7 +67,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
// mysql.SetLogger(mysql.Logger(log.New(io.Discard, "", log.Ltime)))
|
||||
// mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime)))
|
||||
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
@@ -113,7 +100,7 @@ func Test(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
// mysql.SetLogger(mysql.Logger(log.New(io.Discard, "", log.Ltime)))
|
||||
// mysql.SetLogger(mysql.Logger(log.New(ioutil.Discard, "", log.Ltime)))
|
||||
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
@@ -150,44 +137,6 @@ func TestMigrate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateAnsiQuotes(t *testing.T) {
|
||||
// mysql.SetLogger(mysql.Logger(log.New(io.Discard, "", log.Ltime)))
|
||||
|
||||
dktesting.ParallelTest(t, specsAnsiQuotes, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port)
|
||||
p := &Mysql{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "public", d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
|
||||
// check ensureVersionTable
|
||||
if err := d.(*Mysql).ensureVersionTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// check again
|
||||
if err := d.(*Mysql).ensureVersionTable(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestLockWorks(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
@@ -226,57 +175,6 @@ func TestLockWorks(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestNoLockParamValidation(t *testing.T) {
|
||||
ip := "127.0.0.1"
|
||||
port := 3306
|
||||
addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port)
|
||||
p := &Mysql{}
|
||||
_, err := p.Open(addr + "?x-no-lock=not-a-bool")
|
||||
if !errors.Is(err, strconv.ErrSyntax) {
|
||||
t.Fatal("Expected syntax error when passing a non-bool as x-no-lock parameter")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoLockWorks(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("mysql://root:root@tcp(%v:%v)/public", ip, port)
|
||||
p := &Mysql{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
lock := d.(*Mysql)
|
||||
|
||||
p = &Mysql{}
|
||||
d, err = p.Open(addr + "?x-no-lock=true")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
noLock := d.(*Mysql)
|
||||
|
||||
// Should be possible to take real lock and no-lock at the same time
|
||||
if err = lock.Lock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = noLock.Lock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = lock.Unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = noLock.Unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestExtractCustomQueryParams(t *testing.T) {
|
||||
testcases := []struct {
|
||||
name string
|
||||
@@ -333,42 +231,7 @@ func TestExtractCustomQueryParams(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func createTmpCert(t *testing.T) string {
|
||||
tmpCertFile, err := os.CreateTemp("", "migrate_test_cert")
|
||||
if err != nil {
|
||||
t.Fatal("Failed to create temp cert file:", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := os.Remove(tmpCertFile.Name()); err != nil {
|
||||
t.Log("Failed to cleanup temp cert file:", err)
|
||||
}
|
||||
})
|
||||
|
||||
r := rand.New(rand.NewSource(0))
|
||||
pub, priv, err := ed25519.GenerateKey(r)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to generate ed25519 key for temp cert file:", err)
|
||||
}
|
||||
tmpl := x509.Certificate{
|
||||
SerialNumber: big.NewInt(0),
|
||||
}
|
||||
derBytes, err := x509.CreateCertificate(r, &tmpl, &tmpl, pub, priv)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to generate temp cert file:", err)
|
||||
}
|
||||
if err := pem.Encode(tmpCertFile, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
|
||||
t.Fatal("Failed to encode ")
|
||||
}
|
||||
if err := tmpCertFile.Close(); err != nil {
|
||||
t.Fatal("Failed to close temp cert file:", err)
|
||||
}
|
||||
return tmpCertFile.Name()
|
||||
}
|
||||
|
||||
func TestURLToMySQLConfig(t *testing.T) {
|
||||
tmpCertFilename := createTmpCert(t)
|
||||
tmpCertFilenameEscaped := url.PathEscape(tmpCertFilename)
|
||||
|
||||
testcases := []struct {
|
||||
name string
|
||||
urlStr string
|
||||
@@ -389,7 +252,7 @@ func TestURLToMySQLConfig(t *testing.T) {
|
||||
// Not supported yet: https://github.com/go-sql-driver/mysql/issues/591
|
||||
// {name: "user/password - user with encoded :",
|
||||
// urlStr: "mysql://username%3A:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
|
||||
// expectedDSN: "username::password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
|
||||
// expectedDSN: "username::pasword@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
|
||||
{name: "user/password - user with encoded @",
|
||||
urlStr: "mysql://username%40:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
|
||||
expectedDSN: "username@:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
|
||||
@@ -399,9 +262,6 @@ func TestURLToMySQLConfig(t *testing.T) {
|
||||
{name: "user/password - password with encoded @",
|
||||
urlStr: "mysql://username:password%40@tcp(127.0.0.1:3306)/myDB?multiStatements=true",
|
||||
expectedDSN: "username:password@@tcp(127.0.0.1:3306)/myDB?multiStatements=true"},
|
||||
{name: "custom tls",
|
||||
urlStr: "mysql://username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true&tls=custom&x-tls-ca=" + tmpCertFilenameEscaped,
|
||||
expectedDSN: "username:password@tcp(127.0.0.1:3306)/myDB?multiStatements=true&tls=custom&x-tls-ca=" + tmpCertFilenameEscaped},
|
||||
}
|
||||
for _, tc := range testcases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
@@ -2,15 +2,16 @@ package neo4j
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
neturl "net/url"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/neo4j/neo4j-go-driver/neo4j"
|
||||
)
|
||||
|
||||
@@ -141,7 +142,7 @@ func (n *Neo4j) Run(migration io.Reader) (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if cerr := session.Close(); cerr != nil {
|
||||
err = errors.Join(err, cerr)
|
||||
err = multierror.Append(err, cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -172,7 +173,7 @@ func (n *Neo4j) Run(migration io.Reader) (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(migration)
|
||||
body, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -188,7 +189,7 @@ func (n *Neo4j) SetVersion(version int, dirty bool) (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if cerr := session.Close(); cerr != nil {
|
||||
err = errors.Join(err, cerr)
|
||||
err = multierror.Append(err, cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -213,7 +214,7 @@ func (n *Neo4j) Version() (version int, dirty bool, err error) {
|
||||
}
|
||||
defer func() {
|
||||
if cerr := session.Close(); cerr != nil {
|
||||
err = errors.Join(err, cerr)
|
||||
err = multierror.Append(err, cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -261,7 +262,7 @@ func (n *Neo4j) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if cerr := session.Close(); cerr != nil {
|
||||
err = errors.Join(err, cerr)
|
||||
err = multierror.Append(err, cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -278,7 +279,7 @@ func (n *Neo4j) ensureVersionConstraint() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if cerr := session.Close(); cerr != nil {
|
||||
err = errors.Join(err, cerr)
|
||||
err = multierror.Append(err, cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -8,15 +8,13 @@ import (
|
||||
)
|
||||
|
||||
const reservedChars = "!#$%&'()*+,/:;=?@[]"
|
||||
const reservedCharTestNamePrefix = "reserved char "
|
||||
|
||||
const baseUsername = "username"
|
||||
|
||||
const scheme = "database://"
|
||||
|
||||
// TestUserUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in usernames with
|
||||
// net/url Parse()
|
||||
func TestUserUnencodedReservedURLChars(t *testing.T) {
|
||||
scheme := "database://"
|
||||
urlSuffix := "password@localhost:12345/myDB?someParam=true"
|
||||
urlSuffixAndSep := ":" + urlSuffix
|
||||
|
||||
@@ -66,7 +64,7 @@ func TestUserUnencodedReservedURLChars(t *testing.T) {
|
||||
testedChars := make([]string, 0, len(reservedChars))
|
||||
for _, tc := range testcases {
|
||||
testedChars = append(testedChars, tc.char)
|
||||
t.Run(reservedCharTestNamePrefix+tc.char, func(t *testing.T) {
|
||||
t.Run("reserved char "+tc.char, func(t *testing.T) {
|
||||
s := scheme + baseUsername + tc.char + urlSuffixAndSep
|
||||
u, err := url.Parse(s)
|
||||
if err == nil {
|
||||
@@ -100,12 +98,13 @@ func TestUserUnencodedReservedURLChars(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestUserEncodedReservedURLChars(t *testing.T) {
|
||||
scheme := "database://"
|
||||
urlSuffix := "password@localhost:12345/myDB?someParam=true"
|
||||
urlSuffixAndSep := ":" + urlSuffix
|
||||
|
||||
for _, c := range reservedChars {
|
||||
c := string(c)
|
||||
t.Run(reservedCharTestNamePrefix+c, func(t *testing.T) {
|
||||
t.Run("reserved char "+c, func(t *testing.T) {
|
||||
encodedChar := "%" + hex.EncodeToString([]byte(c))
|
||||
s := scheme + baseUsername + encodedChar + urlSuffixAndSep
|
||||
expectedUsername := baseUsername + c
|
||||
@@ -127,7 +126,7 @@ func TestUserEncodedReservedURLChars(t *testing.T) {
|
||||
// with net/url Parse()
|
||||
func TestPasswordUnencodedReservedURLChars(t *testing.T) {
|
||||
username := baseUsername
|
||||
schemeAndUsernameAndSep := scheme + username + ":"
|
||||
schemeAndUsernameAndSep := "database://" + username + ":"
|
||||
basePassword := "password"
|
||||
urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
|
||||
|
||||
@@ -175,7 +174,7 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) {
|
||||
testedChars := make([]string, 0, len(reservedChars))
|
||||
for _, tc := range testcases {
|
||||
testedChars = append(testedChars, tc.char)
|
||||
t.Run(reservedCharTestNamePrefix+tc.char, func(t *testing.T) {
|
||||
t.Run("reserved char "+tc.char, func(t *testing.T) {
|
||||
s := schemeAndUsernameAndSep + basePassword + tc.char + urlSuffixAndSep
|
||||
u, err := url.Parse(s)
|
||||
if err == nil {
|
||||
@@ -214,13 +213,13 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) {
|
||||
|
||||
func TestPasswordEncodedReservedURLChars(t *testing.T) {
|
||||
username := baseUsername
|
||||
schemeAndUsernameAndSep := scheme + username + ":"
|
||||
schemeAndUsernameAndSep := "database://" + username + ":"
|
||||
basePassword := "password"
|
||||
urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
|
||||
|
||||
for _, c := range reservedChars {
|
||||
c := string(c)
|
||||
t.Run(reservedCharTestNamePrefix+c, func(t *testing.T) {
|
||||
t.Run("reserved char "+c, func(t *testing.T) {
|
||||
encodedChar := "%" + hex.EncodeToString([]byte(c))
|
||||
s := schemeAndUsernameAndSep + basePassword + encodedChar + urlSuffixAndSep
|
||||
expectedPassword := basePassword + c
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# pgx
|
||||
|
||||
This package is for [pgx/v4](https://pkg.go.dev/github.com/jackc/pgx/v4). A backend for the newer [pgx/v5](https://pkg.go.dev/github.com/jackc/pgx/v5) is [also available](v5).
|
||||
|
||||
`pgx://user:password@host:port/dbname?query`
|
||||
|
||||
| URL Query | WithInstance Config | Description |
|
||||
|------------|---------------------|-------------|
|
||||
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
|
||||
| `x-migrations-table-quoted` | `MigrationsTableQuoted` | By default, migrate quotes the migration table for SQL injection safety reasons. This option disable quoting and naively checks that you have quoted the migration table name. e.g. `"my_schema"."schema_migrations"` |
|
||||
| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds |
|
||||
| `x-multi-statement` | `MultiStatementEnabled` | Enable multi-statement execution (default: false) |
|
||||
| `x-multi-statement-max-size` | `MultiStatementMaxSize` | Maximum size of single statement in bytes (default: 10MB) |
|
||||
| `x-lock-strategy` | `LockStrategy` | Strategy used for locking during migration (default: advisory) |
|
||||
| `x-lock-table` | `LockTable` | Name of the table which maintains the migration lock (default: schema_lock) |
|
||||
| `dbname` | `DatabaseName` | The name of the database to connect to |
|
||||
| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. |
|
||||
| `user` | | The user to sign in as |
|
||||
| `password` | | The user's password |
|
||||
| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) |
|
||||
| `port` | | The port to bind to. (default is 5432) |
|
||||
| `fallback_application_name` | | An application_name to fall back to if one isn't provided. |
|
||||
| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. |
|
||||
| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
|
||||
| `sslkey` | | Key file location. The file must contain PEM encoded data. |
|
||||
| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
|
||||
| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
|
||||
|
||||
|
||||
## Upgrading from v1
|
||||
|
||||
1. Write down the current migration version from schema_migrations
|
||||
1. `DROP TABLE schema_migrations`
|
||||
2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration.
|
||||
3. Download and install the latest migrate version.
|
||||
4. Force the current migration version with `migrate force <current_version>`.
|
||||
|
||||
## Multi-statement mode
|
||||
|
||||
In PostgreSQL running multiple SQL statements in one `Exec` executes them inside a transaction. Sometimes this
|
||||
behavior is not desirable because some statements can be only run outside of transaction (e.g.
|
||||
`CREATE INDEX CONCURRENTLY`). If you want to use `CREATE INDEX CONCURRENTLY` without activating multi-statement mode
|
||||
you have to put such statements in a separate migration files.
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS users;
|
||||
@@ -1,5 +0,0 @@
|
||||
CREATE TABLE users (
|
||||
user_id integer unique,
|
||||
name varchar(40),
|
||||
email varchar(40)
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE users DROP COLUMN IF EXISTS city;
|
||||
@@ -1,3 +0,0 @@
|
||||
ALTER TABLE users ADD COLUMN city varchar(100);
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
DROP INDEX IF EXISTS users_email_index;
|
||||
@@ -1,3 +0,0 @@
|
||||
CREATE UNIQUE INDEX CONCURRENTLY users_email_index ON users (email);
|
||||
|
||||
-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS books;
|
||||
@@ -1,5 +0,0 @@
|
||||
CREATE TABLE books (
|
||||
user_id integer,
|
||||
name varchar(40),
|
||||
author varchar(40)
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS movies;
|
||||
@@ -1,5 +0,0 @@
|
||||
CREATE TABLE movies (
|
||||
user_id integer,
|
||||
name varchar(40),
|
||||
director varchar(40)
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.
|
||||
@@ -1 +0,0 @@
|
||||
-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.
|
||||
@@ -1 +0,0 @@
|
||||
-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.
|
||||
@@ -1 +0,0 @@
|
||||
-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere.
|
||||
@@ -1,617 +0,0 @@
|
||||
//go:build go1.9
|
||||
|
||||
package pgx
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
"github.com/jackc/pgconn"
|
||||
"github.com/jackc/pgerrcode"
|
||||
_ "github.com/jackc/pgx/v4/stdlib"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
const (
|
||||
LockStrategyAdvisory = "advisory"
|
||||
LockStrategyTable = "table"
|
||||
)
|
||||
|
||||
func init() {
|
||||
db := Postgres{}
|
||||
database.Register("pgx", &db)
|
||||
database.Register("pgx4", &db)
|
||||
}
|
||||
|
||||
var (
|
||||
multiStmtDelimiter = []byte(";")
|
||||
|
||||
DefaultMigrationsTable = "schema_migrations"
|
||||
DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB
|
||||
DefaultLockTable = "schema_lock"
|
||||
DefaultLockStrategy = LockStrategyAdvisory
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
ErrNoSchema = fmt.Errorf("no schema")
|
||||
ErrDatabaseDirty = fmt.Errorf("database is dirty")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
SchemaName string
|
||||
LockTable string
|
||||
LockStrategy string
|
||||
migrationsSchemaName string
|
||||
migrationsTableName string
|
||||
StatementTimeout time.Duration
|
||||
MigrationsTableQuoted bool
|
||||
MultiStatementEnabled bool
|
||||
MultiStatementMaxSize int
|
||||
}
|
||||
|
||||
type Postgres struct {
|
||||
// Locking and unlocking need to use the same connection
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
|
||||
// Open and WithInstance need to guarantee that config is never nil
|
||||
config *Config
|
||||
}
|
||||
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if config.DatabaseName == "" {
|
||||
query := `SELECT CURRENT_DATABASE()`
|
||||
var databaseName string
|
||||
if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(databaseName) == 0 {
|
||||
return nil, ErrNoDatabaseName
|
||||
}
|
||||
|
||||
config.DatabaseName = databaseName
|
||||
}
|
||||
|
||||
if config.SchemaName == "" {
|
||||
query := `SELECT CURRENT_SCHEMA()`
|
||||
var schemaName string
|
||||
if err := instance.QueryRow(query).Scan(&schemaName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(schemaName) == 0 {
|
||||
return nil, ErrNoSchema
|
||||
}
|
||||
|
||||
config.SchemaName = schemaName
|
||||
}
|
||||
|
||||
if len(config.MigrationsTable) == 0 {
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
if len(config.LockTable) == 0 {
|
||||
config.LockTable = DefaultLockTable
|
||||
}
|
||||
|
||||
if len(config.LockStrategy) == 0 {
|
||||
config.LockStrategy = DefaultLockStrategy
|
||||
}
|
||||
|
||||
config.migrationsSchemaName = config.SchemaName
|
||||
config.migrationsTableName = config.MigrationsTable
|
||||
if config.MigrationsTableQuoted {
|
||||
re := regexp.MustCompile(`"(.*?)"`)
|
||||
result := re.FindAllStringSubmatch(config.MigrationsTable, -1)
|
||||
config.migrationsTableName = result[len(result)-1][1]
|
||||
if len(result) == 2 {
|
||||
config.migrationsSchemaName = result[0][1]
|
||||
} else if len(result) > 2 {
|
||||
return nil, fmt.Errorf("\"%s\" MigrationsTable contains too many dot characters", config.MigrationsTable)
|
||||
}
|
||||
}
|
||||
|
||||
conn, err := instance.Conn(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
px := &Postgres{
|
||||
conn: conn,
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := px.ensureLockTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := px.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return px, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Open(url string) (database.Driver, error) {
|
||||
purl, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Driver is registered as pgx, but connection string must use postgres schema
|
||||
// when making actual connection
|
||||
// i.e. pgx://user:password@host:port/db => postgres://user:password@host:port/db
|
||||
purl.Scheme = "postgres"
|
||||
|
||||
db, err := sql.Open("pgx/v4", migrate.FilterCustomQuery(purl).String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
migrationsTable := purl.Query().Get("x-migrations-table")
|
||||
migrationsTableQuoted := false
|
||||
if s := purl.Query().Get("x-migrations-table-quoted"); len(s) > 0 {
|
||||
migrationsTableQuoted, err = strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse option x-migrations-table-quoted: %w", err)
|
||||
}
|
||||
}
|
||||
if (len(migrationsTable) > 0) && (migrationsTableQuoted) && ((migrationsTable[0] != '"') || (migrationsTable[len(migrationsTable)-1] != '"')) {
|
||||
return nil, fmt.Errorf("x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: %s", migrationsTable)
|
||||
}
|
||||
|
||||
statementTimeoutString := purl.Query().Get("x-statement-timeout")
|
||||
statementTimeout := 0
|
||||
if statementTimeoutString != "" {
|
||||
statementTimeout, err = strconv.Atoi(statementTimeoutString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
multiStatementMaxSize := DefaultMultiStatementMaxSize
|
||||
if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 {
|
||||
multiStatementMaxSize, err = strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if multiStatementMaxSize <= 0 {
|
||||
multiStatementMaxSize = DefaultMultiStatementMaxSize
|
||||
}
|
||||
}
|
||||
|
||||
multiStatementEnabled := false
|
||||
if s := purl.Query().Get("x-multi-statement"); len(s) > 0 {
|
||||
multiStatementEnabled, err = strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse option x-multi-statement: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
lockStrategy := purl.Query().Get("x-lock-strategy")
|
||||
lockTable := purl.Query().Get("x-lock-table")
|
||||
|
||||
px, err := WithInstance(db, &Config{
|
||||
DatabaseName: purl.Path,
|
||||
MigrationsTable: migrationsTable,
|
||||
MigrationsTableQuoted: migrationsTableQuoted,
|
||||
StatementTimeout: time.Duration(statementTimeout) * time.Millisecond,
|
||||
MultiStatementEnabled: multiStatementEnabled,
|
||||
MultiStatementMaxSize: multiStatementMaxSize,
|
||||
LockStrategy: lockStrategy,
|
||||
LockTable: lockTable,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return px, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Close() error {
|
||||
connErr := p.conn.Close()
|
||||
dbErr := p.db.Close()
|
||||
if connErr != nil || dbErr != nil {
|
||||
return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Lock() error {
|
||||
return database.CasRestoreOnErr(&p.isLocked, false, true, database.ErrLocked, func() error {
|
||||
switch p.config.LockStrategy {
|
||||
case LockStrategyAdvisory:
|
||||
return p.applyAdvisoryLock()
|
||||
case LockStrategyTable:
|
||||
return p.applyTableLock()
|
||||
default:
|
||||
return fmt.Errorf("unknown lock strategy \"%s\"", p.config.LockStrategy)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Postgres) Unlock() error {
|
||||
return database.CasRestoreOnErr(&p.isLocked, true, false, database.ErrNotLocked, func() error {
|
||||
switch p.config.LockStrategy {
|
||||
case LockStrategyAdvisory:
|
||||
return p.releaseAdvisoryLock()
|
||||
case LockStrategyTable:
|
||||
return p.releaseTableLock()
|
||||
default:
|
||||
return fmt.Errorf("unknown lock strategy \"%s\"", p.config.LockStrategy)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS
|
||||
func (p *Postgres) applyAdvisoryLock() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// This will wait indefinitely until the lock can be acquired.
|
||||
query := `SELECT pg_advisory_lock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) applyTableLock() error {
|
||||
tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{})
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
defer func() {
|
||||
errRollback := tx.Rollback()
|
||||
if errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
}()
|
||||
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := "SELECT * FROM " + pq.QuoteIdentifier(p.config.LockTable) + " WHERE lock_id = $1"
|
||||
rows, err := tx.Query(query, aid)
|
||||
if err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)}
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if errClose := rows.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
// If row exists at all, lock is present
|
||||
locked := rows.Next()
|
||||
if locked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
|
||||
query = "INSERT INTO " + pq.QuoteIdentifier(p.config.LockTable) + " (lock_id) VALUES ($1)"
|
||||
if _, err := tx.Exec(query, aid); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (p *Postgres) releaseAdvisoryLock() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := `SELECT pg_advisory_unlock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) releaseTableLock() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := "DELETE FROM " + pq.QuoteIdentifier(p.config.LockTable) + " WHERE lock_id = $1"
|
||||
if _, err := p.db.Exec(query, aid); err != nil {
|
||||
return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Run(migration io.Reader) error {
|
||||
if p.config.MultiStatementEnabled {
|
||||
var err error
|
||||
if e := multistmt.Parse(migration, multiStmtDelimiter, p.config.MultiStatementMaxSize, func(m []byte) bool {
|
||||
if err = p.runStatement(m); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); e != nil {
|
||||
return e
|
||||
}
|
||||
return err
|
||||
}
|
||||
migr, err := io.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return p.runStatement(migr)
|
||||
}
|
||||
|
||||
func (p *Postgres) runStatement(statement []byte) error {
|
||||
ctx := context.Background()
|
||||
if p.config.StatementTimeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, p.config.StatementTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
query := string(statement)
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := p.conn.ExecContext(ctx, query); err != nil {
|
||||
|
||||
if pgErr, ok := err.(*pgconn.PgError); ok {
|
||||
var line uint
|
||||
var col uint
|
||||
var lineColOK bool
|
||||
line, col, lineColOK = computeLineFromPos(query, int(pgErr.Position))
|
||||
message := fmt.Sprintf("migration failed: %s", pgErr.Message)
|
||||
if lineColOK {
|
||||
message = fmt.Sprintf("%s (column %d)", message, col)
|
||||
}
|
||||
if pgErr.Detail != "" {
|
||||
message = fmt.Sprintf("%s, %s", message, pgErr.Detail)
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: message, Query: statement, Line: line}
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: statement}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) {
|
||||
// replace crlf with lf
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
// pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes
|
||||
runes := []rune(s)
|
||||
if pos > len(runes) {
|
||||
return 0, 0, false
|
||||
}
|
||||
sel := runes[:pos]
|
||||
line = uint(runesCount(sel, newLine) + 1)
|
||||
col = uint(pos - 1 - runesLastIndex(sel, newLine))
|
||||
return line, col, true
|
||||
}
|
||||
|
||||
const newLine = '\n'
|
||||
|
||||
func runesCount(input []rune, target rune) int {
|
||||
var count int
|
||||
for _, r := range input {
|
||||
if r == target {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func runesLastIndex(input []rune, target rune) int {
|
||||
for i := len(input) - 1; i >= 0; i-- {
|
||||
if input[i] == target {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (p *Postgres) SetVersion(version int, dirty bool) error {
|
||||
tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{})
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
|
||||
query := `TRUNCATE ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName)
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// Also re-write the schema version for nil dirty versions to prevent
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
query = `INSERT INTO ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` (version, dirty) VALUES ($1, $2)`
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction commit failed"}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Version() (version int, dirty bool, err error) {
|
||||
query := `SELECT version, dirty FROM ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` LIMIT 1`
|
||||
err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty)
|
||||
switch {
|
||||
case err == sql.ErrNoRows:
|
||||
return database.NilVersion, false, nil
|
||||
|
||||
case err != nil:
|
||||
if e, ok := err.(*pgconn.PgError); ok {
|
||||
if e.SQLState() == pgerrcode.UndefinedTable {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
}
|
||||
return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
|
||||
default:
|
||||
return version, dirty, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Postgres) Drop() (err error) {
|
||||
// select all tables in current schema
|
||||
query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'`
|
||||
tables, err := p.conn.QueryContext(context.Background(), query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
// delete one table after another
|
||||
tableNames := make([]string, 0)
|
||||
for tables.Next() {
|
||||
var tableName string
|
||||
if err := tables.Scan(&tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// do not drop lock table
|
||||
if tableName == p.config.LockTable && p.config.LockStrategy == LockStrategyTable {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(tableName) > 0 {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// delete one by one ...
|
||||
for _, t := range tableNames {
|
||||
query = `DROP TABLE IF EXISTS ` + quoteIdentifier(t) + ` CASCADE`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the Postgres type.
|
||||
func (p *Postgres) ensureVersionTable() (err error) {
|
||||
if err = p.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := p.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
}
|
||||
}()
|
||||
|
||||
// This block checks whether the `MigrationsTable` already exists. This is useful because it allows read only postgres
|
||||
// users to also check the current version of the schema. Previously, even if `MigrationsTable` existed, the
|
||||
// `CREATE TABLE IF NOT EXISTS...` query would fail because the user does not have the CREATE permission.
|
||||
// Taken from https://github.com/mattes/migrate/blob/master/database/postgres/postgres.go#L258
|
||||
query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2 LIMIT 1`
|
||||
row := p.conn.QueryRowContext(context.Background(), query, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
|
||||
var count int
|
||||
err = row.Scan(&count)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if count == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query = `CREATE TABLE IF NOT EXISTS ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` (version bigint not null primary key, dirty boolean not null)`
|
||||
if _, err = p.conn.ExecContext(context.Background(), query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) ensureLockTable() error {
|
||||
if p.config.LockStrategy != LockStrategyTable {
|
||||
return nil
|
||||
}
|
||||
|
||||
var count int
|
||||
query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1`
|
||||
if err := p.db.QueryRow(query, p.config.LockTable).Scan(&count); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
if count == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query = `CREATE TABLE ` + pq.QuoteIdentifier(p.config.LockTable) + ` (lock_id BIGINT NOT NULL PRIMARY KEY)`
|
||||
if _, err := p.db.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611
|
||||
func quoteIdentifier(name string) string {
|
||||
end := strings.IndexRune(name, 0)
|
||||
if end > -1 {
|
||||
name = name[:end]
|
||||
}
|
||||
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
|
||||
}
|
||||
@@ -1,786 +0,0 @@
|
||||
package pgx
|
||||
|
||||
// error codes https://github.com/jackc/pgerrcode/blob/master/errcode.go
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
sqldriver "database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/dhui/dktest"
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
"github.com/golang-migrate/migrate/v4/dktesting"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
)
|
||||
|
||||
const (
|
||||
pgPassword = "postgres"
|
||||
)
|
||||
|
||||
var (
|
||||
opts = dktest.Options{
|
||||
Env: map[string]string{"POSTGRES_PASSWORD": pgPassword},
|
||||
PortRequired: true, ReadyFunc: isReady}
|
||||
// Supported versions: https://www.postgresql.org/support/versioning/
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "postgres:13", Options: opts},
|
||||
{ImageName: "postgres:14", Options: opts},
|
||||
{ImageName: "postgres:15", Options: opts},
|
||||
{ImageName: "postgres:16", Options: opts},
|
||||
{ImageName: "postgres:17", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
func pgConnectionString(host, port string, options ...string) string {
|
||||
options = append(options, "sslmode=disable")
|
||||
return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?%s", pgPassword, host, port, strings.Join(options, "&"))
|
||||
}
|
||||
|
||||
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
db, err := sql.Open("pgx", pgConnectionString(ip, port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
log.Println("close error:", err)
|
||||
}
|
||||
}()
|
||||
if err = db.PingContext(ctx); err != nil {
|
||||
switch err {
|
||||
case sqldriver.ErrBadConn, io.EOF:
|
||||
return false
|
||||
default:
|
||||
log.Println(err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, d database.Driver, statements []string) {
|
||||
for _, statement := range statements {
|
||||
if err := d.Run(strings.NewReader(statement)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "pgx", d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrateLockTable(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port, "x-lock-strategy=table", "x-lock-table=lock_table")
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "pgx", d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipleStatements(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil {
|
||||
t.Fatalf("expected err to be nil, got %v", err)
|
||||
}
|
||||
|
||||
// make sure second table exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table bar to exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipleStatementsInMultiStatementMode(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port, "x-multi-statement=true")
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE INDEX CONCURRENTLY idx_foo ON foo (foo);")); err != nil {
|
||||
t.Fatalf("expected err to be nil, got %v", err)
|
||||
}
|
||||
|
||||
// make sure created index exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = (SELECT current_schema()) AND indexname = 'idx_foo')").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table bar to exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestErrorParsing(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` +
|
||||
`(foo text); CREATE TABLEE bar (bar text); (details: ERROR: syntax error at or near "TABLEE" (SQLSTATE 42601))`
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil {
|
||||
t.Fatal("expected err but got nil")
|
||||
} else if err.Error() != wantErr {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFilterCustomQuery(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port, "x-custom=foobar")
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithSchema(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create foobar schema
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.SetVersion(1, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(pgConnectionString(ip, port, "search_path=foobar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
version, _, err := d2.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != database.NilVersion {
|
||||
t.Fatal("expected NilVersion")
|
||||
}
|
||||
|
||||
// now update version and compare
|
||||
if err := d2.SetVersion(2, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version, _, err = d2.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 2 {
|
||||
t.Fatal("expected version 2")
|
||||
}
|
||||
|
||||
// meanwhile, the public schema still has the other version
|
||||
version, _, err = d.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 1 {
|
||||
t.Fatal("expected version 2")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrationTableOption(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, _ := p.Open(addr)
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create migrate schema
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA migrate AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// bad unquoted x-migrations-table parameter
|
||||
wantErr := "x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: migrate.schema_migrations"
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if (err != nil) && (err.Error() != wantErr) {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
|
||||
// too many quoted x-migrations-table parameters
|
||||
wantErr = "\"\"migrate\".\"schema_migrations\".\"toomany\"\" MigrationsTable contains too many dot characters"
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\".\"toomany\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if (err != nil) && (err.Error() != wantErr) {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
|
||||
// good quoted x-migrations-table parameter
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// make sure migrate.schema_migrations table exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'schema_migrations' AND table_schema = 'migrate')").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table migrate.schema_migrations to exist")
|
||||
}
|
||||
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations",
|
||||
pgPassword, ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'migrate.schema_migrations' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table 'migrate.schema_migrations' to exist")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func TestFailToCreateTableWithoutPermissions(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
|
||||
// Check that opening the postgres connection returns NilVersion
|
||||
p := &Postgres{}
|
||||
|
||||
d, err := p.Open(addr)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine
|
||||
// since this is a test environment and we're not expecting to the pgPassword to be malicious
|
||||
mustRun(t, d, []string{
|
||||
"CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'",
|
||||
"CREATE SCHEMA barfoo AUTHORIZATION postgres",
|
||||
"GRANT USAGE ON SCHEMA barfoo TO not_owner",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
defer func() {
|
||||
if d2 == nil {
|
||||
return
|
||||
}
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
var e *database.Error
|
||||
if !errors.As(err, &e) || err == nil {
|
||||
t.Fatal("Unexpected error, want permission denied error. Got: ", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
// re-connect using that x-migrations-table and x-migrations-table-quoted
|
||||
d2, err = p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"barfoo\".\"schema_migrations\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if !errors.As(err, &e) || err == nil {
|
||||
t.Fatal("Unexpected error, want permission denied error. Got: ", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckBeforeCreateTable(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
|
||||
// Check that opening the postgres connection returns NilVersion
|
||||
p := &Postgres{}
|
||||
|
||||
d, err := p.Open(addr)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine
|
||||
// since this is a test environment and we're not expecting to the pgPassword to be malicious
|
||||
mustRun(t, d, []string{
|
||||
"CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'",
|
||||
"CREATE SCHEMA barfoo AUTHORIZATION postgres",
|
||||
"GRANT USAGE ON SCHEMA barfoo TO not_owner",
|
||||
"GRANT CREATE ON SCHEMA barfoo TO not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// revoke privileges
|
||||
mustRun(t, d, []string{
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d3, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
version, _, err := d3.Version()
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if version != database.NilVersion {
|
||||
t.Fatal("Unexpected version, want database.NilVersion. Got: ", version)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d3.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func TestParallelSchema(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create foo and bar schemas
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA foo AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA bar AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// re-connect using that schemas
|
||||
dfoo, err := p.Open(pgConnectionString(ip, port, "search_path=foo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := dfoo.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
dbar, err := p.Open(pgConnectionString(ip, port, "search_path=bar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := dbar.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := dfoo.Lock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := dbar.Lock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := dbar.Unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := dfoo.Unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPostgres_Lock(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
|
||||
ps := d.(*Postgres)
|
||||
|
||||
err = ps.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ps.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ps.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ps.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithInstance_Concurrent(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The number of concurrent processes running WithInstance
|
||||
const concurrency = 30
|
||||
|
||||
// We can instantiate a single database handle because it is
|
||||
// actually a connection pool, and so, each of the below go
|
||||
// routines will have a high probability of using a separate
|
||||
// connection, which is something we want to exercise.
|
||||
db, err := sql.Open("pgx", pgConnectionString(ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
db.SetMaxIdleConns(concurrency)
|
||||
db.SetMaxOpenConns(concurrency)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
|
||||
wg.Add(concurrency)
|
||||
for i := 0; i < concurrency; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
_, err := WithInstance(db, &Config{})
|
||||
if err != nil {
|
||||
t.Errorf("process %d error: %s", i, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
})
|
||||
}
|
||||
func Test_computeLineFromPos(t *testing.T) {
|
||||
testcases := []struct {
|
||||
pos int
|
||||
wantLine uint
|
||||
wantCol uint
|
||||
input string
|
||||
wantOk bool
|
||||
}{
|
||||
{
|
||||
15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists
|
||||
},
|
||||
{
|
||||
16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line
|
||||
},
|
||||
{
|
||||
25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error
|
||||
},
|
||||
{
|
||||
27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines
|
||||
},
|
||||
{
|
||||
10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo
|
||||
},
|
||||
{
|
||||
11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line
|
||||
},
|
||||
{
|
||||
17, 2, 8, "SELECT *\nFROM foo", true, // last character
|
||||
},
|
||||
{
|
||||
18, 0, 0, "SELECT *\nFROM foo", false, // invalid position
|
||||
},
|
||||
}
|
||||
for i, tc := range testcases {
|
||||
t.Run("tc"+strconv.Itoa(i), func(t *testing.T) {
|
||||
run := func(crlf bool, nonASCII bool) {
|
||||
var name string
|
||||
if crlf {
|
||||
name = "crlf"
|
||||
} else {
|
||||
name = "lf"
|
||||
}
|
||||
if nonASCII {
|
||||
name += "-nonascii"
|
||||
} else {
|
||||
name += "-ascii"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
input := tc.input
|
||||
if crlf {
|
||||
input = strings.ReplaceAll(input, "\n", "\r\n")
|
||||
}
|
||||
if nonASCII {
|
||||
input = strings.ReplaceAll(input, "FROM", "FRÖM")
|
||||
}
|
||||
gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos)
|
||||
|
||||
if tc.wantOk {
|
||||
t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input)
|
||||
}
|
||||
|
||||
if gotOK != tc.wantOk {
|
||||
t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK)
|
||||
}
|
||||
if gotLine != tc.wantLine {
|
||||
t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine)
|
||||
}
|
||||
if gotCol != tc.wantCol {
|
||||
t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol)
|
||||
}
|
||||
})
|
||||
}
|
||||
run(false, false)
|
||||
run(true, false)
|
||||
run(false, true)
|
||||
run(true, true)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
# pgx
|
||||
|
||||
This package is for [pgx/v5](https://pkg.go.dev/github.com/jackc/pgx/v5). A backend for the older [pgx/v4](https://pkg.go.dev/github.com/jackc/pgx/v4). is [also available](..).
|
||||
|
||||
`pgx5://user:password@host:port/dbname?query`
|
||||
|
||||
| URL Query | WithInstance Config | Description |
|
||||
|------------|---------------------|-------------|
|
||||
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
|
||||
| `x-migrations-table-quoted` | `MigrationsTableQuoted` | By default, migrate quotes the migration table for SQL injection safety reasons. This option disable quoting and naively checks that you have quoted the migration table name. e.g. `"my_schema"."schema_migrations"` |
|
||||
| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds |
|
||||
| `x-multi-statement` | `MultiStatementEnabled` | Enable multi-statement execution (default: false) |
|
||||
| `x-multi-statement-max-size` | `MultiStatementMaxSize` | Maximum size of single statement in bytes (default: 10MB) |
|
||||
| `dbname` | `DatabaseName` | The name of the database to connect to |
|
||||
| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. |
|
||||
| `user` | | The user to sign in as |
|
||||
| `password` | | The user's password |
|
||||
| `host` | | The host to connect to. Values that start with / are for unix domain sockets. (default is localhost) |
|
||||
| `port` | | The port to bind to. (default is 5432) |
|
||||
| `fallback_application_name` | | An application_name to fall back to if one isn't provided. |
|
||||
| `connect_timeout` | | Maximum wait for connection, in seconds. Zero or not specified means wait indefinitely. |
|
||||
| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
|
||||
| `sslkey` | | Key file location. The file must contain PEM encoded data. |
|
||||
| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
|
||||
| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
|
||||
|
||||
|
||||
## Upgrading from v1
|
||||
|
||||
1. Write down the current migration version from schema_migrations
|
||||
1. `DROP TABLE schema_migrations`
|
||||
2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration.
|
||||
3. Download and install the latest migrate version.
|
||||
4. Force the current migration version with `migrate force <current_version>`.
|
||||
|
||||
## Multi-statement mode
|
||||
|
||||
In PostgreSQL running multiple SQL statements in one `Exec` executes them inside a transaction. Sometimes this
|
||||
behavior is not desirable because some statements can be only run outside of transaction (e.g.
|
||||
`CREATE INDEX CONCURRENTLY`). If you want to use `CREATE INDEX CONCURRENTLY` without activating multi-statement mode
|
||||
you have to put such statements in a separate migration files.
|
||||
@@ -1,480 +0,0 @@
|
||||
//go:build go1.9
|
||||
|
||||
package pgx
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
"github.com/jackc/pgerrcode"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
_ "github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
|
||||
func init() {
|
||||
db := Postgres{}
|
||||
database.Register("pgx5", &db)
|
||||
}
|
||||
|
||||
var (
|
||||
multiStmtDelimiter = []byte(";")
|
||||
|
||||
DefaultMigrationsTable = "schema_migrations"
|
||||
DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
ErrNoSchema = fmt.Errorf("no schema")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
SchemaName string
|
||||
migrationsSchemaName string
|
||||
migrationsTableName string
|
||||
StatementTimeout time.Duration
|
||||
MigrationsTableQuoted bool
|
||||
MultiStatementEnabled bool
|
||||
MultiStatementMaxSize int
|
||||
}
|
||||
|
||||
type Postgres struct {
|
||||
// Locking and unlocking need to use the same connection
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
|
||||
// Open and WithInstance need to guarantee that config is never nil
|
||||
config *Config
|
||||
}
|
||||
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if config.DatabaseName == "" {
|
||||
query := `SELECT CURRENT_DATABASE()`
|
||||
var databaseName string
|
||||
if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(databaseName) == 0 {
|
||||
return nil, ErrNoDatabaseName
|
||||
}
|
||||
|
||||
config.DatabaseName = databaseName
|
||||
}
|
||||
|
||||
if config.SchemaName == "" {
|
||||
query := `SELECT CURRENT_SCHEMA()`
|
||||
var schemaName string
|
||||
if err := instance.QueryRow(query).Scan(&schemaName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(schemaName) == 0 {
|
||||
return nil, ErrNoSchema
|
||||
}
|
||||
|
||||
config.SchemaName = schemaName
|
||||
}
|
||||
|
||||
if len(config.MigrationsTable) == 0 {
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
config.migrationsSchemaName = config.SchemaName
|
||||
config.migrationsTableName = config.MigrationsTable
|
||||
if config.MigrationsTableQuoted {
|
||||
re := regexp.MustCompile(`"(.*?)"`)
|
||||
result := re.FindAllStringSubmatch(config.MigrationsTable, -1)
|
||||
config.migrationsTableName = result[len(result)-1][1]
|
||||
if len(result) == 2 {
|
||||
config.migrationsSchemaName = result[0][1]
|
||||
} else if len(result) > 2 {
|
||||
return nil, fmt.Errorf("\"%s\" MigrationsTable contains too many dot characters", config.MigrationsTable)
|
||||
}
|
||||
}
|
||||
|
||||
conn, err := instance.Conn(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
px := &Postgres{
|
||||
conn: conn,
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := px.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return px, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Open(url string) (database.Driver, error) {
|
||||
purl, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Driver is registered as pgx, but connection string must use postgres schema
|
||||
// when making actual connection
|
||||
// i.e. pgx://user:password@host:port/db => postgres://user:password@host:port/db
|
||||
purl.Scheme = "postgres"
|
||||
|
||||
db, err := sql.Open("pgx/v5", migrate.FilterCustomQuery(purl).String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
migrationsTable := purl.Query().Get("x-migrations-table")
|
||||
migrationsTableQuoted := false
|
||||
if s := purl.Query().Get("x-migrations-table-quoted"); len(s) > 0 {
|
||||
migrationsTableQuoted, err = strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse option x-migrations-table-quoted: %w", err)
|
||||
}
|
||||
}
|
||||
if (len(migrationsTable) > 0) && (migrationsTableQuoted) && ((migrationsTable[0] != '"') || (migrationsTable[len(migrationsTable)-1] != '"')) {
|
||||
return nil, fmt.Errorf("x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: %s", migrationsTable)
|
||||
}
|
||||
|
||||
statementTimeoutString := purl.Query().Get("x-statement-timeout")
|
||||
statementTimeout := 0
|
||||
if statementTimeoutString != "" {
|
||||
statementTimeout, err = strconv.Atoi(statementTimeoutString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
multiStatementMaxSize := DefaultMultiStatementMaxSize
|
||||
if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 {
|
||||
multiStatementMaxSize, err = strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if multiStatementMaxSize <= 0 {
|
||||
multiStatementMaxSize = DefaultMultiStatementMaxSize
|
||||
}
|
||||
}
|
||||
|
||||
multiStatementEnabled := false
|
||||
if s := purl.Query().Get("x-multi-statement"); len(s) > 0 {
|
||||
multiStatementEnabled, err = strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse option x-multi-statement: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
px, err := WithInstance(db, &Config{
|
||||
DatabaseName: purl.Path,
|
||||
MigrationsTable: migrationsTable,
|
||||
MigrationsTableQuoted: migrationsTableQuoted,
|
||||
StatementTimeout: time.Duration(statementTimeout) * time.Millisecond,
|
||||
MultiStatementEnabled: multiStatementEnabled,
|
||||
MultiStatementMaxSize: multiStatementMaxSize,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return px, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Close() error {
|
||||
connErr := p.conn.Close()
|
||||
dbErr := p.db.Close()
|
||||
if connErr != nil || dbErr != nil {
|
||||
return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS
|
||||
func (p *Postgres) Lock() error {
|
||||
return database.CasRestoreOnErr(&p.isLocked, false, true, database.ErrLocked, func() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// This will wait indefinitely until the lock can be acquired.
|
||||
query := `SELECT pg_advisory_lock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Postgres) Unlock() error {
|
||||
return database.CasRestoreOnErr(&p.isLocked, true, false, database.ErrNotLocked, func() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := `SELECT pg_advisory_unlock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Postgres) Run(migration io.Reader) error {
|
||||
if p.config.MultiStatementEnabled {
|
||||
var err error
|
||||
if e := multistmt.Parse(migration, multiStmtDelimiter, p.config.MultiStatementMaxSize, func(m []byte) bool {
|
||||
if err = p.runStatement(m); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); e != nil {
|
||||
return e
|
||||
}
|
||||
return err
|
||||
}
|
||||
migr, err := io.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return p.runStatement(migr)
|
||||
}
|
||||
|
||||
func (p *Postgres) runStatement(statement []byte) error {
|
||||
ctx := context.Background()
|
||||
if p.config.StatementTimeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, p.config.StatementTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
query := string(statement)
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return nil
|
||||
}
|
||||
if _, err := p.conn.ExecContext(ctx, query); err != nil {
|
||||
|
||||
if pgErr, ok := err.(*pgconn.PgError); ok {
|
||||
var line uint
|
||||
var col uint
|
||||
var lineColOK bool
|
||||
line, col, lineColOK = computeLineFromPos(query, int(pgErr.Position))
|
||||
message := fmt.Sprintf("migration failed: %s", pgErr.Message)
|
||||
if lineColOK {
|
||||
message = fmt.Sprintf("%s (column %d)", message, col)
|
||||
}
|
||||
if pgErr.Detail != "" {
|
||||
message = fmt.Sprintf("%s, %s", message, pgErr.Detail)
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: message, Query: statement, Line: line}
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: statement}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) {
|
||||
// replace crlf with lf
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
// pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes
|
||||
runes := []rune(s)
|
||||
if pos > len(runes) {
|
||||
return 0, 0, false
|
||||
}
|
||||
sel := runes[:pos]
|
||||
line = uint(runesCount(sel, newLine) + 1)
|
||||
col = uint(pos - 1 - runesLastIndex(sel, newLine))
|
||||
return line, col, true
|
||||
}
|
||||
|
||||
const newLine = '\n'
|
||||
|
||||
func runesCount(input []rune, target rune) int {
|
||||
var count int
|
||||
for _, r := range input {
|
||||
if r == target {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func runesLastIndex(input []rune, target rune) int {
|
||||
for i := len(input) - 1; i >= 0; i-- {
|
||||
if input[i] == target {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func (p *Postgres) SetVersion(version int, dirty bool) error {
|
||||
tx, err := p.conn.BeginTx(context.Background(), &sql.TxOptions{})
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
|
||||
query := `TRUNCATE ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName)
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// Also re-write the schema version for nil dirty versions to prevent
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
query = `INSERT INTO ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` (version, dirty) VALUES ($1, $2)`
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction commit failed"}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Version() (version int, dirty bool, err error) {
|
||||
query := `SELECT version, dirty FROM ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` LIMIT 1`
|
||||
err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty)
|
||||
switch {
|
||||
case err == sql.ErrNoRows:
|
||||
return database.NilVersion, false, nil
|
||||
|
||||
case err != nil:
|
||||
if e, ok := err.(*pgconn.PgError); ok {
|
||||
if e.SQLState() == pgerrcode.UndefinedTable {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
}
|
||||
return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
|
||||
default:
|
||||
return version, dirty, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Postgres) Drop() (err error) {
|
||||
// select all tables in current schema
|
||||
query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema()) AND table_type='BASE TABLE'`
|
||||
tables, err := p.conn.QueryContext(context.Background(), query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
// delete one table after another
|
||||
tableNames := make([]string, 0)
|
||||
for tables.Next() {
|
||||
var tableName string
|
||||
if err := tables.Scan(&tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tableName) > 0 {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// delete one by one ...
|
||||
for _, t := range tableNames {
|
||||
query = `DROP TABLE IF EXISTS ` + quoteIdentifier(t) + ` CASCADE`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the Postgres type.
|
||||
func (p *Postgres) ensureVersionTable() (err error) {
|
||||
if err = p.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := p.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
}
|
||||
}()
|
||||
|
||||
// This block checks whether the `MigrationsTable` already exists. This is useful because it allows read only postgres
|
||||
// users to also check the current version of the schema. Previously, even if `MigrationsTable` existed, the
|
||||
// `CREATE TABLE IF NOT EXISTS...` query would fail because the user does not have the CREATE permission.
|
||||
// Taken from https://github.com/mattes/migrate/blob/master/database/postgres/postgres.go#L258
|
||||
query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2 LIMIT 1`
|
||||
row := p.conn.QueryRowContext(context.Background(), query, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
|
||||
var count int
|
||||
err = row.Scan(&count)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if count == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query = `CREATE TABLE IF NOT EXISTS ` + quoteIdentifier(p.config.migrationsSchemaName) + `.` + quoteIdentifier(p.config.migrationsTableName) + ` (version bigint not null primary key, dirty boolean not null)`
|
||||
if _, err = p.conn.ExecContext(context.Background(), query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611
|
||||
func quoteIdentifier(name string) string {
|
||||
end := strings.IndexRune(name, 0)
|
||||
if end > -1 {
|
||||
name = name[:end]
|
||||
}
|
||||
return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`
|
||||
}
|
||||
@@ -1,761 +0,0 @@
|
||||
package pgx
|
||||
|
||||
// error codes https://github.com/jackc/pgerrcode/blob/master/errcode.go
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
sqldriver "database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
|
||||
"github.com/dhui/dktest"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
"github.com/golang-migrate/migrate/v4/dktesting"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
)
|
||||
|
||||
const (
|
||||
pgPassword = "postgres"
|
||||
)
|
||||
|
||||
var (
|
||||
opts = dktest.Options{
|
||||
Env: map[string]string{"POSTGRES_PASSWORD": pgPassword},
|
||||
PortRequired: true, ReadyFunc: isReady}
|
||||
// Supported versions: https://www.postgresql.org/support/versioning/
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "postgres:13", Options: opts},
|
||||
{ImageName: "postgres:14", Options: opts},
|
||||
{ImageName: "postgres:15", Options: opts},
|
||||
{ImageName: "postgres:16", Options: opts},
|
||||
{ImageName: "postgres:17", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
func pgConnectionString(host, port string, options ...string) string {
|
||||
options = append(options, "sslmode=disable")
|
||||
return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?%s", pgPassword, host, port, strings.Join(options, "&"))
|
||||
}
|
||||
|
||||
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
db, err := sql.Open("pgx", pgConnectionString(ip, port))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
log.Println("close error:", err)
|
||||
}
|
||||
}()
|
||||
if err = db.PingContext(ctx); err != nil {
|
||||
switch err {
|
||||
case sqldriver.ErrBadConn, io.EOF:
|
||||
return false
|
||||
default:
|
||||
log.Println(err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, d database.Driver, statements []string) {
|
||||
for _, statement := range statements {
|
||||
if err := d.Run(strings.NewReader(statement)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
m, err := migrate.NewWithDatabaseInstance("file://../examples/migrations", "pgx", d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipleStatements(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLE bar (bar text);")); err != nil {
|
||||
t.Fatalf("expected err to be nil, got %v", err)
|
||||
}
|
||||
|
||||
// make sure second table exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'bar' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table bar to exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestMultipleStatementsInMultiStatementMode(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port, "x-multi-statement=true")
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE INDEX CONCURRENTLY idx_foo ON foo (foo);")); err != nil {
|
||||
t.Fatalf("expected err to be nil, got %v", err)
|
||||
}
|
||||
|
||||
// make sure created index exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = (SELECT current_schema()) AND indexname = 'idx_foo')").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table bar to exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestErrorParsing(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
wantErr := `migration failed: syntax error at or near "TABLEE" (column 37) in line 1: CREATE TABLE foo ` +
|
||||
`(foo text); CREATE TABLEE bar (bar text); (details: ERROR: syntax error at or near "TABLEE" (SQLSTATE 42601))`
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE TABLEE bar (bar text);")); err == nil {
|
||||
t.Fatal("expected err but got nil")
|
||||
} else if err.Error() != wantErr {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestFilterCustomQuery(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port, "x-custom=foobar")
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithSchema(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create foobar schema
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA foobar AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.SetVersion(1, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(pgConnectionString(ip, port, "search_path=foobar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
version, _, err := d2.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != database.NilVersion {
|
||||
t.Fatal("expected NilVersion")
|
||||
}
|
||||
|
||||
// now update version and compare
|
||||
if err := d2.SetVersion(2, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
version, _, err = d2.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 2 {
|
||||
t.Fatal("expected version 2")
|
||||
}
|
||||
|
||||
// meanwhile, the public schema still has the other version
|
||||
version, _, err = d.Version()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != 1 {
|
||||
t.Fatal("expected version 2")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrationTableOption(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, _ := p.Open(addr)
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create migrate schema
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA migrate AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// bad unquoted x-migrations-table parameter
|
||||
wantErr := "x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: migrate.schema_migrations"
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if (err != nil) && (err.Error() != wantErr) {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
|
||||
// too many quoted x-migrations-table parameters
|
||||
wantErr = "\"\"migrate\".\"schema_migrations\".\"toomany\"\" MigrationsTable contains too many dot characters"
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\".\"toomany\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if (err != nil) && (err.Error() != wantErr) {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
|
||||
// good quoted x-migrations-table parameter
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// make sure migrate.schema_migrations table exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'schema_migrations' AND table_schema = 'migrate')").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table migrate.schema_migrations to exist")
|
||||
}
|
||||
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations",
|
||||
pgPassword, ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'migrate.schema_migrations' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table 'migrate.schema_migrations' to exist")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func TestFailToCreateTableWithoutPermissions(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
|
||||
// Check that opening the postgres connection returns NilVersion
|
||||
p := &Postgres{}
|
||||
|
||||
d, err := p.Open(addr)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine
|
||||
// since this is a test environment and we're not expecting to the pgPassword to be malicious
|
||||
mustRun(t, d, []string{
|
||||
"CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'",
|
||||
"CREATE SCHEMA barfoo AUTHORIZATION postgres",
|
||||
"GRANT USAGE ON SCHEMA barfoo TO not_owner",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
defer func() {
|
||||
if d2 == nil {
|
||||
return
|
||||
}
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
var e *database.Error
|
||||
if !errors.As(err, &e) || err == nil {
|
||||
t.Fatal("Unexpected error, want permission denied error. Got: ", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
// re-connect using that x-migrations-table and x-migrations-table-quoted
|
||||
d2, err = p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"barfoo\".\"schema_migrations\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if !errors.As(err, &e) || err == nil {
|
||||
t.Fatal("Unexpected error, want permission denied error. Got: ", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCheckBeforeCreateTable(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
|
||||
// Check that opening the postgres connection returns NilVersion
|
||||
p := &Postgres{}
|
||||
|
||||
d, err := p.Open(addr)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine
|
||||
// since this is a test environment and we're not expecting to the pgPassword to be malicious
|
||||
mustRun(t, d, []string{
|
||||
"CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'",
|
||||
"CREATE SCHEMA barfoo AUTHORIZATION postgres",
|
||||
"GRANT USAGE ON SCHEMA barfoo TO not_owner",
|
||||
"GRANT CREATE ON SCHEMA barfoo TO not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// revoke privileges
|
||||
mustRun(t, d, []string{
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d3, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
version, _, err := d3.Version()
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if version != database.NilVersion {
|
||||
t.Fatal("Unexpected version, want database.NilVersion. Got: ", version)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d3.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func TestParallelSchema(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create foo and bar schemas
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA foo AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA bar AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// re-connect using that schemas
|
||||
dfoo, err := p.Open(pgConnectionString(ip, port, "search_path=foo"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := dfoo.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
dbar, err := p.Open(pgConnectionString(ip, port, "search_path=bar"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := dbar.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := dfoo.Lock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := dbar.Lock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := dbar.Unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := dfoo.Unlock(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestPostgres_Lock(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
|
||||
ps := d.(*Postgres)
|
||||
|
||||
err = ps.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ps.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ps.Lock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
err = ps.Unlock()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithInstance_Concurrent(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The number of concurrent processes running WithInstance
|
||||
const concurrency = 30
|
||||
|
||||
// We can instantiate a single database handle because it is
|
||||
// actually a connection pool, and so, each of the below go
|
||||
// routines will have a high probability of using a separate
|
||||
// connection, which is something we want to exercise.
|
||||
db, err := sql.Open("pgx", pgConnectionString(ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
db.SetMaxIdleConns(concurrency)
|
||||
db.SetMaxOpenConns(concurrency)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
defer wg.Wait()
|
||||
|
||||
wg.Add(concurrency)
|
||||
for i := 0; i < concurrency; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
_, err := WithInstance(db, &Config{})
|
||||
if err != nil {
|
||||
t.Errorf("process %d error: %s", i, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
})
|
||||
}
|
||||
func Test_computeLineFromPos(t *testing.T) {
|
||||
testcases := []struct {
|
||||
pos int
|
||||
wantLine uint
|
||||
wantCol uint
|
||||
input string
|
||||
wantOk bool
|
||||
}{
|
||||
{
|
||||
15, 2, 6, "SELECT *\nFROM foo", true, // foo table does not exists
|
||||
},
|
||||
{
|
||||
16, 3, 6, "SELECT *\n\nFROM foo", true, // foo table does not exists, empty line
|
||||
},
|
||||
{
|
||||
25, 3, 7, "SELECT *\nFROM foo\nWHERE x", true, // x column error
|
||||
},
|
||||
{
|
||||
27, 5, 7, "SELECT *\n\nFROM foo\n\nWHERE x", true, // x column error, empty lines
|
||||
},
|
||||
{
|
||||
10, 2, 1, "SELECT *\nFROMM foo", true, // FROMM typo
|
||||
},
|
||||
{
|
||||
11, 3, 1, "SELECT *\n\nFROMM foo", true, // FROMM typo, empty line
|
||||
},
|
||||
{
|
||||
17, 2, 8, "SELECT *\nFROM foo", true, // last character
|
||||
},
|
||||
{
|
||||
18, 0, 0, "SELECT *\nFROM foo", false, // invalid position
|
||||
},
|
||||
}
|
||||
for i, tc := range testcases {
|
||||
t.Run("tc"+strconv.Itoa(i), func(t *testing.T) {
|
||||
run := func(crlf bool, nonASCII bool) {
|
||||
var name string
|
||||
if crlf {
|
||||
name = "crlf"
|
||||
} else {
|
||||
name = "lf"
|
||||
}
|
||||
if nonASCII {
|
||||
name += "-nonascii"
|
||||
} else {
|
||||
name += "-ascii"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
input := tc.input
|
||||
if crlf {
|
||||
input = strings.ReplaceAll(input, "\n", "\r\n")
|
||||
}
|
||||
if nonASCII {
|
||||
input = strings.ReplaceAll(input, "FROM", "FRÖM")
|
||||
}
|
||||
gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos)
|
||||
|
||||
if tc.wantOk {
|
||||
t.Logf("pos %d, want %d:%d, %#v", tc.pos, tc.wantLine, tc.wantCol, input)
|
||||
}
|
||||
|
||||
if gotOK != tc.wantOk {
|
||||
t.Fatalf("expected ok %v but got %v", tc.wantOk, gotOK)
|
||||
}
|
||||
if gotLine != tc.wantLine {
|
||||
t.Fatalf("expected line %d but got %d", tc.wantLine, gotLine)
|
||||
}
|
||||
if gotCol != tc.wantCol {
|
||||
t.Fatalf("expected col %d but got %d", tc.wantCol, gotCol)
|
||||
}
|
||||
})
|
||||
}
|
||||
run(false, false)
|
||||
run(true, false)
|
||||
run(false, true)
|
||||
run(true, true)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,10 +5,7 @@
|
||||
| URL Query | WithInstance Config | Description |
|
||||
|------------|---------------------|-------------|
|
||||
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
|
||||
| `x-migrations-table-quoted` | `MigrationsTableQuoted` | By default, migrate quotes the migration table for SQL injection safety reasons. This option disable quoting and naively checks that you have quoted the migration table name. e.g. `"my_schema"."schema_migrations"` |
|
||||
| `x-statement-timeout` | `StatementTimeout` | Abort any statement that takes more than the specified number of milliseconds |
|
||||
| `x-multi-statement` | `MultiStatementEnabled` | Enable multi-statement execution (default: false) |
|
||||
| `x-multi-statement-max-size` | `MultiStatementMaxSize` | Maximum size of single statement in bytes (default: 10MB) |
|
||||
| `dbname` | `DatabaseName` | The name of the database to connect to |
|
||||
| `search_path` | | This variable specifies the order in which schemas are searched when an object is referenced by a simple name with no schema specified. |
|
||||
| `user` | | The user to sign in as |
|
||||
@@ -30,10 +27,3 @@
|
||||
2. Wrap your existing migrations in transactions ([BEGIN/COMMIT](https://www.postgresql.org/docs/current/static/transaction-iso.html)) if you use multiple statements within one migration.
|
||||
3. Download and install the latest migrate version.
|
||||
4. Force the current migration version with `migrate force <current_version>`.
|
||||
|
||||
## Multi-statement mode
|
||||
|
||||
In PostgreSQL running multiple SQL statements in one `Exec` executes them inside a transaction. Sometimes this
|
||||
behavior is not desirable because some statements can be only run outside of transaction (e.g.
|
||||
`CREATE INDEX CONCURRENTLY`). If you want to use `CREATE INDEX CONCURRENTLY` without activating multi-statement mode
|
||||
you have to put such statements in a separate migration files.
|
||||
|
||||
@@ -7,7 +7,7 @@ Our user here is `postgres`, password `password`, and host is `localhost`.
|
||||
```
|
||||
psql -h localhost -U postgres -w -c "create database example;"
|
||||
```
|
||||
When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convenience:
|
||||
When using Migrate CLI we need to pass to database URL. Let's export it to a variable for convienience:
|
||||
```
|
||||
export POSTGRESQL_URL='postgres://postgres:password@localhost:5432/example?sslmode=disable'
|
||||
```
|
||||
@@ -27,7 +27,7 @@ If there were no errors, we should have two files available under `db/migrations
|
||||
Note the `sql` extension that we provided.
|
||||
|
||||
In the `.up.sql` file let's create the table:
|
||||
```sql
|
||||
```
|
||||
CREATE TABLE IF NOT EXISTS users(
|
||||
user_id serial PRIMARY KEY,
|
||||
username VARCHAR (50) UNIQUE NOT NULL,
|
||||
@@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS users(
|
||||
);
|
||||
```
|
||||
And in the `.down.sql` let's delete it:
|
||||
```sql
|
||||
```
|
||||
DROP TABLE IF EXISTS users;
|
||||
```
|
||||
By adding `IF EXISTS/IF NOT EXISTS` we are making migrations idempotent - you can read more about idempotency in [getting started](../../GETTING_STARTED.md#create-migrations)
|
||||
@@ -79,7 +79,7 @@ Again, it should create for us two migrations files:
|
||||
In Postgres, when we want our queries to be done in a transaction, we need to wrap it with `BEGIN` and `COMMIT` commands.
|
||||
In our example, we are going to add a column to our database that can only accept enumerable values or NULL.
|
||||
Migration up:
|
||||
```sql
|
||||
```
|
||||
BEGIN;
|
||||
|
||||
CREATE TYPE enum_mood AS ENUM (
|
||||
@@ -92,7 +92,7 @@ ALTER TABLE users ADD COLUMN mood enum_mood;
|
||||
COMMIT;
|
||||
```
|
||||
Migration down:
|
||||
```sql
|
||||
```
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE users DROP COLUMN mood;
|
||||
@@ -124,7 +124,7 @@ Indexes:
|
||||
|
||||
## Optional: Run migrations within your Go app
|
||||
Here is a very simple app running migrations for the above configuration:
|
||||
```go
|
||||
```
|
||||
import (
|
||||
"log"
|
||||
|
||||
@@ -146,22 +146,3 @@ func main() {
|
||||
}
|
||||
```
|
||||
You can find details [here](README.md#use-in-your-go-project)
|
||||
|
||||
## Fix issue where migrations run twice
|
||||
|
||||
When the schema and role names are the same, you might run into issues if you create this schema using migrations.
|
||||
This is caused by the fact that the [default `search_path`](https://www.postgresql.org/docs/current/ddl-schemas.html#DDL-SCHEMAS-PATH) is `"$user", public`.
|
||||
In the first run (with an empty database) the migrate table is created in `public`.
|
||||
When the migrations create the `$user` schema, the next run will store (a new) migrate table in this schema (due to order of schemas in `search_path`) and tries to apply all migrations again (most likely failing).
|
||||
|
||||
To solve this you need to change the default `search_path` by removing the `$user` component, so the migrate table is always stored in the (available) `public` schema.
|
||||
This can be done using the [`search_path` query parameter in the URL](https://github.com/jexia/migrate/blob/fix-postgres-version-table/database/postgres/README.md#postgres).
|
||||
|
||||
For example to force the migrations table in the public schema you can use:
|
||||
```
|
||||
export POSTGRESQL_URL='postgres://postgres:password@localhost:5432/example?sslmode=disable&search_path=public'
|
||||
```
|
||||
|
||||
Note that you need to explicitly add the schema names to the table names in your migrations when you to modify the tables of the non-public schema.
|
||||
|
||||
Alternatively you can add the non-public schema manually (before applying the migrations) if that is possible in your case and let the tool store the migrations table in this schema as well.
|
||||
|
||||
+80
-181
@@ -1,23 +1,21 @@
|
||||
//go:build go1.9
|
||||
// +build go1.9
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/golang-migrate/migrate/v4/database/multistmt"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
@@ -27,12 +25,7 @@ func init() {
|
||||
database.Register("postgresql", &db)
|
||||
}
|
||||
|
||||
var (
|
||||
multiStmtDelimiter = []byte(";")
|
||||
|
||||
DefaultMigrationsTable = "schema_migrations"
|
||||
DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB
|
||||
)
|
||||
var DefaultMigrationsTable = "schema_migrations"
|
||||
|
||||
var (
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
@@ -42,40 +35,35 @@ var (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MigrationsTable string
|
||||
MigrationsTableQuoted bool
|
||||
MultiStatementEnabled bool
|
||||
DatabaseName string
|
||||
SchemaName string
|
||||
migrationsSchemaName string
|
||||
migrationsTableName string
|
||||
StatementTimeout time.Duration
|
||||
MultiStatementMaxSize int
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
SchemaName string
|
||||
StatementTimeout time.Duration
|
||||
}
|
||||
|
||||
type Postgres struct {
|
||||
// Locking and unlocking need to use the same connection
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
|
||||
// Open and WithInstance need to guarantee that config is never nil
|
||||
config *Config
|
||||
}
|
||||
|
||||
func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Postgres, error) {
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := conn.PingContext(ctx); err != nil {
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if config.DatabaseName == "" {
|
||||
query := `SELECT CURRENT_DATABASE()`
|
||||
var databaseName string
|
||||
if err := conn.QueryRowContext(ctx, query).Scan(&databaseName); err != nil {
|
||||
if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
@@ -88,37 +76,31 @@ func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Postg
|
||||
|
||||
if config.SchemaName == "" {
|
||||
query := `SELECT CURRENT_SCHEMA()`
|
||||
var schemaName sql.NullString
|
||||
if err := conn.QueryRowContext(ctx, query).Scan(&schemaName); err != nil {
|
||||
var schemaName string
|
||||
if err := instance.QueryRow(query).Scan(&schemaName); err != nil {
|
||||
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if !schemaName.Valid {
|
||||
if len(schemaName) == 0 {
|
||||
return nil, ErrNoSchema
|
||||
}
|
||||
|
||||
config.SchemaName = schemaName.String
|
||||
config.SchemaName = schemaName
|
||||
}
|
||||
|
||||
if len(config.MigrationsTable) == 0 {
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
config.migrationsSchemaName = config.SchemaName
|
||||
config.migrationsTableName = config.MigrationsTable
|
||||
if config.MigrationsTableQuoted {
|
||||
re := regexp.MustCompile(`"(.*?)"`)
|
||||
result := re.FindAllStringSubmatch(config.MigrationsTable, -1)
|
||||
config.migrationsTableName = result[len(result)-1][1]
|
||||
if len(result) == 2 {
|
||||
config.migrationsSchemaName = result[0][1]
|
||||
} else if len(result) > 2 {
|
||||
return nil, fmt.Errorf("\"%s\" MigrationsTable contains too many dot characters", config.MigrationsTable)
|
||||
}
|
||||
conn, err := instance.Conn(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
px := &Postgres{
|
||||
conn: conn,
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
|
||||
@@ -129,26 +111,6 @@ func WithConnection(ctx context.Context, conn *sql.Conn, config *Config) (*Postg
|
||||
return px, nil
|
||||
}
|
||||
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, err := instance.Conn(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
px, err := WithConnection(ctx, conn, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
px.db = instance
|
||||
return px, nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Open(url string) (database.Driver, error) {
|
||||
purl, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
@@ -161,17 +123,6 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
|
||||
}
|
||||
|
||||
migrationsTable := purl.Query().Get("x-migrations-table")
|
||||
migrationsTableQuoted := false
|
||||
if s := purl.Query().Get("x-migrations-table-quoted"); len(s) > 0 {
|
||||
migrationsTableQuoted, err = strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse option x-migrations-table-quoted: %w", err)
|
||||
}
|
||||
}
|
||||
if (len(migrationsTable) > 0) && (migrationsTableQuoted) && ((migrationsTable[0] != '"') || (migrationsTable[len(migrationsTable)-1] != '"')) {
|
||||
return nil, fmt.Errorf("x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: %s", migrationsTable)
|
||||
}
|
||||
|
||||
statementTimeoutString := purl.Query().Get("x-statement-timeout")
|
||||
statementTimeout := 0
|
||||
if statementTimeoutString != "" {
|
||||
@@ -181,32 +132,10 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
|
||||
}
|
||||
}
|
||||
|
||||
multiStatementMaxSize := DefaultMultiStatementMaxSize
|
||||
if s := purl.Query().Get("x-multi-statement-max-size"); len(s) > 0 {
|
||||
multiStatementMaxSize, err = strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if multiStatementMaxSize <= 0 {
|
||||
multiStatementMaxSize = DefaultMultiStatementMaxSize
|
||||
}
|
||||
}
|
||||
|
||||
multiStatementEnabled := false
|
||||
if s := purl.Query().Get("x-multi-statement"); len(s) > 0 {
|
||||
multiStatementEnabled, err = strconv.ParseBool(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to parse option x-multi-statement: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
px, err := WithInstance(db, &Config{
|
||||
DatabaseName: purl.Path,
|
||||
MigrationsTable: migrationsTable,
|
||||
MigrationsTableQuoted: migrationsTableQuoted,
|
||||
StatementTimeout: time.Duration(statementTimeout) * time.Millisecond,
|
||||
MultiStatementEnabled: multiStatementEnabled,
|
||||
MultiStatementMaxSize: multiStatementMaxSize,
|
||||
DatabaseName: purl.Path,
|
||||
MigrationsTable: migrationsTable,
|
||||
StatementTimeout: time.Duration(statementTimeout) * time.Millisecond,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
@@ -218,11 +147,7 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
|
||||
|
||||
func (p *Postgres) Close() error {
|
||||
connErr := p.conn.Close()
|
||||
var dbErr error
|
||||
if p.db != nil {
|
||||
dbErr = p.db.Close()
|
||||
}
|
||||
|
||||
dbErr := p.db.Close()
|
||||
if connErr != nil || dbErr != nil {
|
||||
return fmt.Errorf("conn: %v, db: %v", connErr, dbErr)
|
||||
}
|
||||
@@ -231,68 +156,56 @@ func (p *Postgres) Close() error {
|
||||
|
||||
// https://www.postgresql.org/docs/9.6/static/explicit-locking.html#ADVISORY-LOCKS
|
||||
func (p *Postgres) Lock() error {
|
||||
return database.CasRestoreOnErr(&p.isLocked, false, true, database.ErrLocked, func() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// This will wait indefinitely until the lock can be acquired.
|
||||
query := `SELECT pg_advisory_lock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Postgres) Unlock() error {
|
||||
return database.CasRestoreOnErr(&p.isLocked, true, false, database.ErrNotLocked, func() error {
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := `SELECT pg_advisory_unlock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (p *Postgres) Run(migration io.Reader) error {
|
||||
if p.config.MultiStatementEnabled {
|
||||
var err error
|
||||
if e := multistmt.Parse(migration, multiStmtDelimiter, p.config.MultiStatementMaxSize, func(m []byte) bool {
|
||||
if err = p.runStatement(m); err != nil {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}); e != nil {
|
||||
return e
|
||||
}
|
||||
return err
|
||||
if p.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
migr, err := io.ReadAll(migration)
|
||||
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.SchemaName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return p.runStatement(migr)
|
||||
|
||||
// This will wait indefinitely until the lock can be acquired.
|
||||
query := `SELECT pg_advisory_lock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)}
|
||||
}
|
||||
|
||||
p.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) runStatement(statement []byte) error {
|
||||
func (p *Postgres) Unlock() error {
|
||||
if !p.isLocked {
|
||||
return nil
|
||||
}
|
||||
|
||||
aid, err := database.GenerateAdvisoryLockId(p.config.DatabaseName, p.config.SchemaName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := `SELECT pg_advisory_unlock($1)`
|
||||
if _, err := p.conn.ExecContext(context.Background(), query, aid); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
p.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Postgres) Run(migration io.Reader) error {
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx := context.Background()
|
||||
if p.config.StatementTimeout != 0 {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, p.config.StatementTimeout)
|
||||
defer cancel()
|
||||
}
|
||||
query := string(statement)
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return nil
|
||||
}
|
||||
// run migration
|
||||
query := string(migr[:])
|
||||
if _, err := p.conn.ExecContext(ctx, query); err != nil {
|
||||
if pgErr, ok := err.(*pq.Error); ok {
|
||||
var line uint
|
||||
@@ -310,16 +223,17 @@ func (p *Postgres) runStatement(statement []byte) error {
|
||||
if pgErr.Detail != "" {
|
||||
message = fmt.Sprintf("%s, %s", message, pgErr.Detail)
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: message, Query: statement, Line: line}
|
||||
return database.Error{OrigErr: err, Err: message, Query: migr, Line: line}
|
||||
}
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: statement}
|
||||
return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) {
|
||||
// replace crlf with lf
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
s = strings.Replace(s, "\r\n", "\n", -1)
|
||||
// pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes
|
||||
runes := []rune(s)
|
||||
if pos > len(runes) {
|
||||
@@ -358,10 +272,10 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
|
||||
query := `TRUNCATE ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName)
|
||||
query := `TRUNCATE ` + pq.QuoteIdentifier(p.config.MigrationsTable)
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -370,10 +284,11 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
query = `INSERT INTO ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) + ` (version, dirty) VALUES ($1, $2)`
|
||||
query = `INSERT INTO ` + pq.QuoteIdentifier(p.config.MigrationsTable) +
|
||||
` (version, dirty) VALUES ($1, $2)`
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -387,7 +302,7 @@ func (p *Postgres) SetVersion(version int, dirty bool) error {
|
||||
}
|
||||
|
||||
func (p *Postgres) Version() (version int, dirty bool, err error) {
|
||||
query := `SELECT version, dirty FROM ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) + ` LIMIT 1`
|
||||
query := `SELECT version, dirty FROM ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` LIMIT 1`
|
||||
err = p.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty)
|
||||
switch {
|
||||
case err == sql.ErrNoRows:
|
||||
@@ -415,7 +330,7 @@ func (p *Postgres) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -430,9 +345,6 @@ func (p *Postgres) Drop() (err error) {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// delete one by one ...
|
||||
@@ -457,28 +369,15 @@ func (p *Postgres) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := p.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// This block checks whether the `MigrationsTable` already exists. This is useful because it allows read only postgres
|
||||
// users to also check the current version of the schema. Previously, even if `MigrationsTable` existed, the
|
||||
// `CREATE TABLE IF NOT EXISTS...` query would fail because the user does not have the CREATE permission.
|
||||
// Taken from https://github.com/mattes/migrate/blob/master/database/postgres/postgres.go#L258
|
||||
query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_schema = $1 AND table_name = $2 LIMIT 1`
|
||||
row := p.conn.QueryRowContext(context.Background(), query, p.config.migrationsSchemaName, p.config.migrationsTableName)
|
||||
|
||||
var count int
|
||||
err = row.Scan(&count)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if count == 1 {
|
||||
return nil
|
||||
}
|
||||
|
||||
query = `CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(p.config.migrationsSchemaName) + `.` + pq.QuoteIdentifier(p.config.migrationsTableName) + ` (version bigint not null primary key, dirty boolean not null)`
|
||||
query := `CREATE TABLE IF NOT EXISTS ` + pq.QuoteIdentifier(p.config.MigrationsTable) + ` (version bigint not null primary key, dirty boolean not null)`
|
||||
if _, err = p.conn.ExecContext(context.Background(), query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
@@ -6,17 +6,16 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
sqldriver "database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
|
||||
"github.com/dhui/dktest"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
@@ -35,17 +34,16 @@ var (
|
||||
PortRequired: true, ReadyFunc: isReady}
|
||||
// Supported versions: https://www.postgresql.org/support/versioning/
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "postgres:13", Options: opts},
|
||||
{ImageName: "postgres:14", Options: opts},
|
||||
{ImageName: "postgres:15", Options: opts},
|
||||
{ImageName: "postgres:16", Options: opts},
|
||||
{ImageName: "postgres:17", Options: opts},
|
||||
{ImageName: "postgres:9.5", Options: opts},
|
||||
{ImageName: "postgres:9.6", Options: opts},
|
||||
{ImageName: "postgres:10", Options: opts},
|
||||
{ImageName: "postgres:11", Options: opts},
|
||||
{ImageName: "postgres:12", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
func pgConnectionString(host, port string, options ...string) string {
|
||||
options = append(options, "sslmode=disable")
|
||||
return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?%s", pgPassword, host, port, strings.Join(options, "&"))
|
||||
func pgConnectionString(host, port string) string {
|
||||
return fmt.Sprintf("postgres://postgres:%s@%s:%s/postgres?sslmode=disable", pgPassword, host, port)
|
||||
}
|
||||
|
||||
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
@@ -76,41 +74,7 @@ func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, d database.Driver, statements []string) {
|
||||
for _, statement := range statements {
|
||||
if err := d.Run(strings.NewReader(statement)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
t.Run("test", test)
|
||||
t.Run("testMigrate", testMigrate)
|
||||
t.Run("testMultipleStatements", testMultipleStatements)
|
||||
t.Run("testMultipleStatementsInMultiStatementMode", testMultipleStatementsInMultiStatementMode)
|
||||
t.Run("testErrorParsing", testErrorParsing)
|
||||
t.Run("testFilterCustomQuery", testFilterCustomQuery)
|
||||
t.Run("testWithSchema", testWithSchema)
|
||||
t.Run("testMigrationTableOption", testMigrationTableOption)
|
||||
t.Run("testFailToCreateTableWithoutPermissions", testFailToCreateTableWithoutPermissions)
|
||||
t.Run("testCheckBeforeCreateTable", testCheckBeforeCreateTable)
|
||||
t.Run("testParallelSchema", testParallelSchema)
|
||||
t.Run("testPostgresLock", testPostgresLock)
|
||||
t.Run("testWithInstanceConcurrent", testWithInstanceConcurrent)
|
||||
t.Run("testWithConnection", testWithConnection)
|
||||
|
||||
t.Cleanup(func() {
|
||||
for _, spec := range specs {
|
||||
t.Log("Cleaning up ", spec.ImageName)
|
||||
if err := spec.Cleanup(); err != nil {
|
||||
t.Error("Error removing ", spec.ImageName, "error:", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func test(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -132,7 +96,7 @@ func test(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testMigrate(t *testing.T) {
|
||||
func TestMigrate(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -158,7 +122,7 @@ func testMigrate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testMultipleStatements(t *testing.T) {
|
||||
func TestMultiStatement(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -191,40 +155,7 @@ func testMultipleStatements(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testMultipleStatementsInMultiStatementMode(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port, "x-multi-statement=true")
|
||||
p := &Postgres{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
if err := d.Run(strings.NewReader("CREATE TABLE foo (foo text); CREATE INDEX CONCURRENTLY idx_foo ON foo (foo);")); err != nil {
|
||||
t.Fatalf("expected err to be nil, got %v", err)
|
||||
}
|
||||
|
||||
// make sure created index exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM pg_indexes WHERE schemaname = (SELECT current_schema()) AND indexname = 'idx_foo')").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table bar to exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testErrorParsing(t *testing.T) {
|
||||
func TestErrorParsing(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -253,7 +184,7 @@ func testErrorParsing(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testFilterCustomQuery(t *testing.T) {
|
||||
func TestFilterCustomQuery(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -275,7 +206,7 @@ func testFilterCustomQuery(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testWithSchema(t *testing.T) {
|
||||
func TestWithSchema(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -345,222 +276,7 @@ func testWithSchema(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testMigrationTableOption(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
p := &Postgres{}
|
||||
d, _ := p.Open(addr)
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create migrate schema
|
||||
if err := d.Run(strings.NewReader("CREATE SCHEMA migrate AUTHORIZATION postgres")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// bad unquoted x-migrations-table parameter
|
||||
wantErr := "x-migrations-table must be quoted (for instance '\"migrate\".\"schema_migrations\"') when x-migrations-table-quoted is enabled, current value is: migrate.schema_migrations"
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if (err != nil) && (err.Error() != wantErr) {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
|
||||
// too many quoted x-migrations-table parameters
|
||||
wantErr = "\"\"migrate\".\"schema_migrations\".\"toomany\"\" MigrationsTable contains too many dot characters"
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\".\"toomany\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if (err != nil) && (err.Error() != wantErr) {
|
||||
t.Fatalf("expected '%s' but got '%s'", wantErr, err.Error())
|
||||
}
|
||||
|
||||
// good quoted x-migrations-table parameter
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"migrate\".\"schema_migrations\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// make sure migrate.schema_migrations table exists
|
||||
var exists bool
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'schema_migrations' AND table_schema = 'migrate')").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table migrate.schema_migrations to exist")
|
||||
}
|
||||
|
||||
d, err = p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=migrate.schema_migrations",
|
||||
pgPassword, ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := d.(*Postgres).conn.QueryRowContext(context.Background(), "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'migrate.schema_migrations' AND table_schema = (SELECT current_schema()))").Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatalf("expected table 'migrate.schema_migrations' to exist")
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func testFailToCreateTableWithoutPermissions(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
|
||||
// Check that opening the postgres connection returns NilVersion
|
||||
p := &Postgres{}
|
||||
|
||||
d, err := p.Open(addr)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine
|
||||
// since this is a test environment and we're not expecting to the pgPassword to be malicious
|
||||
mustRun(t, d, []string{
|
||||
"CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'",
|
||||
"CREATE SCHEMA barfoo AUTHORIZATION postgres",
|
||||
"GRANT USAGE ON SCHEMA barfoo TO not_owner",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
defer func() {
|
||||
if d2 == nil {
|
||||
return
|
||||
}
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
var e *database.Error
|
||||
if !errors.As(err, &e) || err == nil {
|
||||
t.Fatal("Unexpected error, want permission denied error. Got: ", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
|
||||
// re-connect using that x-migrations-table and x-migrations-table-quoted
|
||||
d2, err = p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&x-migrations-table=\"barfoo\".\"schema_migrations\"&x-migrations-table-quoted=1",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if !errors.As(err, &e) || err == nil {
|
||||
t.Fatal("Unexpected error, want permission denied error. Got: ", err)
|
||||
}
|
||||
|
||||
if !strings.Contains(e.OrigErr.Error(), "permission denied for schema barfoo") {
|
||||
t.Fatal(e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testCheckBeforeCreateTable(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := pgConnectionString(ip, port)
|
||||
|
||||
// Check that opening the postgres connection returns NilVersion
|
||||
p := &Postgres{}
|
||||
|
||||
d, err := p.Open(addr)
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
// create user who is not the owner. Although we're concatenating strings in an sql statement it should be fine
|
||||
// since this is a test environment and we're not expecting to the pgPassword to be malicious
|
||||
mustRun(t, d, []string{
|
||||
"CREATE USER not_owner WITH ENCRYPTED PASSWORD '" + pgPassword + "'",
|
||||
"CREATE SCHEMA barfoo AUTHORIZATION postgres",
|
||||
"GRANT USAGE ON SCHEMA barfoo TO not_owner",
|
||||
"GRANT CREATE ON SCHEMA barfoo TO not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := d2.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// revoke privileges
|
||||
mustRun(t, d, []string{
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM PUBLIC",
|
||||
"REVOKE CREATE ON SCHEMA barfoo FROM not_owner",
|
||||
})
|
||||
|
||||
// re-connect using that schema
|
||||
d3, err := p.Open(fmt.Sprintf("postgres://not_owner:%s@%v:%v/postgres?sslmode=disable&search_path=barfoo",
|
||||
pgPassword, ip, port))
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
version, _, err := d3.Version()
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if version != database.NilVersion {
|
||||
t.Fatal("Unexpected version, want database.NilVersion. Got: ", version)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := d3.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func testParallelSchema(t *testing.T) {
|
||||
func TestParallelSchema(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -628,7 +344,11 @@ func testParallelSchema(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testPostgresLock(t *testing.T) {
|
||||
func TestWithInstance(t *testing.T) {
|
||||
|
||||
}
|
||||
|
||||
func TestPostgres_Lock(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -668,7 +388,7 @@ func testPostgresLock(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func testWithInstanceConcurrent(t *testing.T) {
|
||||
func TestWithInstance_Concurrent(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
@@ -710,44 +430,6 @@ func testWithInstanceConcurrent(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testWithConnection(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.FirstPort()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("postgres", pgConnectionString(ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
ctx := context.Background()
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
p, err := WithConnection(ctx, conn, &Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := p.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
dt.Test(t, p, []byte("SELECT 1"))
|
||||
})
|
||||
}
|
||||
|
||||
func Test_computeLineFromPos(t *testing.T) {
|
||||
testcases := []struct {
|
||||
pos int
|
||||
@@ -798,10 +480,10 @@ func Test_computeLineFromPos(t *testing.T) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
input := tc.input
|
||||
if crlf {
|
||||
input = strings.ReplaceAll(input, "\n", "\r\n")
|
||||
input = strings.Replace(input, "\n", "\r\n", -1)
|
||||
}
|
||||
if nonASCII {
|
||||
input = strings.ReplaceAll(input, "FROM", "FRÖM")
|
||||
input = strings.Replace(input, "FROM", "FRÖM", -1)
|
||||
}
|
||||
gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos)
|
||||
|
||||
@@ -826,4 +508,5 @@ func Test_computeLineFromPos(t *testing.T) {
|
||||
run(true, true)
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+19
-17
@@ -2,12 +2,13 @@ package ql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
nurl "net/url"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
@@ -33,7 +34,7 @@ type Config struct {
|
||||
|
||||
type Ql struct {
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
|
||||
config *Config
|
||||
}
|
||||
@@ -71,7 +72,11 @@ func (m *Ql) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := m.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -128,10 +133,9 @@ func (m *Ql) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
tableNames := make([]string, 0)
|
||||
for tables.Next() {
|
||||
var tableName string
|
||||
@@ -144,10 +148,6 @@ func (m *Ql) Drop() (err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
for _, t := range tableNames {
|
||||
query := "DROP TABLE " + t
|
||||
@@ -161,19 +161,21 @@ func (m *Ql) Drop() (err error) {
|
||||
return nil
|
||||
}
|
||||
func (m *Ql) Lock() error {
|
||||
if !m.isLocked.CompareAndSwap(false, true) {
|
||||
if m.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
m.isLocked = true
|
||||
return nil
|
||||
}
|
||||
func (m *Ql) Unlock() error {
|
||||
if !m.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
if !m.isLocked {
|
||||
return nil
|
||||
}
|
||||
m.isLocked = false
|
||||
return nil
|
||||
}
|
||||
func (m *Ql) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -188,7 +190,7 @@ func (m *Ql) executeQuery(query string) error {
|
||||
}
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -216,7 +218,7 @@ func (m *Ql) SetVersion(version int, dirty bool) error {
|
||||
m.config.MigrationsTable)
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
+20
-2
@@ -3,6 +3,8 @@ package ql
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
@@ -13,7 +15,15 @@ import (
|
||||
)
|
||||
|
||||
func Test(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dir, err := ioutil.TempDir("", "ql-driver-test")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "ql.db"))
|
||||
p := &Ql{}
|
||||
addr := fmt.Sprintf("ql://%s", filepath.Join(dir, "ql.db"))
|
||||
@@ -35,7 +45,15 @@ func Test(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
dir, err := ioutil.TempDir("", "ql-driver-test")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "ql.db"))
|
||||
|
||||
db, err := sql.Open("ql", filepath.Join(dir, "ql.db"))
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
//go:build go1.9
|
||||
// +build go1.9
|
||||
|
||||
package redshift
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
@@ -36,7 +36,7 @@ type Config struct {
|
||||
}
|
||||
|
||||
type Redshift struct {
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
|
||||
@@ -126,21 +126,20 @@ func (p *Redshift) Close() error {
|
||||
|
||||
// Redshift does not support advisory lock functions: https://docs.aws.amazon.com/redshift/latest/dg/c_unsupported-postgresql-functions.html
|
||||
func (p *Redshift) Lock() error {
|
||||
if !p.isLocked.CompareAndSwap(false, true) {
|
||||
if p.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
p.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Redshift) Unlock() error {
|
||||
if !p.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
p.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Redshift) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -174,7 +173,7 @@ func (p *Redshift) Run(migration io.Reader) error {
|
||||
|
||||
func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) {
|
||||
// replace crlf with lf
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
s = strings.Replace(s, "\r\n", "\n", -1)
|
||||
// pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes
|
||||
runes := []rune(s)
|
||||
if pos > len(runes) {
|
||||
@@ -216,7 +215,7 @@ func (p *Redshift) SetVersion(version int, dirty bool) error {
|
||||
query := `DELETE FROM "` + p.config.MigrationsTable + `"`
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -228,7 +227,7 @@ func (p *Redshift) SetVersion(version int, dirty bool) error {
|
||||
query = `INSERT INTO "` + p.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)`
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -270,7 +269,7 @@ func (p *Redshift) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -285,9 +284,6 @@ func (p *Redshift) Drop() (err error) {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// delete one by one ...
|
||||
@@ -312,7 +308,11 @@ func (p *Redshift) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := p.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -28,19 +28,10 @@ import (
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
)
|
||||
|
||||
const (
|
||||
pgPassword = "redshift"
|
||||
)
|
||||
|
||||
var (
|
||||
opts = dktest.Options{
|
||||
Env: map[string]string{"POSTGRES_PASSWORD": pgPassword},
|
||||
PortRequired: true,
|
||||
ReadyFunc: isReady,
|
||||
}
|
||||
|
||||
opts = dktest.Options{PortRequired: true, ReadyFunc: isReady}
|
||||
specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "migrate/postgres8:8", Options: opts},
|
||||
{ImageName: "postgres:8", Options: opts},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -53,7 +44,7 @@ func pgConnectionString(host, port string) string {
|
||||
}
|
||||
|
||||
func connectionString(schema, host, port string) string {
|
||||
return fmt.Sprintf("%s://postgres:%s@%s:%s/postgres?sslmode=disable", schema, pgPassword, host, port)
|
||||
return fmt.Sprintf("%s://postgres@%s:%s/postgres?sslmode=disable", schema, host, port)
|
||||
}
|
||||
|
||||
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
@@ -201,7 +192,7 @@ func TestFilterCustomQuery(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&x-custom=foobar", pgPassword, ip, port)
|
||||
addr := fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&x-custom=foobar", ip, port)
|
||||
p := &Redshift{}
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
@@ -243,7 +234,7 @@ func TestWithSchema(t *testing.T) {
|
||||
}
|
||||
|
||||
// re-connect using that schema
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://postgres:%s@%v:%v/postgres?sslmode=disable&search_path=foobar", pgPassword, ip, port))
|
||||
d2, err := p.Open(fmt.Sprintf("postgres://postgres@%v:%v/postgres?sslmode=disable&search_path=foobar", ip, port))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -378,10 +369,10 @@ func Test_computeLineFromPos(t *testing.T) {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
input := tc.input
|
||||
if crlf {
|
||||
input = strings.ReplaceAll(input, "\n", "\r\n")
|
||||
input = strings.Replace(input, "\n", "\r\n", -1)
|
||||
}
|
||||
if nonASCII {
|
||||
input = strings.ReplaceAll(input, "FROM", "FRÖM")
|
||||
input = strings.Replace(input, "FROM", "FRÖM", -1)
|
||||
}
|
||||
gotLine, gotCol, gotOK := computeLineFromPos(input, tc.pos)
|
||||
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
# rqlite
|
||||
|
||||
`rqlite://admin:secret@server1.example.com:4001/?level=strong&timeout=5`
|
||||
|
||||
The `rqlite` url scheme is used for both secure and insecure connections. If connecting to an insecure database, pass `x-connect-insecure` in your URL query, or use `WithInstance` to pass an established connection.
|
||||
|
||||
The migrations table name is configurable through the `x-migrations-table` URL query parameter, or by using `WithInstance` and passing `MigrationsTable` through `Config`.
|
||||
|
||||
Other connect parameters are directly passed through to the database driver. For examples of connection strings, see https://github.com/rqlite/gorqlite#examples.
|
||||
|
||||
| URL Query | WithInstance Config | Description |
|
||||
|------------|---------------------|-------------|
|
||||
| `x-connect-insecure` | n/a: set on instance | Boolean to indicate whether to use an insecure connection. Defaults to `false`. |
|
||||
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table. Defaults to `schema_migrations`. |
|
||||
|
||||
## Notes
|
||||
|
||||
* Uses the https://github.com/rqlite/gorqlite driver
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS pets;
|
||||
@@ -1,3 +0,0 @@
|
||||
CREATE TABLE pets (
|
||||
name string
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS pets;
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE pets ADD predator bool;
|
||||
@@ -1,328 +0,0 @@
|
||||
package rqlite
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/rqlite/gorqlite"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Register("rqlite", &Rqlite{})
|
||||
}
|
||||
|
||||
const (
|
||||
// DefaultMigrationsTable defines the default rqlite migrations table
|
||||
DefaultMigrationsTable = "schema_migrations"
|
||||
|
||||
// DefaultConnectInsecure defines the default setting for connect insecure
|
||||
DefaultConnectInsecure = false
|
||||
)
|
||||
|
||||
// ErrNilConfig is returned if no configuration was passed to WithInstance
|
||||
var ErrNilConfig = fmt.Errorf("no config")
|
||||
|
||||
// ErrBadConfig is returned if configuration was invalid
|
||||
var ErrBadConfig = fmt.Errorf("bad parameter")
|
||||
|
||||
// Config defines the driver configuration
|
||||
type Config struct {
|
||||
// ConnectInsecure sets whether the connection uses TLS. Ineffectual when using WithInstance
|
||||
ConnectInsecure bool
|
||||
// MigrationsTable configures the migrations table name
|
||||
MigrationsTable string
|
||||
}
|
||||
|
||||
type Rqlite struct {
|
||||
db *gorqlite.Connection
|
||||
isLocked atomic.Bool
|
||||
|
||||
config *Config
|
||||
}
|
||||
|
||||
// WithInstance creates a rqlite database driver with an existing gorqlite database connection
|
||||
// and a Config struct
|
||||
func WithInstance(instance *gorqlite.Connection, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
// we use the consistency level check as a database ping
|
||||
if _, err := instance.ConsistencyLevel(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(config.MigrationsTable) == 0 {
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
driver := &Rqlite{
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := driver.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return driver, nil
|
||||
}
|
||||
|
||||
// OpenURL creates a rqlite database driver from a connect URL
|
||||
func OpenURL(url string) (database.Driver, error) {
|
||||
d := &Rqlite{}
|
||||
return d.Open(url)
|
||||
}
|
||||
|
||||
func (r *Rqlite) ensureVersionTable() (err error) {
|
||||
if err = r.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := r.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
}
|
||||
}()
|
||||
|
||||
stmts := []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (version uint64, dirty bool)`, r.config.MigrationsTable),
|
||||
fmt.Sprintf(`CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version)`, r.config.MigrationsTable),
|
||||
}
|
||||
|
||||
if _, err := r.db.Write(stmts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Open returns a new driver instance configured with parameters
|
||||
// coming from the URL string. Migrate will call this function
|
||||
// only once per instance.
|
||||
func (r *Rqlite) Open(url string) (database.Driver, error) {
|
||||
dburl, config, err := parseUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.config = config
|
||||
|
||||
r.db, err = gorqlite.Open(dburl.String())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := r.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Close closes the underlying database instance managed by the driver.
|
||||
// Migrate will call this function only once per instance.
|
||||
func (r *Rqlite) Close() error {
|
||||
r.db.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Lock should acquire a database lock so that only one migration process
|
||||
// can run at a time. Migrate will call this function before Run is called.
|
||||
// If the implementation can't provide this functionality, return nil.
|
||||
// Return database.ErrLocked if database is already locked.
|
||||
func (r *Rqlite) Lock() error {
|
||||
if !r.isLocked.CompareAndSwap(false, true) {
|
||||
return database.ErrLocked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unlock should release the lock. Migrate will call this function after
|
||||
// all migrations have been run.
|
||||
func (r *Rqlite) Unlock() error {
|
||||
if !r.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run applies a migration to the database. migration is guaranteed to be not nil.
|
||||
func (r *Rqlite) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := string(migr[:])
|
||||
if _, err := r.db.WriteOne(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetVersion saves version and dirty state.
|
||||
// Migrate will call this function before and after each call to Run.
|
||||
// version must be >= -1. -1 means NilVersion.
|
||||
func (r *Rqlite) SetVersion(version int, dirty bool) error {
|
||||
deleteQuery := fmt.Sprintf(`DELETE FROM %s`, r.config.MigrationsTable)
|
||||
statements := []gorqlite.ParameterizedStatement{
|
||||
{
|
||||
Query: deleteQuery,
|
||||
},
|
||||
}
|
||||
|
||||
// Also re-write the schema version for nil dirty versions to prevent
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
insertQuery := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, r.config.MigrationsTable)
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
statements = append(statements, gorqlite.ParameterizedStatement{
|
||||
Query: insertQuery,
|
||||
Arguments: []interface{}{
|
||||
version,
|
||||
dirty,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
wr, err := r.db.WriteParameterized(statements)
|
||||
if err != nil {
|
||||
for i, res := range wr {
|
||||
if res.Err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(statements[i].Query)}
|
||||
}
|
||||
}
|
||||
|
||||
// if somehow we're still here, return the original error with combined queries
|
||||
return &database.Error{OrigErr: err, Query: []byte(deleteQuery + "\n" + insertQuery)}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Version returns the currently active version and if the database is dirty.
|
||||
// When no migration has been applied, it must return version -1.
|
||||
// Dirty means, a previous migration failed and user interaction is required.
|
||||
func (r *Rqlite) Version() (version int, dirty bool, err error) {
|
||||
query := "SELECT version, dirty FROM " + r.config.MigrationsTable + " LIMIT 1"
|
||||
|
||||
qr, err := r.db.QueryOne(query)
|
||||
if err != nil {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
|
||||
if !qr.Next() {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
|
||||
if err := qr.Scan(&version, &dirty); err != nil {
|
||||
return database.NilVersion, false, &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
return version, dirty, nil
|
||||
}
|
||||
|
||||
// Drop deletes everything in the database.
|
||||
// Note that this is a breaking action, a new call to Open() is necessary to
|
||||
// ensure subsequent calls work as expected.
|
||||
func (r *Rqlite) Drop() error {
|
||||
query := `SELECT name FROM sqlite_master WHERE type = 'table'`
|
||||
|
||||
tables, err := r.db.QueryOne(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
statements := make([]string, 0)
|
||||
for tables.Next() {
|
||||
var tableName string
|
||||
if err := tables.Scan(&tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(tableName) > 0 {
|
||||
statement := fmt.Sprintf(`DROP TABLE %s`, tableName)
|
||||
statements = append(statements, statement)
|
||||
}
|
||||
}
|
||||
|
||||
// return if nothing to do
|
||||
if len(statements) <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
wr, err := r.db.Write(statements)
|
||||
if err != nil {
|
||||
for i, res := range wr {
|
||||
if res.Err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(statements[i])}
|
||||
}
|
||||
}
|
||||
|
||||
// if somehow we're still here, return the original error with combined queries
|
||||
return &database.Error{OrigErr: err, Query: []byte(strings.Join(statements, "\n"))}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseUrl(url string) (*nurl.URL, *Config, error) {
|
||||
parsedUrl, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
config, err := parseConfigFromQuery(parsedUrl.Query())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if parsedUrl.Scheme != "rqlite" {
|
||||
return nil, nil, fmt.Errorf("bad scheme: %w", ErrBadConfig)
|
||||
}
|
||||
|
||||
// adapt from rqlite to http/https schemes
|
||||
if config.ConnectInsecure {
|
||||
parsedUrl.Scheme = "http"
|
||||
} else {
|
||||
parsedUrl.Scheme = "https"
|
||||
}
|
||||
|
||||
filteredUrl := migrate.FilterCustomQuery(parsedUrl)
|
||||
|
||||
return filteredUrl, config, nil
|
||||
}
|
||||
|
||||
func parseConfigFromQuery(queryVals nurl.Values) (*Config, error) {
|
||||
c := Config{
|
||||
ConnectInsecure: DefaultConnectInsecure,
|
||||
MigrationsTable: DefaultMigrationsTable,
|
||||
}
|
||||
|
||||
migrationsTable := queryVals.Get("x-migrations-table")
|
||||
if migrationsTable != "" {
|
||||
if strings.HasPrefix(migrationsTable, "sqlite_") {
|
||||
return nil, fmt.Errorf("invalid value for x-migrations-table: %w", ErrBadConfig)
|
||||
}
|
||||
c.MigrationsTable = migrationsTable
|
||||
}
|
||||
|
||||
connectInsecureStr := queryVals.Get("x-connect-insecure")
|
||||
if connectInsecureStr != "" {
|
||||
connectInsecure, err := strconv.ParseBool(connectInsecureStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid value for x-connect-insecure: %w", ErrBadConfig)
|
||||
}
|
||||
c.ConnectInsecure = connectInsecure
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
@@ -1,324 +0,0 @@
|
||||
package rqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/dhui/dktest"
|
||||
"github.com/rqlite/gorqlite"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
"github.com/golang-migrate/migrate/v4/dktesting"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
)
|
||||
|
||||
var defaultPort uint16 = 4001
|
||||
|
||||
var opts = dktest.Options{
|
||||
Env: map[string]string{"NODE_ID": "1"},
|
||||
PortRequired: true,
|
||||
ReadyFunc: isReady,
|
||||
}
|
||||
var specs = []dktesting.ContainerSpec{
|
||||
{ImageName: "rqlite/rqlite:7.21.4", Options: opts},
|
||||
{ImageName: "rqlite/rqlite:8.0.6", Options: opts},
|
||||
{ImageName: "rqlite/rqlite:8.11.1", Options: opts},
|
||||
{ImageName: "rqlite/rqlite:8.12.3", Options: opts},
|
||||
}
|
||||
|
||||
func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
if err != nil {
|
||||
fmt.Println("error getting port")
|
||||
return false
|
||||
}
|
||||
|
||||
statusString := fmt.Sprintf("http://%s:%s/status", ip, port)
|
||||
fmt.Println(statusString)
|
||||
|
||||
var readyResp struct {
|
||||
Store struct {
|
||||
Ready bool `json:"ready"`
|
||||
} `json:"store"`
|
||||
}
|
||||
|
||||
resp, err := http.Get(statusString)
|
||||
if err != nil {
|
||||
fmt.Println("error getting status")
|
||||
return false
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
fmt.Println("statusCode != 200")
|
||||
return false
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Println("error reading body")
|
||||
return false
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(body, &readyResp); err != nil {
|
||||
fmt.Println("error unmarshaling body")
|
||||
return false
|
||||
}
|
||||
|
||||
return readyResp.Store.Ready
|
||||
}
|
||||
|
||||
func Test(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
connectString := fmt.Sprintf("rqlite://%s:%s?level=strong&disableClusterDiscovery=true&x-connect-insecure=true", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
|
||||
r := &Rqlite{}
|
||||
d, err := r.Open(connectString)
|
||||
assert.NoError(t, err)
|
||||
|
||||
dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);"))
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
connectString := fmt.Sprintf("rqlite://%s:%s?level=strong&disableClusterDiscovery=true&x-connect-insecure=true", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
|
||||
driver, err := OpenURL(connectString)
|
||||
assert.NoError(t, err)
|
||||
defer func() {
|
||||
if err := driver.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
"file://./examples/migrations",
|
||||
"ql", driver)
|
||||
assert.NoError(t, err)
|
||||
|
||||
dt.TestMigrate(t, m)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBadConnectInsecureParam(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
connectString := fmt.Sprintf("rqlite://%s:%s?x-connect-insecure=foo", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
|
||||
_, err = OpenURL(connectString)
|
||||
assert.ErrorIs(t, err, ErrBadConfig)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBadProtocol(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
connectString := fmt.Sprintf("postgres://%s:%s/database", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
|
||||
_, err = OpenURL(connectString)
|
||||
assert.ErrorIs(t, err, ErrBadConfig)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNoConfig(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// gorqlite expects http(s) schemes
|
||||
connectString := fmt.Sprintf("http://%s:%s?level=strong&disableClusterDiscovery=true", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
db, err := gorqlite.Open(connectString)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = WithInstance(db, nil)
|
||||
assert.ErrorIs(t, err, ErrNilConfig)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWithInstanceEmptyConfig(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// gorqlite expects http(s) schemes
|
||||
connectString := fmt.Sprintf("http://%s:%s?level=strong&disableClusterDiscovery=true", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
db, err := gorqlite.Open(connectString)
|
||||
assert.NoError(t, err)
|
||||
|
||||
driver, err := WithInstance(db, &Config{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
if err := driver.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
"file://./examples/migrations",
|
||||
"ql", driver)
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("UP")
|
||||
err = m.Up()
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = db.QueryOne(fmt.Sprintf("SELECT * FROM %s", DefaultMigrationsTable))
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("DOWN")
|
||||
err = m.Down()
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigrationTable(t *testing.T) {
|
||||
dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
|
||||
ip, port, err := c.Port(defaultPort)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// gorqlite expects http(s) schemes
|
||||
connectString := fmt.Sprintf("http://%s:%s?level=strong&disableClusterDiscovery=true", ip, port)
|
||||
t.Logf("DB connect string : %s\n", connectString)
|
||||
db, err := gorqlite.Open(connectString)
|
||||
assert.NoError(t, err)
|
||||
|
||||
config := Config{MigrationsTable: "my_migration_table"}
|
||||
driver, err := WithInstance(db, &config)
|
||||
assert.NoError(t, err)
|
||||
|
||||
defer func() {
|
||||
if err := driver.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
"file://./examples/migrations",
|
||||
"ql", driver)
|
||||
assert.NoError(t, err)
|
||||
|
||||
t.Log("UP")
|
||||
err = m.Up()
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = db.QueryOne(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable))
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = db.WriteOne(`INSERT INTO pets (name, predator) VALUES ("franklin", true)`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
res, err := db.QueryOne(`SELECT name, predator FROM pets LIMIT 1`)
|
||||
assert.NoError(t, err)
|
||||
|
||||
_ = res.Next()
|
||||
|
||||
// make sure we can use the migrated table
|
||||
var petName string
|
||||
var petPredator int
|
||||
err = res.Scan(&petName, &petPredator)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, petName, "franklin")
|
||||
assert.Equal(t, petPredator, 1)
|
||||
|
||||
t.Log("DOWN")
|
||||
err = m.Down()
|
||||
assert.NoError(t, err)
|
||||
|
||||
_, err = db.QueryOne(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable))
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseUrl(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
passedUrl string
|
||||
expectedUrl string
|
||||
expectedConfig *Config
|
||||
expectedErr string
|
||||
}{
|
||||
{
|
||||
"defaults",
|
||||
"rqlite://localhost:4001",
|
||||
"https://localhost:4001",
|
||||
&Config{ConnectInsecure: DefaultConnectInsecure, MigrationsTable: DefaultMigrationsTable},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"configure migration table",
|
||||
"rqlite://localhost:4001?x-migrations-table=foo",
|
||||
"https://localhost:4001",
|
||||
&Config{ConnectInsecure: DefaultConnectInsecure, MigrationsTable: "foo"},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"configure connect insecure",
|
||||
"rqlite://localhost:4001?x-connect-insecure=true",
|
||||
"http://localhost:4001",
|
||||
&Config{ConnectInsecure: true, MigrationsTable: DefaultMigrationsTable},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"invalid migration table",
|
||||
"rqlite://localhost:4001?x-migrations-table=sqlite_bar",
|
||||
"",
|
||||
nil,
|
||||
"invalid value for x-migrations-table: bad parameter",
|
||||
},
|
||||
{
|
||||
"invalid connect insecure",
|
||||
"rqlite://localhost:4001?x-connect-insecure=baz",
|
||||
"",
|
||||
nil,
|
||||
"invalid value for x-connect-insecure: bad parameter",
|
||||
},
|
||||
{
|
||||
"invalid url",
|
||||
string([]byte{0x7f}),
|
||||
"",
|
||||
nil,
|
||||
"parse \"\\x7f\": net/url: invalid control character in URL",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
actualUrl, actualConfig, actualErr := parseUrl(tt.passedUrl)
|
||||
if tt.expectedUrl != "" {
|
||||
assert.Equal(t, tt.expectedUrl, actualUrl.String())
|
||||
} else {
|
||||
assert.Nil(t, actualUrl)
|
||||
}
|
||||
|
||||
assert.Equal(t, tt.expectedConfig, actualConfig)
|
||||
|
||||
if tt.expectedErr == "" {
|
||||
assert.NoError(t, actualErr)
|
||||
} else {
|
||||
assert.EqualError(t, actualErr, tt.expectedErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -3,15 +3,15 @@ package snowflake
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
nurl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/lib/pq"
|
||||
sf "github.com/snowflakedb/gosnowflake"
|
||||
)
|
||||
@@ -37,7 +37,7 @@ type Config struct {
|
||||
}
|
||||
|
||||
type Snowflake struct {
|
||||
isLocked atomic.Bool
|
||||
isLocked bool
|
||||
conn *sql.Conn
|
||||
db *sql.DB
|
||||
|
||||
@@ -158,21 +158,20 @@ func (p *Snowflake) Close() error {
|
||||
}
|
||||
|
||||
func (p *Snowflake) Lock() error {
|
||||
if !p.isLocked.CompareAndSwap(false, true) {
|
||||
if p.isLocked {
|
||||
return database.ErrLocked
|
||||
}
|
||||
p.isLocked = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Snowflake) Unlock() error {
|
||||
if !p.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
p.isLocked = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Snowflake) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -206,7 +205,7 @@ func (p *Snowflake) Run(migration io.Reader) error {
|
||||
|
||||
func computeLineFromPos(s string, pos int) (line uint, col uint, ok bool) {
|
||||
// replace crlf with lf
|
||||
s = strings.ReplaceAll(s, "\r\n", "\n")
|
||||
s = strings.Replace(s, "\r\n", "\n", -1)
|
||||
// pg docs: pos uses index 1 for the first character, and positions are measured in characters not bytes
|
||||
runes := []rune(s)
|
||||
if pos > len(runes) {
|
||||
@@ -248,7 +247,7 @@ func (p *Snowflake) SetVersion(version int, dirty bool) error {
|
||||
query := `DELETE FROM "` + p.config.MigrationsTable + `"`
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -262,7 +261,7 @@ func (p *Snowflake) SetVersion(version int, dirty bool) error {
|
||||
` + strconv.FormatBool(dirty) + `)`
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
err = multierror.Append(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
@@ -304,7 +303,7 @@ func (p *Snowflake) Drop() (err error) {
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
err = multierror.Append(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -319,9 +318,6 @@ func (p *Snowflake) Drop() (err error) {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
// delete one by one ...
|
||||
@@ -346,7 +342,11 @@ func (p *Snowflake) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := p.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -2,14 +2,12 @@
|
||||
|
||||
## Usage
|
||||
|
||||
See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for
|
||||
more details.
|
||||
|
||||
The DSN must be given in the following format.
|
||||
|
||||
`spanner://projects/{projectId}/instances/{instanceId}/databases/{databaseName}?param=true`
|
||||
`spanner://projects/{projectId}/instances/{instanceId}/databases/{databaseName}`
|
||||
|
||||
See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for details.
|
||||
|
||||
as described in [README.md#database-urls](../../README.md#database-urls)
|
||||
|
||||
| Param | WithInstance Config | Description |
|
||||
| ----- | ------------------- | ----------- |
|
||||
@@ -20,6 +18,7 @@ as described in [README.md#database-urls](../../README.md#database-urls)
|
||||
| `instanceId` || The id of the instance running Spanner
|
||||
| `databaseName` || The name of the Spanner database
|
||||
|
||||
|
||||
> **Note:** Google Cloud Spanner migrations can take a considerable amount of
|
||||
> time. The migrations provided as part of the example take about 6 minutes to
|
||||
> run on a small instance.
|
||||
@@ -33,7 +32,7 @@ as described in [README.md#database-urls](../../README.md#database-urls)
|
||||
## DDL with comments
|
||||
|
||||
At the moment the GCP Spanner backed does not seem to allow for comments (See https://issuetracker.google.com/issues/159730604)
|
||||
so in order to be able to use migration with DDL containing comments `x-clean-statements` is required
|
||||
so in order to be able to use migration with DDL containing comments `x-clean-stamements` is required
|
||||
|
||||
## Multiple statements
|
||||
|
||||
@@ -43,4 +42,4 @@ In order to be able to use more than 1 DDL statement in the same migration file,
|
||||
|
||||
To unit test the `spanner` driver, `SPANNER_DATABASE` needs to be set. You'll
|
||||
need to sign-up to Google Cloud Platform (GCP) and have a running Spanner
|
||||
instance since it is not possible to run Google Spanner outside GCP.
|
||||
instance since it is not possible to run Google Spanner outside GCP.
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE Transactions
|
||||
@@ -1,6 +0,0 @@
|
||||
CREATE TABLE Transactions (
|
||||
UserId INT64,
|
||||
TransactionId STRING(40),
|
||||
Total NUMERIC
|
||||
) PRIMARY KEY(UserId, TransactionId),
|
||||
INTERLEAVE IN PARENT Users ON DELETE CASCADE
|
||||
+17
-22
@@ -1,16 +1,16 @@
|
||||
package spanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
nurl "net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"golang.org/x/net/context"
|
||||
|
||||
"cloud.google.com/go/spanner"
|
||||
sdb "cloud.google.com/go/spanner/admin/database/apiv1"
|
||||
@@ -19,8 +19,9 @@ import (
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
|
||||
adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"google.golang.org/api/iterator"
|
||||
adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -33,12 +34,10 @@ const DefaultMigrationsTable = "SchemaMigrations"
|
||||
|
||||
// Driver errors
|
||||
var (
|
||||
ErrNilConfig = errors.New("no config")
|
||||
ErrNoDatabaseName = errors.New("no database name")
|
||||
ErrNoSchema = errors.New("no schema")
|
||||
ErrDatabaseDirty = errors.New("database is dirty")
|
||||
ErrLockHeld = errors.New("unable to obtain lock")
|
||||
ErrLockNotHeld = errors.New("unable to release already released lock")
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
ErrNoSchema = fmt.Errorf("no schema")
|
||||
ErrDatabaseDirty = fmt.Errorf("database is dirty")
|
||||
)
|
||||
|
||||
// Config used for a Spanner instance
|
||||
@@ -57,8 +56,6 @@ type Spanner struct {
|
||||
db *DB
|
||||
|
||||
config *Config
|
||||
|
||||
lock atomic.Bool
|
||||
}
|
||||
|
||||
type DB struct {
|
||||
@@ -146,23 +143,17 @@ func (s *Spanner) Close() error {
|
||||
// Lock implements database.Driver but doesn't do anything because Spanner only
|
||||
// enqueues the UpdateDatabaseDdlRequest.
|
||||
func (s *Spanner) Lock() error {
|
||||
if swapped := s.lock.CompareAndSwap(false, true); swapped {
|
||||
return nil
|
||||
}
|
||||
return ErrLockHeld
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unlock implements database.Driver but no action required, see Lock.
|
||||
func (s *Spanner) Unlock() error {
|
||||
if swapped := s.lock.CompareAndSwap(true, false); swapped {
|
||||
return nil
|
||||
}
|
||||
return ErrLockNotHeld
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run implements database.Driver
|
||||
func (s *Spanner) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
migr, err := ioutil.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -298,7 +289,11 @@ func (s *Spanner) ensureVersionTable() (err error) {
|
||||
|
||||
defer func() {
|
||||
if e := s.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
if err == nil {
|
||||
err = e
|
||||
} else {
|
||||
err = multierror.Append(err, e)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@@ -8,57 +8,52 @@ import (
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
|
||||
"cloud.google.com/go/spanner/spannertest"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator
|
||||
func withSpannerEmulator(t *testing.T, testFunc func(t *testing.T)) {
|
||||
t.Helper()
|
||||
srv, err := spannertest.NewServer("localhost:0")
|
||||
if err != nil {
|
||||
t.Fatal("Failed to create Spanner emulator:", err)
|
||||
}
|
||||
// This is not thread-safe
|
||||
if err := os.Setenv("SPANNER_EMULATOR_HOST", srv.Addr); err != nil {
|
||||
t.Fatal("Failed to set SPANNER_EMULATOR_HOST env var:", err)
|
||||
}
|
||||
defer srv.Close()
|
||||
testFunc(t)
|
||||
|
||||
}
|
||||
|
||||
const db = "projects/abc/instances/def/databases/testdb"
|
||||
|
||||
func Test(t *testing.T) {
|
||||
withSpannerEmulator(t, func(t *testing.T) {
|
||||
uri := fmt.Sprintf("spanner://%s", db)
|
||||
s := &Spanner{}
|
||||
d, err := s.Open(uri)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.Test(t, d, []byte("CREATE TABLE test (id BOOL) PRIMARY KEY (id)"))
|
||||
})
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode.")
|
||||
}
|
||||
|
||||
db, ok := os.LookupEnv("SPANNER_DATABASE")
|
||||
if !ok {
|
||||
t.Skip("SPANNER_DATABASE not set, skipping test.")
|
||||
}
|
||||
|
||||
s := &Spanner{}
|
||||
addr := fmt.Sprintf("spanner://%s", db)
|
||||
d, err := s.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.Test(t, d, []byte("SELECT 1"))
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
withSpannerEmulator(t, func(t *testing.T) {
|
||||
s := &Spanner{}
|
||||
uri := fmt.Sprintf("spanner://%s", db)
|
||||
d, err := s.Open(uri)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", uri, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
})
|
||||
if testing.Short() {
|
||||
t.Skip("skipping test in short mode.")
|
||||
}
|
||||
|
||||
db, ok := os.LookupEnv("SPANNER_DATABASE")
|
||||
if !ok {
|
||||
t.Skip("SPANNER_DATABASE not set, skipping test.")
|
||||
}
|
||||
|
||||
s := &Spanner{}
|
||||
addr := fmt.Sprintf("spanner://%s", db)
|
||||
d, err := s.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", db, d)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
}
|
||||
|
||||
func TestCleanStatements(t *testing.T) {
|
||||
@@ -145,20 +140,6 @@ func TestCleanStatements(t *testing.T) {
|
||||
id STRING(255) NOT NULL,
|
||||
) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
|
||||
},
|
||||
{
|
||||
name: "alter table with SET OPTIONS",
|
||||
multiStatement: `ALTER TABLE users ALTER COLUMN created
|
||||
SET OPTIONS (allow_commit_timestamp=true);`,
|
||||
expected: []string{"ALTER TABLE users ALTER COLUMN created SET OPTIONS (allow_commit_timestamp = true)"},
|
||||
},
|
||||
{
|
||||
name: "column with NUMERIC type",
|
||||
multiStatement: `CREATE TABLE table_name (
|
||||
id STRING(255) NOT NULL,
|
||||
sum NUMERIC,
|
||||
) PRIMARY KEY (id)`,
|
||||
expected: []string{"CREATE TABLE table_name (\n id STRING(255) NOT NULL,\n sum NUMERIC,\n) PRIMARY KEY(id)"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# sqlcipher
|
||||
|
||||
This is just a copy of the [sqlite3](https://github.com/golang-migrate/migrate/blob/master/database/sqlite3) driver except that it imports `github.com/mutecomm/go-sqlcipher`.
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS pets;
|
||||
@@ -1,3 +0,0 @@
|
||||
CREATE TABLE pets (
|
||||
name string
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS pets;
|
||||
@@ -1 +0,0 @@
|
||||
ALTER TABLE pets ADD predator bool;
|
||||
@@ -1,264 +0,0 @@
|
||||
package sqlcipher
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
nurl "net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
_ "github.com/mutecomm/go-sqlcipher/v4"
|
||||
)
|
||||
|
||||
func init() {
|
||||
database.Register("sqlcipher", &Sqlite{})
|
||||
}
|
||||
|
||||
var DefaultMigrationsTable = "schema_migrations"
|
||||
var (
|
||||
ErrDatabaseDirty = fmt.Errorf("database is dirty")
|
||||
ErrNilConfig = fmt.Errorf("no config")
|
||||
ErrNoDatabaseName = fmt.Errorf("no database name")
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
MigrationsTable string
|
||||
DatabaseName string
|
||||
NoTxWrap bool
|
||||
}
|
||||
|
||||
type Sqlite struct {
|
||||
db *sql.DB
|
||||
isLocked atomic.Bool
|
||||
|
||||
config *Config
|
||||
}
|
||||
|
||||
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
||||
if config == nil {
|
||||
return nil, ErrNilConfig
|
||||
}
|
||||
|
||||
if err := instance.Ping(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(config.MigrationsTable) == 0 {
|
||||
config.MigrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
mx := &Sqlite{
|
||||
db: instance,
|
||||
config: config,
|
||||
}
|
||||
if err := mx.ensureVersionTable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mx, nil
|
||||
}
|
||||
|
||||
// ensureVersionTable checks if versions table exists and, if not, creates it.
|
||||
// Note that this function locks the database, which deviates from the usual
|
||||
// convention of "caller locks" in the Sqlite type.
|
||||
func (m *Sqlite) ensureVersionTable() (err error) {
|
||||
if err = m.Lock(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if e := m.Unlock(); e != nil {
|
||||
err = errors.Join(err, e)
|
||||
}
|
||||
}()
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version);
|
||||
`, m.config.MigrationsTable, m.config.MigrationsTable)
|
||||
|
||||
if _, err := m.db.Exec(query); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Open(url string) (database.Driver, error) {
|
||||
purl, err := nurl.Parse(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite3://", "", 1)
|
||||
db, err := sql.Open("sqlite3", dbfile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
qv := purl.Query()
|
||||
|
||||
migrationsTable := qv.Get("x-migrations-table")
|
||||
if len(migrationsTable) == 0 {
|
||||
migrationsTable = DefaultMigrationsTable
|
||||
}
|
||||
|
||||
noTxWrap := false
|
||||
if v := qv.Get("x-no-tx-wrap"); v != "" {
|
||||
noTxWrap, err = strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("x-no-tx-wrap: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
mx, err := WithInstance(db, &Config{
|
||||
DatabaseName: purl.Path,
|
||||
MigrationsTable: migrationsTable,
|
||||
NoTxWrap: noTxWrap,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return mx, nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Close() error {
|
||||
return m.db.Close()
|
||||
}
|
||||
|
||||
func (m *Sqlite) Drop() (err error) {
|
||||
query := `SELECT name FROM sqlite_master WHERE type = 'table';`
|
||||
tables, err := m.db.Query(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
defer func() {
|
||||
if errClose := tables.Close(); errClose != nil {
|
||||
err = errors.Join(err, errClose)
|
||||
}
|
||||
}()
|
||||
|
||||
tableNames := make([]string, 0)
|
||||
for tables.Next() {
|
||||
var tableName string
|
||||
if err := tables.Scan(&tableName); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tableName) > 0 {
|
||||
tableNames = append(tableNames, tableName)
|
||||
}
|
||||
}
|
||||
if err := tables.Err(); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
if len(tableNames) > 0 {
|
||||
for _, t := range tableNames {
|
||||
query := "DROP TABLE " + t
|
||||
err = m.executeQuery(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
query := "VACUUM"
|
||||
_, err = m.db.Query(query)
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Lock() error {
|
||||
if !m.isLocked.CompareAndSwap(false, true) {
|
||||
return database.ErrLocked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Unlock() error {
|
||||
if !m.isLocked.CompareAndSwap(true, false) {
|
||||
return database.ErrNotLocked
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Run(migration io.Reader) error {
|
||||
migr, err := io.ReadAll(migration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
query := string(migr[:])
|
||||
|
||||
if m.config.NoTxWrap {
|
||||
return m.executeQueryNoTx(query)
|
||||
}
|
||||
return m.executeQuery(query)
|
||||
}
|
||||
|
||||
func (m *Sqlite) executeQuery(query string) error {
|
||||
tx, err := m.db.Begin()
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction commit failed"}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) executeQueryNoTx(query string) error {
|
||||
if _, err := m.db.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) SetVersion(version int, dirty bool) error {
|
||||
tx, err := m.db.Begin()
|
||||
if err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction start failed"}
|
||||
}
|
||||
|
||||
query := "DELETE FROM " + m.config.MigrationsTable
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
|
||||
// Also re-write the schema version for nil dirty versions to prevent
|
||||
// empty schema version for failed down migration on the first migration
|
||||
// See: https://github.com/golang-migrate/migrate/issues/330
|
||||
if version >= 0 || (version == database.NilVersion && dirty) {
|
||||
query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, m.config.MigrationsTable)
|
||||
if _, err := tx.Exec(query, version, dirty); err != nil {
|
||||
if errRollback := tx.Rollback(); errRollback != nil {
|
||||
err = errors.Join(err, errRollback)
|
||||
}
|
||||
return &database.Error{OrigErr: err, Query: []byte(query)}
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return &database.Error{OrigErr: err, Err: "transaction commit failed"}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Sqlite) Version() (version int, dirty bool, err error) {
|
||||
query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1"
|
||||
err = m.db.QueryRow(query).Scan(&version, &dirty)
|
||||
if err != nil {
|
||||
return database.NilVersion, false, nil
|
||||
}
|
||||
return version, dirty, nil
|
||||
}
|
||||
@@ -1,120 +0,0 @@
|
||||
package sqlcipher
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
dt "github.com/golang-migrate/migrate/v4/database/testing"
|
||||
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||
_ "github.com/mutecomm/go-sqlcipher/v4"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
|
||||
p := &Sqlite{}
|
||||
addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db"))
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);"))
|
||||
}
|
||||
|
||||
func TestMigrate(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
|
||||
|
||||
db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
driver, err := WithInstance(db, &Config{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
"file://./examples/migrations",
|
||||
"ql", driver)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dt.TestMigrate(t, m)
|
||||
}
|
||||
|
||||
func TestMigrationTable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
|
||||
|
||||
db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := db.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
config := &Config{
|
||||
MigrationsTable: "my_migration_table",
|
||||
}
|
||||
driver, err := WithInstance(db, config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, err := migrate.NewWithDatabaseInstance(
|
||||
"file://./examples/migrations",
|
||||
"ql", driver)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Log("UP")
|
||||
err = m.Up()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = db.Query(fmt.Sprintf("SELECT * FROM %s", config.MigrationsTable))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoTxWrap(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
|
||||
p := &Sqlite{}
|
||||
addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=true", filepath.Join(dir, "sqlite3.db"))
|
||||
d, err := p.Open(addr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// An explicit BEGIN statement would ordinarily fail without x-no-tx-wrap.
|
||||
// (Transactions in sqlite may not be nested.)
|
||||
dt.Test(t, d, []byte("BEGIN; CREATE TABLE t (Qty int, Name string); COMMIT;"))
|
||||
}
|
||||
|
||||
func TestNoTxWrapInvalidValue(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Logf("DB path : %s\n", filepath.Join(dir, "sqlite3.db"))
|
||||
p := &Sqlite{}
|
||||
addr := fmt.Sprintf("sqlite3://%s?x-no-tx-wrap=yeppers", filepath.Join(dir, "sqlite3.db"))
|
||||
_, err := p.Open(addr)
|
||||
if assert.Error(t, err) {
|
||||
assert.Contains(t, err.Error(), "x-no-tx-wrap")
|
||||
assert.Contains(t, err.Error(), "invalid syntax")
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
# sqlite
|
||||
|
||||
`sqlite://path/to/database?query`
|
||||
|
||||
Unlike other migrate database drivers, the sqlite driver will automatically wrap each migration in an implicit transaction by default. Migrations must not contain explicit `BEGIN` or `COMMIT` statements. This behavior may change in a future major release. (See below for a workaround.)
|
||||
|
||||
The auxiliary query parameters listed below may be supplied to tailor migrate behavior. All auxiliary query parameters are optional.
|
||||
|
||||
| URL Query | WithInstance Config | Description |
|
||||
|------------|---------------------|-------------|
|
||||
| `x-migrations-table` | `MigrationsTable` | Name of the migrations table. Defaults to `schema_migrations`. |
|
||||
| `x-no-tx-wrap` | `NoTxWrap` | Disable implicit transactions when `true`. Migrations may, and should, contain explicit `BEGIN` and `COMMIT` statements. |
|
||||
|
||||
## Notes
|
||||
|
||||
* Uses the `modernc.org/sqlite` sqlite db driver (pure Go)
|
||||
* Has [limited `GOOS` and `GOARCH` support](https://pkg.go.dev/modernc.org/sqlite?utm_source=godoc#hdr-Supported_platforms_and_architectures)
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS pets;
|
||||
@@ -1,3 +0,0 @@
|
||||
CREATE TABLE pets (
|
||||
name string
|
||||
);
|
||||
@@ -1 +0,0 @@
|
||||
DROP TABLE IF EXISTS pets;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user